nexus-ai-pro 2.0.0 → 2.2.0

package/dist/evaluate/run.js

@@ -1,5 +1,7 @@
 import { randomBytes } from 'node:crypto';
+import { NEXUS_VERSION } from '../version.js';
 import { evaluationCacheKey } from './cache.js';
+import { declaredProvenance } from './provenance.js';
 export async function evaluate(target, dataset, evaluators, options = {}) {
     if (dataset.examples.length === 0)
         throw new RangeError(`Dataset "${dataset.name}" has no examples`);
@@ -14,8 +16,9 @@ export async function evaluate(target, dataset, evaluators, options = {}) {
         example,
         run,
     })));
+    const produced = evaluators.map(() => new Set());
     await runConcurrently(jobs, options.concurrency ?? 4, async ({ example, run }) => {
-        const result = await evaluateOne(target, example, run, evaluators, options);
+        const result = await evaluateOne(target, example, run, evaluators, options, produced);
         results.push(result);
         options.onResult?.(result);
     }, options.signal);
@@ -35,6 +38,16 @@ export async function evaluate(target, dataset, evaluators, options = {}) {
         results,
         metrics: summarize(results),
         summary,
+        evaluators: evaluators.map((evaluator, position) => {
+            const declared = declaredProvenance(evaluator);
+            return {
+                ...declared,
+                position,
+                name: declared?.name ?? (evaluator.name || `evaluator-${position}`),
+                keys: [...(produced[position] ?? [])],
+            };
+        }),
+        framework: { name: 'nexus-ai-pro', version: NEXUS_VERSION },
         ...(options.metadata ? { metadata: options.metadata } : {}),
         ...(options.cache
             ? {
@@ -57,7 +70,7 @@ function repetitionsOf(example, setting) {
     }
     return count;
 }
-async function evaluateOne(target, example, run, evaluators, options) {
+async function evaluateOne(target, example, run, evaluators, options, produced) {
     const started = Date.now();
     let output;
     let error;
@@ -97,9 +110,12 @@ async function evaluateOne(target, example, run, evaluators, options) {
         });
     }
     const scores = [];
-    for (const evaluator of evaluators) {
+    for (const [position, evaluator] of evaluators.entries()) {
         try {
-            scores.push(...normalizeScores(await evaluator(context), evaluator));
+            const made = normalizeScores(await evaluator(context), evaluator);
+            for (const score of made)
+                produced[position]?.add(score.key);
+            scores.push(...made);
         }
         catch (caught) {
             scores.push({

package/dist/graph/durability.d.ts

@@ -0,0 +1,27 @@
+import type { DurabilityMode } from '../types/graph.js';
+/**
+ * Writes one run's checkpoints under a durability mode.
+ *
+ * `sync` writes each one before returning. `async` chains writes behind each other, so they land in
+ * order, and makes the run wait only when `maxPending` are already in flight. `exit` keeps only the
+ * latest and writes it when the run stops. A write that fails is kept and thrown from the next call,
+ * so a background failure still fails the run.
+ */
+export declare class CheckpointWriter<T> {
+    private readonly write;
+    private readonly mode;
+    private readonly maxPending;
+    private chain;
+    private readonly inFlight;
+    private failure;
+    private unwritten;
+    constructor(write: (checkpoint: T) => Promise<void>, mode: DurabilityMode, maxPending?: number);
+    /**
+     * Hands over a checkpoint. A `durable` one is persisted, with everything before it, before this
+     * resolves, whatever the mode: a pause, a failure, the end of a run, or a recovery decision.
+     */
+    persist(checkpoint: T, durable?: boolean): Promise<void>;
+    /** Waits for every write handed over so far, and writes what `exit` mode was holding. */
+    flush(): Promise<void>;
+    private throwFailure;
+}

package/dist/graph/durability.js

@@ -0,0 +1,66 @@
+const DEFAULT_MAX_PENDING = 8;
+export class CheckpointWriter {
+    write;
+    mode;
+    maxPending;
+    chain = Promise.resolve();
+    inFlight = [];
+    failure;
+    unwritten;
+    constructor(write, mode, maxPending = DEFAULT_MAX_PENDING) {
+        this.write = write;
+        this.mode = mode;
+        this.maxPending = maxPending;
+    }
+    async persist(checkpoint, durable = false) {
+        this.throwFailure();
+        if (this.mode === 'sync') {
+            await this.write(checkpoint);
+            return;
+        }
+        if (this.mode === 'exit' && !durable) {
+            this.unwritten = checkpoint;
+            return;
+        }
+        if (this.mode === 'exit') {
+            this.unwritten = undefined;
+            await this.write(checkpoint);
+            return;
+        }
+        const link = this.chain
+            .then(() => this.write(checkpoint))
+            .catch((error) => {
+            this.failure ??= { error };
+        })
+            .finally(() => {
+            const index = this.inFlight.indexOf(link);
+            if (index >= 0)
+                this.inFlight.splice(index, 1);
+        });
+        this.chain = link;
+        this.inFlight.push(link);
+        if (durable) {
+            await this.flush();
+            return;
+        }
+        while (this.inFlight.length > Math.max(1, this.maxPending))
+            await this.inFlight[0];
+        this.throwFailure();
+    }
+    async flush() {
+        await this.chain;
+        if (this.unwritten !== undefined) {
+            const latest = this.unwritten;
+            this.unwritten = undefined;
+            await this.write(latest);
+        }
+        this.throwFailure();
+    }
+    throwFailure() {
+        if (!this.failure)
+            return;
+        const { error } = this.failure;
+        this.failure = undefined;
+        throw error;
+    }
+}

package/dist/graph/errors.d.ts

@@ -98,7 +98,8 @@ export declare class GraphNotInterruptedError extends GraphError {
     status: string);
 }
 /**
- * Raised when a node outlives its `timeoutMs`.
+ * Raised when a node outlives its run limit, or goes longer than its idle limit without showing
+ * progress.
  *
  * Separate from an ordinary failure because it is usually worth retrying, and because the node's
  * signal was aborted underneath it rather than the node choosing to stop.
@@ -106,13 +107,38 @@ export declare class GraphNotInterruptedError extends GraphError {
 export declare class GraphNodeTimeoutError extends GraphError {
     /** The node. */
     readonly node: string;
-    /** The timeout it exceeded, in milliseconds. */
+    /** The limit it exceeded, in milliseconds. */
     readonly timeoutMs: number;
+    /** `run` for the whole-attempt limit, `idle` for the limit between signs of progress. */
+    readonly kind: 'run' | 'idle';
     constructor(
     /** The node. */
     node: string, 
-    /** The timeout it exceeded, in milliseconds. */
-    timeoutMs: number);
+    /** The limit it exceeded, in milliseconds. */
+    timeoutMs: number, 
+    /** `run` for the whole-attempt limit, `idle` for the limit between signs of progress. */
+    kind?: 'run' | 'idle');
+}
+/**
+ * Raised when a run stops because its `RunControl` was drained.
+ *
+ * Not a failure: the superstep in flight finished and its checkpoint was written, so the thread is
+ * intact. Call `continue(threadId)` to resume it, on this worker or another.
+ */
+export declare class GraphDrainedError extends GraphError {
+    /** The thread to continue. */
+    readonly threadId: string;
+    /** Supersteps completed before the drain. */
+    readonly step: number;
+    /** Why the drain was requested. */
+    readonly reason?: string | undefined;
+    constructor(
+    /** The thread to continue. */
+    threadId: string, 
+    /** Supersteps completed before the drain. */
+    step: number, 
+    /** Why the drain was requested. */
+    reason?: string | undefined);
 }
 /**
  * Stable key for one interrupt, so a replayed node picks up the answer it was given.

package/dist/graph/errors.js

@@ -71,13 +71,29 @@ export class GraphNotInterruptedError extends GraphError {
 export class GraphNodeTimeoutError extends GraphError {
     node;
     timeoutMs;
-    constructor(node, timeoutMs) {
-        super(`Graph node "${node}" exceeded its ${timeoutMs}ms timeout`, 'GRAPH_NODE_TIMEOUT');
+    kind;
+    constructor(node, timeoutMs, kind = 'run') {
+        super(kind === 'idle'
+            ? `Graph node "${node}" showed no progress for ${timeoutMs}ms, its idle timeout`
+            : `Graph node "${node}" exceeded its ${timeoutMs}ms timeout`, 'GRAPH_NODE_TIMEOUT');
         this.node = node;
         this.timeoutMs = timeoutMs;
+        this.kind = kind;
         this.name = 'GraphNodeTimeoutError';
     }
 }
+export class GraphDrainedError extends GraphError {
+    threadId;
+    step;
+    reason;
+    constructor(threadId, step, reason) {
+        super(`Graph run on thread "${threadId}" drained after step ${step}${reason ? ` (${reason})` : ''}; continue() resumes it`, 'GRAPH_DRAINED');
+        this.threadId = threadId;
+        this.step = step;
+        this.reason = reason;
+        this.name = 'GraphDrainedError';
+    }
+}
 export function interruptKey(pending) {
     return `${pending.taskId ?? pending.node}:${pending.step}:${pending.index}`;
 }

package/dist/graph/event-stream.d.ts

@@ -0,0 +1,75 @@
+import type { ChannelSchema, GraphEvent, GraphResult, GraphStepEvent, GraphStreamEvent, GraphStreamOverflow, GraphStreamProjection, GraphStreamStats } from '../types/graph.js';
+import { GraphError } from './errors.js';
+/** Raised by an event stream whose reader fell behind with `overflow: 'error'`. The run is stopped. */
+export declare class GraphStreamOverflowError extends GraphError {
+    /** The buffer that filled. */
+    readonly maxBuffered: number;
+    constructor(
+    /** The buffer that filled. */
+    maxBuffered: number);
+}
+/** What starts the run behind a stream: given the event listener and a signal, the step events. */
+export type GraphRunStarter<S extends ChannelSchema> = (onEvent: (event: GraphEvent) => void, signal: AbortSignal) => AsyncIterable<GraphStepEvent<S>>;
+interface StreamSettings {
+    include: GraphStreamProjection[];
+    subgraphs: boolean;
+    maxBuffered: number;
+    overflow: GraphStreamOverflow;
+}
+/**
+ * A graph run as one stream of typed events, with each projection readable on its own.
+ *
+ * Iterate the stream for every included event, or take `messages()`, `tools()`, `values()`, or
+ * `updates()` for one kind. Every reader gets its own bounded buffer. The run starts when the first
+ * reader asks for an event, so create every reader before reading any. It stops when every reader
+ * has stopped.
+ */
+export declare class GraphEventStream<S extends ChannelSchema, O extends keyof S = keyof S> implements AsyncIterable<GraphStreamEvent<S>> {
+    private readonly start;
+    private readonly toResult;
+    private readonly settings;
+    private readonly readers;
+    private readonly controller;
+    private started;
+    private readonly stats;
+    private resolveResult;
+    private rejectResult;
+    /** Resolves with the run's result once it ends, or rejects with its error. */
+    readonly result: Promise<GraphResult<S, O>>;
+    constructor(start: GraphRunStarter<S>, toResult: (last: GraphStepEvent<S> | undefined) => GraphResult<S, O>, settings: StreamSettings);
+    /** Every included event. */
+    [Symbol.asyncIterator](): AsyncIterator<GraphStreamEvent<S>>;
+    /** Model output, as nodes stream it. */
+    messages(): AsyncIterable<Extract<GraphStreamEvent<S>, {
+        type: 'messages';
+    }>>;
+    /** Tool calls: each starting, progressing, and returning or failing. */
+    tools(): AsyncIterable<Extract<GraphStreamEvent<S>, {
+        type: 'tools';
+    }>>;
+    /** The state after each superstep. */
+    values(): AsyncIterable<Extract<GraphStreamEvent<S>, {
+        type: 'values';
+    }>>;
+    /** Each task's write. */
+    updates(): AsyncIterable<Extract<GraphStreamEvent<S>, {
+        type: 'updates';
+    }>>;
+    /** How the stream kept up with its slowest reader so far. */
+    streamStats(): GraphStreamStats;
+    /** A reader for one projection, registered now so it sees the run from its first event. */
+    private only;
+    private reader;
+    private begin;
+    private publish;
+    private fail;
+    private pump;
+}
+/** Builds a stream's settings from its options. */
+export declare function streamSettings(options: {
+    include?: GraphStreamProjection[];
+    subgraphs?: boolean;
+    maxBuffered?: number;
+    overflow?: GraphStreamOverflow;
+}): StreamSettings;
+export {};

package/dist/graph/event-stream.js

@@ -0,0 +1,289 @@
+import { GraphError } from './errors.js';
+const DEFAULT_INCLUDE = ['values', 'updates', 'messages', 'tools', 'custom'];
+const DEFAULT_MAX_BUFFERED = 1_000;
+export class GraphStreamOverflowError extends GraphError {
+    maxBuffered;
+    constructor(maxBuffered) {
+        super(`A graph event reader fell more than ${maxBuffered} events behind`, 'GRAPH_STREAM_OVERFLOW');
+        this.maxBuffered = maxBuffered;
+        this.name = 'GraphStreamOverflowError';
+    }
+}
+class Reader {
+    accepts;
+    settings;
+    stats;
+    onOverflow;
+    queue = [];
+    waiting;
+    failure;
+    drained = [];
+    ended = false;
+    error;
+    closed = false;
+    constructor(accepts, settings, stats, onOverflow) {
+        this.accepts = accepts;
+        this.settings = settings;
+        this.stats = stats;
+        this.onOverflow = onOverflow;
+    }
+    push(event) {
+        if (this.closed || !this.accepts(event))
+            return;
+        if (this.waiting) {
+            const resolve = this.waiting;
+            this.waiting = undefined;
+            this.failure = undefined;
+            resolve({ value: event, done: false });
+            return;
+        }
+        if (this.queue.length >= this.settings.maxBuffered && !this.makeRoom(event))
+            return;
+        this.queue.push(event);
+        this.stats.peak = Math.max(this.stats.peak, this.queue.length);
+    }
+    makeRoom(event) {
+        switch (this.settings.overflow) {
+            case 'drop-newest':
+                this.stats.dropped += 1;
+                return false;
+            case 'drop-oldest':
+                this.queue.shift();
+                this.stats.dropped += 1;
+                return true;
+            case 'error':
+                this.onOverflow(new GraphStreamOverflowError(this.settings.maxBuffered));
+                return false;
+            default: {
+                if (this.coalesce(event)) {
+                    this.stats.coalesced += 1;
+                    return false;
+                }
+                this.queue.shift();
+                this.stats.dropped += 1;
+                return true;
+            }
+        }
+    }
+    coalesce(event) {
+        for (let index = this.queue.length - 1; index >= 0; index -= 1) {
+            const queued = this.queue[index];
+            if (event.type === 'messages' && queued.type === 'messages') {
+                const sameMessage = queued.taskId === event.taskId &&
+                    (queued.chunk.kind ?? 'text') === (event.chunk.kind ?? 'text') &&
+                    queued.chunk.messageId === event.chunk.messageId &&
+                    queued.namespace.join('/') === event.namespace.join('/');
+                if (!sameMessage)
+                    continue;
+                this.queue[index] = {
+                    ...queued,
+                    chunk: { ...queued.chunk, content: queued.chunk.content + event.chunk.content },
+                };
+                return true;
+            }
+            if (event.type === 'values' &&
+                queued.type === 'values' &&
+                queued.namespace.join('/') === event.namespace.join('/')) {
+                this.queue[index] = event;
+                return true;
+            }
+        }
+        return false;
+    }
+    whenDrained() {
+        if (this.closed || this.queue.length <= this.settings.maxBuffered / 2)
+            return Promise.resolve();
+        return new Promise((resolve) => this.drained.push(resolve));
+    }
+    next() {
+        const event = this.queue.shift();
+        if (this.queue.length <= this.settings.maxBuffered / 2)
+            this.releaseWaiters();
+        if (event)
+            return Promise.resolve({ value: event, done: false });
+        if (this.error !== undefined)
+            return Promise.reject(this.error);
+        if (this.ended || this.closed)
+            return Promise.resolve({ value: undefined, done: true });
+        return new Promise((resolve, reject) => {
+            this.waiting = resolve;
+            this.failure = reject;
+        });
+    }
+    end(error) {
+        this.ended = true;
+        if (error !== undefined)
+            this.error = error;
+        this.releaseWaiters();
+        if (!this.waiting)
+            return;
+        const resolve = this.waiting;
+        const reject = this.failure;
+        this.waiting = undefined;
+        this.failure = undefined;
+        if (error !== undefined)
+            reject?.(error);
+        else
+            resolve({ value: undefined, done: true });
+    }
+    close() {
+        this.closed = true;
+        this.queue.length = 0;
+        this.end();
+    }
+    releaseWaiters() {
+        const waiters = this.drained;
+        this.drained = [];
+        for (const resolve of waiters)
+            resolve();
+    }
+}
+export class GraphEventStream {
+    start;
+    toResult;
+    settings;
+    readers = [];
+    controller = new AbortController();
+    started = false;
+    stats = { peak: 0, coalesced: 0, dropped: 0 };
+    resolveResult;
+    rejectResult;
+    result;
+    constructor(start, toResult, settings) {
+        this.start = start;
+        this.toResult = toResult;
+        this.settings = settings;
+        this.result = new Promise((resolve, reject) => {
+            this.resolveResult = resolve;
+            this.rejectResult = reject;
+        });
+        this.result.catch(() => undefined);
+    }
+    [Symbol.asyncIterator]() {
+        return this.reader(() => true);
+    }
+    messages() {
+        return this.only('messages');
+    }
+    tools() {
+        return this.only('tools');
+    }
+    values() {
+        return this.only('values');
+    }
+    updates() {
+        return this.only('updates');
+    }
+    streamStats() {
+        return { ...this.stats };
+    }
+    only(type) {
+        const iterator = this.reader((event) => event.type === type);
+        return { [Symbol.asyncIterator]: () => iterator };
+    }
+    reader(accepts) {
+        const reader = new Reader(accepts, this.settings, this.stats, (error) => this.fail(error));
+        if (this.started)
+            reader.end();
+        else
+            this.readers.push(reader);
+        return {
+            next: () => {
+                this.begin();
+                return reader.next();
+            },
+            return: async () => {
+                reader.close();
+                if (this.readers.every((item) => item.closed))
+                    this.controller.abort(new Error('Every reader stopped'));
+                return { value: undefined, done: true };
+            },
+        };
+    }
+    begin() {
+        if (this.started)
+            return;
+        this.started = true;
+        void this.pump();
+    }
+    publish(event) {
+        if (event.namespace.length > 0 && !this.settings.subgraphs)
+            return;
+        const projection = event.type;
+        if (!this.settings.include.includes(projection))
+            return;
+        for (const reader of this.readers)
+            reader.push(event);
+    }
+    fail(error) {
+        if (this.controller.signal.aborted)
+            return;
+        this.controller.abort(error);
+        for (const reader of this.readers)
+            reader.end(error);
+        this.rejectResult(error);
+    }
+    async pump() {
+        let last;
+        try {
+            for await (const step of this.start((event) => {
+                for (const item of fromGraphEvent(event))
+                    this.publish(item);
+            }, this.controller.signal)) {
+                last = step;
+                this.publish({ type: 'values', step: step.step, status: step.status, state: step.state, namespace: [] });
+                await Promise.all(this.readers.map((reader) => reader.whenDrained()));
+            }
+            for (const reader of this.readers)
+                reader.end();
+            this.resolveResult(this.toResult(last));
+        }
+        catch (error) {
+            const reason = this.controller.signal.reason;
+            this.fail(reason instanceof GraphStreamOverflowError ? reason : error);
+        }
+    }
+}
+export function streamSettings(options) {
+    return {
+        include: options.include ?? DEFAULT_INCLUDE,
+        subgraphs: options.subgraphs ?? false,
+        maxBuffered: Math.max(2, options.maxBuffered ?? DEFAULT_MAX_BUFFERED),
+        overflow: options.overflow ?? 'coalesce',
+    };
+}
+function fromGraphEvent(event) {
+    const namespace = event.namespace ?? [];
+    switch (event.type) {
+        case 'message':
+            return [
+                { type: 'messages', step: event.step, node: event.node, taskId: event.taskId, chunk: event.chunk, namespace },
+            ];
+        case 'tool':
+            return [{ type: 'tools', step: event.step, node: event.node, taskId: event.taskId, tool: event.tool, namespace }];
+        case 'custom':
+            return [
+                { type: 'custom', step: event.step, node: event.node, taskId: event.taskId, data: event.data, namespace },
+            ];
+        case 'checkpoint':
+            return [{ type: 'checkpoints', step: event.step, status: event.status, next: event.next, namespace }];
+        case 'task_end':
+            return [
+                { type: 'tasks', event, namespace },
+                ...(event.update === undefined
+                    ? []
+                    : [
+                        {
+                            type: 'updates',
+                            step: event.step,
+                            node: event.node,
+                            taskId: event.taskId,
+                            update: event.update,
+                            namespace,
+                        },
+                    ]),
+            ];
+        default:
+            return [{ type: 'tasks', event, namespace }];
+    }
+}