nexus-ai-pro 2.0.0 → 2.2.0

package/dist-cjs/graph/functional.js

@@ -5,18 +5,28 @@ exports.workflow = workflow;
 const checkpointer_js_1 = require("./checkpointer.js");
 const run_js_1 = require("../lifecycle/run.js");
 const checkpoint_migration_js_1 = require("./checkpoint-migration.js");
+const tenant_js_1 = require("../store/tenant.js");
 const errors_js_1 = require("./errors.js");
 class WorkflowStepTimeoutError extends errors_js_1.GraphError {
     stepName;
     timeoutMs;
-    constructor(stepName, timeoutMs) {
-        super(`Step "${stepName}" exceeded its ${timeoutMs} ms timeout`, 'WORKFLOW_STEP_TIMEOUT');
+    kind;
+    constructor(stepName, timeoutMs, kind = 'run') {
+        super(kind === 'idle'
+            ? `Step "${stepName}" made no progress for ${timeoutMs} ms`
+            : `Step "${stepName}" exceeded its ${timeoutMs} ms timeout`, 'WORKFLOW_STEP_TIMEOUT');
         this.stepName = stepName;
         this.timeoutMs = timeoutMs;
+        this.kind = kind;
         this.name = 'WorkflowStepTimeoutError';
     }
 }
 exports.WorkflowStepTimeoutError = WorkflowStepTimeoutError;
+class WorkflowDraining extends Error {
+    constructor() {
+        super('The workflow is draining');
+    }
+}
 class WorkflowSuspended extends Error {
     pending;
     constructor(pending) {
@@ -45,10 +55,16 @@ class Workflow {
     }
     stream(input, runOptions = {}) {
         const threadId = runOptions.threadId ?? generatedId();
-        return this.execute(threadId, runOptions, async () => ({
-            ...this.fresh(threadId, input),
-            step: (await this.latest(threadId))?.step ?? 0,
-        }));
+        return this.execute(threadId, runOptions, async () => {
+            const latest = await this.latest(threadId);
+            if (latest && runOptions.tenantId !== undefined) {
+                const owner = latest.metadata?.tenantId;
+                if (typeof owner === 'string' && owner !== runOptions.tenantId) {
+                    throw new errors_js_1.GraphError(`Thread "${threadId}" cannot be used by this tenant; choose another id`, 'GRAPH_VALIDATION_ERROR');
+                }
+            }
+            return { ...this.fresh(threadId, input), step: latest?.step ?? 0 };
+        });
     }
     resume(threadId, value, runOptions = {}) {
         return this.execute(threadId, runOptions, async () => {
@@ -142,11 +158,21 @@ class Workflow {
     }
     async run(threadId, runOptions, start, queue) {
         const initial = await start();
+        const owner = typeof initial.metadata?.tenantId === 'string' ? initial.metadata.tenantId : undefined;
+        if (runOptions.tenantId !== undefined && owner !== undefined && owner !== runOptions.tenantId) {
+            throw new errors_js_1.GraphThreadNotFoundError(threadId);
+        }
+        const tenantId = runOptions.tenantId ?? owner;
         const checkpointer = this.checkpointer();
         const state = structuredClone(initial.state);
         state.steps ??= {};
         const resolved = { ...initial.resolved };
-        const metadata = { ...initial.metadata, graph: this.name, ...runOptions.metadata };
+        const metadata = {
+            ...initial.metadata,
+            graph: this.name,
+            ...runOptions.metadata,
+            ...(tenantId === undefined ? {} : { tenantId }),
+        };
         let stepNumber = initial.step;
         let writes = Promise.resolve();
         const persist = (checkpoint) => {
@@ -176,11 +202,16 @@ class Workflow {
         const inFlight = new Set();
         const suspended = [];
         let interruptIndex = 0;
+        let drained = false;
         const limiter = new Limiter(this.options.maxConcurrency ?? 16);
+        const defaults = this.options.stepDefaults ?? {};
         const context = {
             threadId,
+            ...(tenantId === undefined ? {} : { tenantId }),
             signal: controller.signal,
-            ...(this.options.store ? { store: this.options.store } : {}),
+            ...(this.options.store
+                ? { store: tenantId === undefined ? this.options.store : (0, tenant_js_1.tenantStore)(this.options.store, tenantId) }
+                : {}),
             step: (name, run, options = {}) => {
                 const count = (counts.get(name) ?? 0) + 1;
                 counts.set(name, count);
@@ -190,10 +221,22 @@ class Workflow {
                     queue.push({ type: 'step', step: stepNumber, name, key, cached: true, state: snapshot(), status: 'running' });
                     return Promise.resolve(recorded.value);
                 }
+                if (runOptions.control?.draining) {
+                    drained = true;
+                    return Promise.reject(new WorkflowDraining());
+                }
+                const merged = {
+                    ...defaults,
+                    ...options,
+                    timeout: { ...defaults.timeout, ...options.timeout },
+                };
                 const work = limiter
-                    .run(() => this.runStep(name, key, run, { ...this.options.retry, ...options.retry }, options, controller.signal, runOptions, stepNumber))
-                    .then(async (value) => {
-                    state.steps[key] = value === undefined ? {} : { value: structuredClone(value) };
+                    .run(() => this.runStep(name, key, run, { ...this.options.retry, ...defaults.retry, ...options.retry }, merged, controller.signal, runOptions, stepNumber))
+                    .then(async ({ value, recovered }) => {
+                    state.steps[key] = {
+                        ...(value === undefined ? {} : { value: structuredClone(value) }),
+                        ...(recovered ? { recovered } : {}),
+                    };
                     await persist({ state: state, next: [this.name], status: 'running' });
                     queue.push({
                         type: 'step',
@@ -267,6 +310,18 @@ class Workflow {
             queue.push({ type: 'interrupt', step: stepNumber, state: snapshot(), status: 'interrupted' });
             return;
         }
+        if (drained && outcome.kind === 'failed') {
+            const reason = runOptions.control?.reason;
+            await persist({
+                state: state,
+                next: [this.name],
+                status: 'interrupted',
+                resolved,
+                drained: { ...(reason ? { reason } : {}), at: this.now().toISOString() },
+            });
+            await writes;
+            throw new errors_js_1.GraphDrainedError(threadId, stepNumber, reason);
+        }
         if (outcome.kind === 'failed') {
             const error = outcome.error;
             await persist({
@@ -289,41 +344,87 @@ class Workflow {
     }
     async runStep(name, key, run, retry, options, signal, runOptions, step) {
         const policy = { ...DEFAULT_RETRY, ...retry };
+        const runMs = options.timeoutMs ?? options.timeout?.runMs;
+        const idleMs = options.timeout?.idleMs;
         for (let attempt = 1;; attempt += 1) {
             notify(runOptions, { type: 'task_start', step, taskId: key, node: name, attempt });
             const stepController = new AbortController();
             const forward = () => stepController.abort(signal.reason);
             signal.addEventListener('abort', forward, { once: true });
-            let timer;
-            let timedOut = false;
+            let runTimer;
+            let idleTimer;
+            let timedOut;
+            let expire = () => undefined;
+            const limits = runMs === undefined && idleMs === undefined
+                ? undefined
+                : new Promise((_, reject) => {
+                    expire = (error) => {
+                        if (timedOut)
+                            return;
+                        timedOut = error;
+                        reject(error);
+                        stepController.abort();
+                    };
+                });
+            limits?.catch(() => undefined);
+            if (runMs !== undefined)
+                runTimer = setTimeout(() => expire(new WorkflowStepTimeoutError(name, runMs)), runMs);
+            const armIdle = () => {
+                if (idleMs === undefined || timedOut)
+                    return;
+                if (idleTimer)
+                    clearTimeout(idleTimer);
+                idleTimer = setTimeout(() => expire(new WorkflowStepTimeoutError(name, idleMs, 'idle')), idleMs);
+            };
+            armIdle();
             try {
-                const work = Promise.resolve(run({ name, key, attempt, signal: stepController.signal }));
-                const value = options.timeoutMs === undefined
-                    ? await work
-                    : await Promise.race([
-                        work,
-                        new Promise((_, reject) => {
-                            timer = setTimeout(() => {
-                                timedOut = true;
-                                reject(new WorkflowStepTimeoutError(name, options.timeoutMs));
-                                stepController.abort();
-                            }, options.timeoutMs);
-                        }),
-                    ]);
+                const work = Promise.resolve(run({ name, key, attempt, signal: stepController.signal, heartbeat: armIdle }));
+                const value = limits ? await Promise.race([work, limits]) : await work;
                 if (timedOut)
-                    throw new WorkflowStepTimeoutError(name, options.timeoutMs);
+                    throw timedOut;
                 notify(runOptions, { type: 'task_end', step, taskId: key, node: name, update: value });
-                return value;
+                return { value };
             }
             catch (error) {
                 if (signal.aborted || error instanceof WorkflowSuspended)
                     throw error;
-                const failure = timedOut ? new WorkflowStepTimeoutError(name, options.timeoutMs) : error;
+                const failure = timedOut ?? error;
                 const retryable = policy.retryOn
                     ? policy.retryOn(failure, attempt)
-                    : !(failure instanceof errors_js_1.GraphError) || timedOut;
-                if (attempt >= Math.max(1, policy.maxAttempts) || !retryable)
-                    throw failure;
+                    : !(failure instanceof errors_js_1.GraphError) || timedOut !== undefined;
+                if (attempt >= Math.max(1, policy.maxAttempts) || !retryable) {
+                    if (!options.onError)
+                        throw failure;
+                    const failed = (recovered) => notify(runOptions, {
+                        type: 'task_failed',
+                        step,
+                        taskId: key,
+                        node: name,
+                        attempts: attempt,
+                        error: failure instanceof Error ? failure.message : String(failure),
+                        recovered,
+                    });
+                    let value;
+                    try {
+                        value = await options.onError({ name, key, attempts: attempt, error: failure });
+                    }
+                    catch (decision) {
+                        failed(false);
+                        throw decision;
+                    }
+                    failed(true);
+                    notify(runOptions, { type: 'task_end', step, taskId: key, node: name, update: value });
+                    return {
+                        value,
+                        recovered: {
+                            error: {
+                                name: failure instanceof Error ? failure.name : 'Error',
+                                message: failure instanceof Error ? failure.message : String(failure),
+                            },
+                            attempts: attempt,
+                        },
+                    };
+                }
                 notify(runOptions, {
                     type: 'task_retry',
                     step,
@@ -335,8 +436,10 @@ class Workflow {
                 await sleep(backoff(policy, attempt), signal);
             }
             finally {
-                if (timer)
-                    clearTimeout(timer);
+                if (runTimer)
+                    clearTimeout(runTimer);
+                if (idleTimer)
+                    clearTimeout(idleTimer);
                 signal.removeEventListener('abort', forward);
             }
         }

package/dist-cjs/graph/graph.d.ts

@@ -1,7 +1,8 @@
-import type { ChannelSchema, CompileOptions, EdgeRouter, GraphCheckpoint, GraphDescription, GraphInput, GraphProgress, GraphResult, GraphRunOptions, GraphStatus, GraphStepEvent, NodeFn, NodeOptions, StateUpdate } from '../types/graph.js';
+import type { ChannelSchema, CompileOptions, EdgeRouter, GraphCheckpoint, GraphDescription, GraphEventsOptions, GraphInput, GraphProgress, GraphResult, GraphRunOptions, GraphStatus, GraphStepEvent, NodeFn, NodeOptions, StateUpdate } from '../types/graph.js';
+import { GraphEventStream } from './event-stream.js';
 interface GraphNodeDefinition<S extends ChannelSchema> {
     fn: NodeFn<S>;
-    options: NodeOptions;
+    options: NodeOptions<S>;
 }
 /** Which channels a caller may write and read, as `createGraph()` declared them. */
 interface GraphScope {
@@ -29,7 +30,7 @@ export declare class StateGraph<S extends ChannelSchema, I extends keyof S = key
     private readonly edges;
     constructor(channels: S, scope?: GraphScope);
     /** Adds a node. Returns the graph, for chaining. */
-    addNode(name: string, fn: NodeFn<S>, options?: NodeOptions): this;
+    addNode(name: string, fn: NodeFn<S>, options?: NodeOptions<S>): this;
     /** Adds an edge that always runs `to` after `from`. Returns the graph, for chaining. */
     addEdge(from: string, to: string): this;
     /**
@@ -55,11 +56,33 @@ export declare class CompiledGraph<S extends ChannelSchema, I extends keyof S =
     private readonly store;
     /** Loaded the first time a node that declares `cache` runs, and not before. */
     private nodeCache;
+    /** The checkpoint writers of runs in progress, so `flush()` can wait for them. */
+    private readonly writers;
     constructor(channels: S, nodes: Map<string, GraphNodeDefinition<S>>, edges: Array<Edge<S>>, options: CompileOptions, scope?: GraphScope);
+    /**
+     * Waits until every checkpoint written by runs in progress is in the store. Only `async` and
+     * `exit` durability leave any outstanding; a run always waits for its own before it returns.
+     */
+    flush(): Promise<void>;
     /** Runs to completion, to an interrupt, or to the step limit. */
     invoke(input?: GraphInput<S, I>, runOptions?: GraphRunOptions): Promise<GraphResult<S, O>>;
     /** Yields one event per superstep, so a caller can render progress as it happens. */
     stream(input?: GraphInput<S, I>, runOptions?: GraphRunOptions): AsyncIterable<GraphStepEvent<S>>;
+    /**
+     * Runs the graph as one stream of typed events: the state after each superstep, each task's write,
+     * model output as nodes stream it, tool calls, task progress, checkpoints, and custom events.
+     *
+     * Iterate it for everything included, or read one projection with `messages()`, `tools()`,
+     * `values()`, or `updates()`. Each reader has its own bounded buffer, and the run waits for the
+     * slowest at the end of every superstep, so a slow reader slows the graph rather than filling
+     * memory. `result` resolves with the run's result.
+     */
+    events(input?: GraphInput<S, I>, options?: GraphEventsOptions): GraphEventStream<S, O>;
+    /** Like `events()`, answering the question a paused thread is waiting on. */
+    resumeEvents(threadId: string, value: unknown, options?: GraphEventsOptions): GraphEventStream<S, O>;
+    /** Like `events()`, carrying a thread on from a breakpoint, a drain, or a failure. */
+    continueEvents(threadId: string, options?: GraphEventsOptions): GraphEventStream<S, O>;
+    private eventStream;
     /**
      * Supplies the value a node asked for and continues.
      *
@@ -88,10 +111,17 @@ export declare class CompiledGraph<S extends ChannelSchema, I extends keyof S =
      * once the rewound checkpoint is written.
      */
     resumeFrom(threadId: string, step: number, runOptions?: GraphRunOptions): AsyncIterable<GraphStepEvent<S>>;
-    /** Latest checkpoint, or the one at `step`. */
-    state(threadId: string, step?: number): Promise<GraphCheckpoint<S> | undefined>;
-    /** Checkpoints for a thread, newest first. */
-    history(threadId: string, limit?: number): Promise<Array<GraphCheckpoint<S>>>;
+    /**
+     * Latest checkpoint, or the one at `step`. With a `tenantId`, a thread another tenant owns reads
+     * as absent.
+     */
+    state(threadId: string, step?: number, options?: {
+        tenantId?: string;
+    }): Promise<GraphCheckpoint<S> | undefined>;
+    /** Checkpoints for a thread, newest first. With a `tenantId`, another tenant's thread has none. */
+    history(threadId: string, limit?: number, options?: {
+        tenantId?: string;
+    }): Promise<Array<GraphCheckpoint<S>>>;
     /**
      * Wraps this graph as a node in another graph.
      *
@@ -120,26 +150,44 @@ export declare class CompiledGraph<S extends ChannelSchema, I extends keyof S =
      */
     updateState(threadId: string, update: StateUpdate<S>, options?: {
         asNode?: string;
+        tenantId?: string;
     }): Promise<GraphCheckpoint<S>>;
     /**
      * Copies a thread's history up to `step` into a new thread, and returns the new thread's id.
      *
      * Rewinding with `resumeFrom()` replaces the original timeline. A fork keeps it: both threads stay
      * readable and runnable, so two answers to the same question can be compared side by side. Every
-     * copied checkpoint records `metadata.forkedFrom`.
+     * copied checkpoint records `metadata.forkedFrom`, and keeps the tenant of the thread it came from.
+     * With a `tenantId`, another tenant's thread is not found.
      */
     fork(threadId: string, options?: {
         step?: number;
         threadId?: string;
+        tenantId?: string;
     }): Promise<string>;
     /**
      * Runs the supersteps, as one operation of the compiled lifecycle when there is one. The run is
      * admitted before its first superstep and finishes with its last, however it ends.
      */
     private run;
+    /**
+     * Runs the step loop with a checkpoint writer for the run's durability mode. However the run ends,
+     * every checkpoint it handed over is in the store before this returns or throws.
+     */
     private runSteps;
+    private steps;
     /** Runs one task to a verdict, applying its retry policy and timeout. */
     private runTask;
+    /**
+     * Hands a failure to the node's `onError`, and turns what it decides into the task's result. With
+     * no handler, or for a mistake in the graph itself, the task fails as it always did.
+     */
+    private recover;
+    /** The node's retry policy over the graph's defaults. */
+    private retryPolicy;
+    /** The node's run and idle limits over the graph's defaults. `timeout.runMs` wins over `timeoutMs`. */
+    private timeoutOf;
+    private errorHandler;
     private runNode;
     private loadNodeCache;
     /** Rejects writes to channels the graph does not accept from a caller. */