nestjs-langchain 2.0.0 → 2.1.0

package/dist/tools/tool-name.util.js

@@ -1,10 +1,8 @@
+import { STRUCTURED_OUTPUT_PREFIX } from '../constants.js';
 import { duplicateToolName, invalidToolName, reservedToolName, } from '../errors/messages.js';
 // OpenAI's limit, the strictest published, restated: no provider SDK exports
 // it. See #90.
 const TOOL_NAME = /^[a-zA-Z0-9_-]{1,64}$/;
-// The agent routes a call with this prefix to structured output, never to the
-// tool. Restated: LangChain does not export it. See #213.
-const STRUCTURED_OUTPUT_PREFIX = 'extract-';
 export const resolveToolName = (declared, method, where) => {
     const name = declared ?? method;
     if (!TOOL_NAME.test(name)) {

package/dist/tools/tool-name.util.js.map

@@ -1 +1 @@
-{"version":3,"file":"tool-name.util.js","sourceRoot":"","sources":["../../lib/tools/tool-name.util.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,eAAe,EACf,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAE/B,6EAA6E;AAC7E,eAAe;AACf,MAAM,SAAS,GAAG,uBAAuB,CAAC;AAE1C,8EAA8E;AAC9E,0DAA0D;AAC1D,MAAM,wBAAwB,GAAG,UAAU,CAAC;AAE5C,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,QAA4B,EAC5B,MAAc,EACd,KAAa,EACL,EAAE;IACV,MAAM,IAAI,GAAG,QAAQ,IAAI,MAAM,CAAC;IAEhC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,eAAe,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,QAAQ,KAAK,SAAS,CAAC,CACxE,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,UAAU,CAAC,wBAAwB,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,wBAAwB,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,KAAwC,EAC9B,EAAE;IACZ,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE7B,IAAI,KAAK,EAAE,CAAC;YACV,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}
+{"version":3,"file":"tool-name.util.js","sourceRoot":"","sources":["../../lib/tools/tool-name.util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EACL,iBAAiB,EACjB,eAAe,EACf,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAE/B,6EAA6E;AAC7E,eAAe;AACf,MAAM,SAAS,GAAG,uBAAuB,CAAC;AAE1C,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,QAA4B,EAC5B,MAAc,EACd,KAAa,EACL,EAAE;IACV,MAAM,IAAI,GAAG,QAAQ,IAAI,MAAM,CAAC;IAEhC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,eAAe,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,QAAQ,KAAK,SAAS,CAAC,CACxE,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,UAAU,CAAC,wBAAwB,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,wBAAwB,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,KAAwC,EAC9B,EAAE;IACZ,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE7B,IAAI,KAAK,EAAE,CAAC;YACV,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}

package/lib/agent.ts

@@ -1,6 +1,7 @@
 import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common';
 import { DiscoveryService } from '@nestjs/core';
 import { createAgent, initChatModel } from 'langchain';
+import type { z, ZodType } from 'zod';
 import {
   AGENT_NAME_TOKEN,
   MODULE_OPTIONS_TOKEN,
@@ -11,6 +12,7 @@ import {
   ModelOption,
 } from './interfaces/langchain-module-options.interface.js';
 import { CompletedRun } from './interfaces/completed-run.interface.js';
+import { RunOptions } from './interfaces/run-options.interface.js';
 import {
   ToolDiscoveryService,
   toolErrors,
@@ -18,8 +20,9 @@ import {
 } from './tools/index.js';
 import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
 import { LOG_CONTEXT, messageOf, RunReporter } from './logging/index.js';
-import { summariseRun } from './run/index.js';
-import { AgentRunError } from './errors/index.js';
+import { summariseRun, UsageAside } from './run/index.js';
+import { outputConfig, RunOutput, structuredOutput } from './output/index.js';
+import { AgentRunError, unwrapMiddlewareError } from './errors/index.js';
 import {
   agentNotBootstrapped,
   agentPrefix,
@@ -106,8 +109,16 @@ export class Agent implements OnModuleInit {
       model: await this.resolveModel(this.options.model),
       tools,
       systemPrompt: this.options.systemPrompt,
-      // The first wraps the next: `toolErrors` sits right around the tool.
-      middleware: [unknownTools(this.reporter), toolErrors(this.reporter)],
+      // The first wraps the next: `toolErrors` sits right around the tool,
+      // `structuredOutput` right around the model.
+      middleware: [
+        unknownTools(
+          this.reporter,
+          tools.map((tool) => tool.name),
+        ),
+        toolErrors(this.reporter),
+        structuredOutput(),
+      ],
     });
 
     this.logger.log(
@@ -179,21 +190,45 @@ export class Agent implements OnModuleInit {
    * as an `AgentRunError`, the original error in its `cause`.
    *
    * @param input The user's message.
-   * @returns The model's answer, the tools it called and what the run cost.
+   * @param options With `output`, the schema the answer must match.
+   * @returns The model's answer, the object `output` parses when given, the
+   * tools it called and what the run cost.
    * @example
    * ```ts
    * const { output, tools } = await agent.run('What is 6 times 7?');
+   *
+   * const { output: ticket } = await agent.run(email, {
+   *   output: z.object({ category: z.enum(['billing', 'bug', 'other']) }),
+   * });
    * ```
    */
-  async run(input: string): Promise<CompletedRun> {
+  run<S extends ZodType>(
+    input: string,
+    options: RunOptions & { output: S },
+  ): Promise<CompletedRun<z.output<S>>>;
+  run(
+    input: string,
+    options?: RunOptions & { output?: undefined },
+  ): Promise<CompletedRun>;
+  // Options typed `RunOptions`, whose schema is not known statically. See #280.
+  run(input: string, options?: RunOptions): Promise<CompletedRun<unknown>>;
+  async run(
+    input: string,
+    options?: RunOptions,
+  ): Promise<CompletedRun<unknown>> {
     const startedAt = Date.now();
     const durationMs = () => Date.now() - startedAt;
 
     try {
-      const { answer, messages } = await this.complete(input);
+      // Read in the `try`, and `null` from untyped JavaScript taken as no
+      // options, as LangChain takes a `null` config. See #291.
+      const schema = options?.output;
+      const output = schema && new RunOutput(schema, this.agentName);
+      const aside = new UsageAside();
+      const { answer, messages } = await this.complete(input, output, aside);
       const finished = {
         durationMs: durationMs(),
-        ...summariseRun(messages),
+        ...summariseRun(messages, output?.toolName, aside.usage),
       };
 
       this.reporter.notify((observer) =>
@@ -202,13 +237,14 @@ export class Agent implements OnModuleInit {
 
       return { status: 'completed', output: answer, ...finished };
     } catch (thrown) {
+      const cause = unwrapMiddlewareError(thrown);
       const error =
-        thrown instanceof AgentRunError
-          ? thrown
+        cause instanceof AgentRunError
+          ? cause
           : new AgentRunError(
-              agentRunFailed(this.prefix, thrown),
+              agentRunFailed(this.prefix, cause),
               this.agentName,
-              { cause: thrown },
+              { cause },
             );
 
       this.reporter.notify((observer) =>
@@ -225,7 +261,9 @@ export class Agent implements OnModuleInit {
 
   private async complete(
     input: string,
-  ): Promise<{ answer: string; messages: AgentState['messages'] }> {
+    output: RunOutput | undefined,
+    aside: UsageAside,
+  ): Promise<{ answer: unknown; messages: AgentState['messages'] }> {
     if (!this.#graph) {
       throw new AgentRunError(
         agentNotBootstrapped(this.prefix),
@@ -233,10 +271,14 @@ export class Agent implements OnModuleInit {
       );
     }
 
-    const { messages }: AgentState = await this.#graph.invoke(
+    const state: AgentState = await this.#graph.invoke(
       { messages: [{ role: 'user', content: input }] },
-      { callbacks: [this.modelErrors, ...(this.options.callbacks ?? [])] },
+      {
+        callbacks: [this.modelErrors, ...(this.options.callbacks ?? [])],
+        configurable: { ...outputConfig(output), ...aside.config },
+      },
     );
+    const { messages } = state;
 
     const last = messages[messages.length - 1];
 
@@ -247,6 +289,10 @@ export class Agent implements OnModuleInit {
       );
     }
 
+    if (output) {
+      return { answer: output.read(state.structuredResponse), messages };
+    }
+
     // `text`, not `content`: block answers are an array.
     if (!last.text) {
       throw new AgentRunError(modelReplyEmpty(this.prefix), this.agentName);

package/lib/constants.ts

@@ -5,6 +5,14 @@ export const TOOL_PARAMS_METADATA = 'nestjs-langchain:tool-params';
 // Emitted by `emitDecoratorMetadata` on every decorated method.
 export const PARAM_TYPES_METADATA = 'design:paramtypes';
 
+// The agent routes a call with this prefix to structured output, never to the
+// tool. Restated: LangChain does not export it. See #213.
+export const STRUCTURED_OUTPUT_PREFIX = 'extract-';
+
+// Ours rather than LangChain's counter or the schema's title: the same tool on
+// every run, under the prefix no `@Tool` can take. See #281.
+export const OUTPUT_TOOL = `${STRUCTURED_OUTPUT_PREFIX}output`;
+
 // The keys `@Module()` writes, restated: `@nestjs/common/constants` is
 // internal. See #97.
 export const MODULE_METADATA_KEYS = [

package/lib/errors/index.ts

@@ -1,2 +1,4 @@
 export { AgentRunError } from './agent-run.error.js';
+export { StructuredOutputError } from './structured-output.error.js';
 export { ToolConfigurationError } from './tool-configuration.error.js';
+export { unwrapMiddlewareError } from './unwrap-middleware-error.util.js';

package/lib/errors/messages.ts

@@ -96,14 +96,17 @@ export const schemaContradictsSignature = (
   `${where}: the schema for "${param}" describes ${declared.join(' | ')} ` +
   `but the signature declares ${expected}.`;
 
+// The advice of both messages on a schema with no JSON Schema form.
+const jsonSchemaAdvice =
+  'Declare a schema JSON Schema can express and convert it with .transform(), ' +
+  'such as z.iso.date() or z.iso.datetime() for a date.';
+
 export const noJsonSchemaForm = (
   where: string,
   param: string,
   reason: string,
 ): string =>
-  `${where}: "${param}" has no JSON Schema form (${reason}). ` +
-  'Declare a schema JSON Schema can express and convert it with .transform(), ' +
-  'such as z.iso.datetime() for a date.';
+  `${where}: "${param}" has no JSON Schema form (${reason}). ${jsonSchemaAdvice}`;
 
 export const duplicateParamName = (
   where: string,
@@ -155,4 +158,41 @@ export const modelNeverReplied = (
 export const modelReplyEmpty = (prefix: string): string =>
   `The ${prefix}agent's model replied with no text content.`;
 
+export const noOutputJsonSchemaForm = (
+  prefix: string,
+  reason: string,
+): string =>
+  `The ${prefix}agent's \`output\` schema has no JSON Schema form (${reason}). ` +
+  jsonSchemaAdvice;
+
+export const outputNotAnObject = (prefix: string): string =>
+  `The ${prefix}agent's \`output\` schema is not an object with named fields: ` +
+  "the model fills it in as a tool's parameters. Wrap it, such as " +
+  '`z.object({ items: z.array(...) })`.';
+
+export const structuredOutputMissing = (prefix: string): string =>
+  `The ${prefix}agent's model replied without the requested structured output.`;
+
+export const structuredOutputInvalid = (
+  prefix: string,
+  reason: string,
+): string =>
+  `The ${prefix}agent's model replied with an output that does not match ` +
+  `the schema: ${reason}`;
+
+export const structuredOutputMultiple = (
+  prefix: string,
+  count: number,
+): string =>
+  `The ${prefix}agent's model replied with ${count} outputs instead of one.`;
+
 export const unknownTool = (): string => 'not a tool of this agent';
+
+// LangChain's answer to a tool the agent does not have, restated:
+// `getInvalidToolError` is not exported. See #264.
+export const invalidTool = (tool: string, tools: string[]): string =>
+  `Error: ${tool} is not a valid tool, try one of [${tools.join(', ')}].`;
+
+export const toolNotRun = (tool: string): string =>
+  `Error: '${tool}' was not run: it came with the final answer. ` +
+  'Call it on its own first if it is needed.';

package/lib/errors/structured-output.error.ts

@@ -0,0 +1,23 @@
+import { AgentRunError } from './agent-run.error.js';
+
+/**
+ * The model's answer to a run with an `output` schema never came, or does not
+ * match the schema. LangChain's or Zod's error, if any, is in `cause`.
+ *
+ * @example
+ * ```ts
+ * try {
+ *   const { output } = await agent.run(ticket, { output: Classification });
+ * } catch (error) {
+ *   if (error instanceof StructuredOutputError) {
+ *     console.error(error.message, error.cause);
+ *   }
+ * }
+ * ```
+ */
+export class StructuredOutputError extends AgentRunError {
+  constructor(message: string, agent: string, options?: { cause?: unknown }) {
+    super(message, agent, options);
+    this.name = 'StructuredOutputError';
+  }
+}

package/lib/errors/unwrap-middleware-error.util.ts

@@ -0,0 +1,19 @@
+import { MiddlewareError } from 'langchain';
+
+/**
+ * The error as thrown, without the `MiddlewareError` LangChain wraps it in at
+ * each `wrapModelCall` middleware it crosses, a provider error included.
+ *
+ * @internal
+ * @param error What the agent's graph threw.
+ * @returns The innermost error.
+ */
+export const unwrapMiddlewareError = (error: unknown): unknown => {
+  let unwrapped = error;
+
+  while (MiddlewareError.isInstance(unwrapped) && unwrapped.cause) {
+    unwrapped = unwrapped.cause;
+  }
+
+  return unwrapped;
+};

package/lib/index.ts

@@ -3,7 +3,11 @@
 export { LangChainModule } from './langchain.module.js';
 export { getAgentToken } from './langchain.module-definition.js';
 export { Agent } from './agent.js';
-export { AgentRunError, ToolConfigurationError } from './errors/index.js';
+export {
+  AgentRunError,
+  StructuredOutputError,
+  ToolConfigurationError,
+} from './errors/index.js';
 export { InjectAgent, Tool, ToolParam } from './decorators/index.js';
 export type { ToolOptions, ToolParamOptions } from './decorators/index.js';
 export type {
@@ -16,6 +20,7 @@ export type {
   ToolStartEvent,
 } from './interfaces/agent-observer.interface.js';
 export type { CompletedRun } from './interfaces/completed-run.interface.js';
+export type { RunOptions } from './interfaces/run-options.interface.js';
 export type {
   AgentCallback,
   LangChainModuleOptions,

package/lib/interfaces/completed-run.interface.ts

@@ -7,7 +7,7 @@ import type { RunSummary } from '../run/index.js';
 export interface CompletedRun<TOutput = string> extends RunSummary {
   /** Always `'completed'`: a run that fails throws instead. */
   status: 'completed';
-  /** The model's text answer. */
+  /** The model's text answer, or the object the run's `output` schema parsed. */
   output: TOutput;
   /** How long the run took. */
   durationMs: number;

package/lib/interfaces/run-options.interface.ts

@@ -0,0 +1,21 @@
+import type { ZodType } from 'zod';
+
+/**
+ * The options of one run. Passed inline, they type `output` from the schema;
+ * typed `RunOptions`, `output` is `unknown`.
+ *
+ * @example
+ * ```ts
+ * const { output } = await agent.run(email, { output: Ticket }); // a Ticket
+ *
+ * const ask = (question: string, options?: RunOptions) =>
+ *   agent.run(question, options); // `output` is unknown
+ * ```
+ */
+export interface RunOptions {
+  /**
+   * The schema the answer must match. `output` is then the object it parses,
+   * typed from it, instead of the model's text.
+   */
+  output?: ZodType;
+}

package/lib/output/index.ts

@@ -0,0 +1,5 @@
+export { RunOutput } from './run-output.js';
+export {
+  outputConfig,
+  structuredOutput,
+} from './structured-output.middleware.js';

package/lib/output/run-output.ts

@@ -0,0 +1,154 @@
+import { toJsonSchema } from '@langchain/core/utils/json_schema';
+import {
+  MultipleStructuredOutputsError,
+  StructuredOutputParsingError,
+  toolStrategy,
+  TypedToolStrategy,
+} from 'langchain';
+import z, { ZodType } from 'zod';
+import { OUTPUT_TOOL } from '../constants.js';
+import { AgentRunError, StructuredOutputError } from '../errors/index.js';
+import {
+  agentPrefix,
+  noOutputJsonSchemaForm,
+  outputNotAnObject,
+  structuredOutputInvalid,
+  structuredOutputMissing,
+  structuredOutputMultiple,
+} from '../errors/messages.js';
+import { messageOf } from '../logging/index.js';
+
+// The default of LangChain's own retry middlewares. See #151.
+const MAX_RETRIES = 2;
+
+// LangChain's default for a Zod schema, which its JSON Schema path leaves
+// empty. See #282.
+const OUTPUT_DESCRIPTION =
+  "Tool for extracting structured output from the model's response.";
+
+/**
+ * The structured output one run asked for: the strategy its model calls use,
+ * and the answer parsed by the schema.
+ *
+ * @internal
+ */
+export class RunOutput {
+  /**
+   * Through a tool, whatever the model (see #274). One per run, which counts
+   * its own retries (see #151).
+   */
+  readonly strategy: TypedToolStrategy;
+  /** The tool the model answers through. */
+  readonly toolName = OUTPUT_TOOL;
+  #retries = 0;
+  private readonly prefix: string;
+
+  /**
+   * @param schema The run's `output` schema.
+   * @param agent The agent's name, for the errors.
+   * @throws AgentRunError when the schema has no JSON Schema form, or is not
+   * an object with named fields.
+   */
+  constructor(
+    private readonly schema: ZodType,
+    private readonly agent: string,
+  ) {
+    this.prefix = agentPrefix(agent);
+
+    // LangChain's own conversion, run before any model call.
+    let json: Record<string, unknown>;
+
+    try {
+      json = toJsonSchema(schema);
+    } catch (error) {
+      throw new AgentRunError(
+        noOutputJsonSchemaForm(this.prefix, messageOf(error)),
+        agent,
+        { cause: error },
+      );
+    }
+
+    // A tool's parameters. LangChain takes them as a JSON Schema with
+    // `properties` at every version, as a Zod schema other than an object at
+    // recent ones only. See #282.
+    if (json.type !== 'object' || !('properties' in json)) {
+      throw new AgentRunError(outputNotAnObject(this.prefix), agent);
+    }
+
+    // LangChain names the tool after the title.
+    this.strategy = toolStrategy(
+      {
+        ...json,
+        type: 'object',
+        title: OUTPUT_TOOL,
+        description: json.description ?? OUTPUT_DESCRIPTION,
+      },
+      { handleError: (error) => this.retry(error) },
+    );
+  }
+
+  // What LangChain sends back by default, until the retries run out.
+  private retry(error: Error): string {
+    this.#retries += 1;
+
+    if (this.#retries > MAX_RETRIES) {
+      throw error;
+    }
+
+    return error.message;
+  }
+
+  /**
+   * @param error What a model call threw.
+   * @returns A `StructuredOutputError` for LangChain's parsing failures, the
+   * error itself otherwise.
+   */
+  failure(error: unknown): unknown {
+    // Never LangChain's messages, which name the internal tool.
+    if (error instanceof StructuredOutputParsingError) {
+      return this.invalid(error.errors.join(' '), error);
+    }
+
+    if (error instanceof MultipleStructuredOutputsError) {
+      return new StructuredOutputError(
+        structuredOutputMultiple(this.prefix, error.toolNames.length),
+        this.agent,
+        { cause: error },
+      );
+    }
+
+    return error;
+  }
+
+  /**
+   * @param structuredResponse What the run left in `structuredResponse`.
+   * @returns The answer, as the schema parses it.
+   * @throws StructuredOutputError when the answer is missing or invalid.
+   */
+  read(structuredResponse: unknown): unknown {
+    if (structuredResponse === undefined) {
+      throw new StructuredOutputError(
+        structuredOutputMissing(this.prefix),
+        this.agent,
+      );
+    }
+
+    // LangChain checks the JSON Schema only: Zod also applies transforms and
+    // refinements. See #151.
+    const parsed = this.schema.safeParse(structuredResponse);
+
+    if (!parsed.success) {
+      throw this.invalid(z.prettifyError(parsed.error), parsed.error);
+    }
+
+    return parsed.data;
+  }
+
+  private invalid(reason: string, cause: unknown): StructuredOutputError {
+    return new StructuredOutputError(
+      structuredOutputInvalid(this.prefix, reason),
+      this.agent,
+      { cause },
+    );
+  }
+}

package/lib/output/structured-output.middleware.ts

@@ -0,0 +1,44 @@
+import { createMiddleware } from 'langchain';
+import { RunOutput } from './run-output.js';
+
+// The run's output travels under our own key in `configurable`, which
+// LangGraph keeps out of the trace metadata and checkpoints. See #151.
+const OUTPUT_KEY = 'nestjs-langchain:output';
+
+/**
+ * The `configurable` entry that hands a run's output to the middleware. A
+ * text run sets it to `null`, or it would inherit the output of the run it is
+ * nested in, which LangChain hands down. See #295.
+ *
+ * @internal
+ * @param output The run's output, if it asked for one.
+ * @returns The entry to put in `configurable`.
+ */
+export const outputConfig = (output: RunOutput | undefined) => ({
+  [OUTPUT_KEY]: output ?? null,
+});
+
+/**
+ * Sets the `responseFormat` of each model call of a run that asked for an
+ * `output`, so that one compiled agent serves text and structured runs.
+ * LangChain's parsing failures come out as a `StructuredOutputError`.
+ *
+ * @internal
+ */
+export const structuredOutput = () =>
+  createMiddleware({
+    name: 'structuredOutput',
+    wrapModelCall: async (request, handler) => {
+      const output = request.runtime.configurable?.[OUTPUT_KEY];
+
+      if (!(output instanceof RunOutput)) {
+        return handler(request);
+      }
+
+      try {
+        return await handler({ ...request, responseFormat: output.strategy });
+      } catch (error) {
+        throw output.failure(error);
+      }
+    },
+  });

package/lib/run/answered-calls.util.ts

@@ -0,0 +1,16 @@
+/**
+ * The ids of the tool calls the messages answer: the tool node answers every
+ * call it runs. Structural rather than `BaseMessage`. See #52.
+ *
+ * @internal
+ * @param messages A run's messages.
+ * @returns The ids their `tool_call_id`s answer.
+ */
+export const answeredCalls = (messages: readonly object[]): Set<string> =>
+  new Set(
+    messages.flatMap((message) =>
+      'tool_call_id' in message && typeof message.tool_call_id === 'string'
+        ? [message.tool_call_id]
+        : [],
+    ),
+  );

package/lib/run/index.ts

@@ -1,2 +1,4 @@
+export { answeredCalls } from './answered-calls.util.js';
 export { summariseRun } from './run-summary.util.js';
+export { UsageAside } from './usage-aside.js';
 export type { RunSummary } from './run-summary.util.js';

package/lib/run/run-summary.util.ts

@@ -1,14 +1,19 @@
 import { mergeUsageMetadata, UsageMetadata } from '@langchain/core/messages';
+import { answeredCalls } from './answered-calls.util.js';
 
 // Structural rather than `BaseMessage`. See #52.
 interface MessageLike {
   usage_metadata?: UsageMetadata;
-  tool_calls?: { name: string }[];
+  tool_calls?: { name: string; id?: string }[];
+  tool_call_id?: string;
 }
 
 /** What a run did and cost. */
 export interface RunSummary {
-  /** Every tool call the model made, failed and unknown ones included. */
+  /**
+   * Every tool call that reached the agent's tools, failed and unknown ones
+   * included, but not the one returning a structured output.
+   */
   tools: string[];
   /**
    * Summed over the run's model calls; absent when the provider reports no
@@ -22,16 +27,33 @@ export interface RunSummary {
  *
  * @internal
  * @param messages The messages of the finished run.
- * @returns The tool calls the model made, and the tokens if reported.
+ * @param outputTool The tool the model returned a structured output through.
+ * @param aside The usage of the replies the run left out of its messages.
+ * @returns The tool calls that reached the tools, and the tokens if reported.
  */
-export const summariseRun = (messages: readonly MessageLike[]): RunSummary => {
-  const tools = messages.flatMap((message) =>
-    (message.tool_calls ?? []).map((call) => call.name),
-  );
-
-  const reported = messages
-    .map((message) => message.usage_metadata)
-    .filter((usage) => usage !== undefined);
+export const summariseRun = (
+  messages: readonly MessageLike[],
+  outputTool?: string,
+  aside: readonly UsageMetadata[] = [],
+): RunSummary => {
+  // One that did not run has no answer (see #290). One with no id cannot be
+  // told, and counts (see #295).
+  const answered = answeredCalls(messages);
+
+  const tools = messages
+    .flatMap((message) => message.tool_calls ?? [])
+    .filter(
+      ({ name, id }) =>
+        name !== outputTool && (id === undefined || answered.has(id)),
+    )
+    .map((call) => call.name);
+
+  const reported = [
+    ...messages
+      .map((message) => message.usage_metadata)
+      .filter((usage) => usage !== undefined),
+    ...aside,
+  ];
 
   if (reported.length === 0) {
     return { tools };