langchain 1.0.6 → 1.1.0

package/dist/agents/middleware/provider/openai/moderation.cjs

@@ -0,0 +1,299 @@
+const require_rolldown_runtime = require('../../../../_virtual/rolldown_runtime.cjs');
+const require_chat_models_universal = require('../../../../chat_models/universal.cjs');
+const require_middleware = require('../../../middleware.cjs');
+const __langchain_core_messages = require_rolldown_runtime.__toESM(require("@langchain/core/messages"));
+
+//#region src/agents/middleware/provider/openai/moderation.ts
+/**
+* Check if the model is an OpenAI model that supports moderation.
+* @param model - The model to check.
+* @returns Whether the model is an OpenAI model that supports moderation.
+*/
+function isOpenAIModel(model) {
+	if (!model || typeof model !== "object" || model === null || !("client" in model) || !("_getClientOptions" in model) || typeof model._getClientOptions !== "function") return false;
+	/**
+	* client may not yet be initialized, so we need to check if the model has a _getClientOptions method.
+	*/
+	model._getClientOptions();
+	return typeof model.client === "object" && model.client !== null && "moderations" in model.client && typeof model.client.moderations === "object" && model.client.moderations !== null && "create" in model.client.moderations && typeof model.client.moderations.create === "function";
+}
+/**
+* Default template for violation messages.
+*/
+const DEFAULT_VIOLATION_TEMPLATE = "I'm sorry, but I can't comply with that request. It was flagged for {categories}.";
+/**
+* Error raised when OpenAI flags content and `exitBehavior` is set to `"error"`.
+*/
+var OpenAIModerationError = class extends Error {
+	content;
+	stage;
+	result;
+	originalMessage;
+	constructor({ content, stage, result, message }) {
+		super(message);
+		this.name = "OpenAIModerationError";
+		this.content = content;
+		this.stage = stage;
+		this.result = result;
+		this.originalMessage = message;
+	}
+};
+/**
+* Middleware that moderates agent traffic using OpenAI's moderation endpoint.
+*
+* This middleware checks messages for content policy violations at different stages:
+* - Input: User messages before they reach the model
+* - Output: AI model responses
+* - Tool results: Results returned from tool executions
+*
+* @param options - Configuration options for the middleware
+* @param options.model - OpenAI model to use for moderation. Can be either a model name or a BaseChatModel instance.
+* @param options.moderationModel - Moderation model to use.
+* @param options.checkInput - Whether to check user input messages.
+* @param options.checkOutput - Whether to check model output messages.
+* @param options.checkToolResults - Whether to check tool result messages.
+* @param options.exitBehavior - How to handle violations.
+* @param options.violationMessage - Custom template for violation messages.
+* @returns Middleware function that can be used to moderate agent traffic.
+*
+* @example  Using model instance
+* ```ts
+* import { createAgent, openAIModerationMiddleware } from "langchain";
+*
+* const middleware = openAIModerationMiddleware({
+*   checkInput: true,
+*   checkOutput: true,
+*   exitBehavior: "end"
+* });
+*
+* const agent = createAgent({
+*   model: "openai:gpt-4o",
+*   tools: [...],
+*   middleware: [middleware],
+* });
+* ```
+*
+* @example Using model name
+* ```ts
+* import { createAgent, openAIModerationMiddleware } from "langchain";
+*
+* const middleware = openAIModerationMiddleware({
+*   model: "gpt-4o-mini",
+*   checkInput: true,
+*   checkOutput: true,
+*   exitBehavior: "end"
+* });
+*
+* const agent = createAgent({
+*   model: "openai:gpt-4o",
+*   tools: [...],
+*   middleware: [middleware],
+* });
+* ```
+*
+* @example Custom violation message
+* ```ts
+* const middleware = openAIModerationMiddleware({
+*   violationMessage: "Content flagged: {categories}. Scores: {category_scores}"
+* });
+* ```
+*/
+function openAIModerationMiddleware(options) {
+	const { model, moderationModel = "omni-moderation-latest", checkInput = true, checkOutput = true, checkToolResults = false, exitBehavior = "end", violationMessage } = options;
+	let openaiModel;
+	const initModerationModel = async () => {
+		if (openaiModel) return openaiModel;
+		const resolvedModel = typeof model === "string" ? await require_chat_models_universal.initChatModel(model) : model;
+		/**
+		* Check if the model is an OpenAI model.
+		*/
+		if (!resolvedModel.getName().includes("ChatOpenAI")) throw new Error(`Model must be an OpenAI model to use moderation middleware. Got: ${resolvedModel.getName()}`);
+		/**
+		* check if OpenAI model package supports moderation.
+		*/
+		if (!isOpenAIModel(resolvedModel)) throw new Error("Model must support moderation to use moderation middleware.");
+		openaiModel = resolvedModel;
+		return openaiModel;
+	};
+	/**
+	* Extract text content from a message.
+	*/
+	const extractText = (message) => {
+		if (message.content == null) return null;
+		const text = message.text;
+		return text || null;
+	};
+	/**
+	* Find the last index of a message type in the messages array.
+	*/
+	const findLastIndex = (messages, messageType) => {
+		for (let idx = messages.length - 1; idx >= 0; idx--) if (messageType.isInstance(messages[idx])) return idx;
+		return null;
+	};
+	/**
+	* Format violation message from moderation result.
+	*/
+	const formatViolationMessage = (content, result) => {
+		const categories = [];
+		const categoriesObj = result.categories;
+		for (const [name, flagged] of Object.entries(categoriesObj)) if (flagged) categories.push(name.replace(/_/g, " "));
+		const categoryLabel = categories.length > 0 ? categories.join(", ") : "OpenAI's safety policies";
+		const template = violationMessage || DEFAULT_VIOLATION_TEMPLATE;
+		const scoresJson = JSON.stringify(result.category_scores, null, 2);
+		try {
+			return template.replace("{categories}", categoryLabel).replace("{category_scores}", scoresJson).replace("{original_content}", content);
+		} catch {
+			return template;
+		}
+	};
+	function moderateContent(input, params) {
+		const clientOptions = openaiModel?._getClientOptions?.();
+		const moderationModel$1 = params?.model ?? "omni-moderation-latest";
+		const moderationRequest = {
+			input,
+			model: moderationModel$1
+		};
+		return openaiModel.client.moderations.create(moderationRequest, clientOptions);
+	}
+	/**
+	* Apply violation handling based on exit behavior.
+	*/
+	const applyViolation = (messages, index, stage, content, result) => {
+		const violationText = formatViolationMessage(content, result);
+		if (exitBehavior === "error") throw new OpenAIModerationError({
+			content,
+			stage,
+			result,
+			message: violationText
+		});
+		if (exitBehavior === "end") return {
+			jumpTo: "end",
+			messages: [new __langchain_core_messages.AIMessage({ content: violationText })]
+		};
+		if (index == null) return void 0;
+		/**
+		* Replace the original message with a new message that contains the violation text.
+		*/
+		const newMessages = [...messages];
+		const original = newMessages[index];
+		const MessageConstructor = Object.getPrototypeOf(original).constructor;
+		newMessages[index] = new MessageConstructor({
+			...original,
+			content: violationText
+		});
+		return { messages: newMessages };
+	};
+	/**
+	* Moderate user input messages.
+	*/
+	const moderateUserMessage = async (messages) => {
+		const idx = findLastIndex(messages, __langchain_core_messages.HumanMessage);
+		if (idx == null) return null;
+		const message = messages[idx];
+		const text = extractText(message);
+		if (!text) return null;
+		await initModerationModel();
+		const response = await moderateContent(text, { model: moderationModel });
+		const flaggedResult = response.results.find((result) => result.flagged);
+		if (!flaggedResult) return null;
+		return applyViolation(messages, idx, "input", text, flaggedResult);
+	};
+	/**
+	* Moderate tool result messages.
+	*/
+	const moderateToolMessages = async (messages) => {
+		const lastAiIdx = findLastIndex(messages, __langchain_core_messages.AIMessage);
+		if (lastAiIdx == null) return null;
+		const working = [...messages];
+		let modified = false;
+		for (let idx = lastAiIdx + 1; idx < working.length; idx++) {
+			const msg = working[idx];
+			if (!__langchain_core_messages.ToolMessage.isInstance(msg)) continue;
+			const text = extractText(msg);
+			if (!text) continue;
+			await initModerationModel();
+			const response = await moderateContent(text, { model: moderationModel });
+			const flaggedResult = response.results.find((result) => result.flagged);
+			if (!flaggedResult) continue;
+			const action = applyViolation(working, idx, "tool", text, flaggedResult);
+			if (action) {
+				if ("jumpTo" in action) return action;
+				if ("messages" in action) {
+					working.splice(0, working.length, ...action.messages);
+					modified = true;
+				}
+			}
+		}
+		if (modified) return { messages: working };
+		return null;
+	};
+	/**
+	* Moderate model output messages.
+	*/
+	const moderateOutput = async (messages) => {
+		const lastAiIdx = findLastIndex(messages, __langchain_core_messages.AIMessage);
+		if (lastAiIdx == null) return null;
+		const aiMessage = messages[lastAiIdx];
+		const text = extractText(aiMessage);
+		if (!text) return null;
+		await initModerationModel();
+		const response = await moderateContent(text, { model: moderationModel });
+		const flaggedResult = response.results.find((result) => result.flagged);
+		if (!flaggedResult) return null;
+		return applyViolation(messages, lastAiIdx, "output", text, flaggedResult);
+	};
+	/**
+	* Moderate inputs (user messages and tool results) before model call.
+	*/
+	const moderateInputs = async (messages) => {
+		const working = [...messages];
+		let modified = false;
+		if (checkToolResults) {
+			const action = await moderateToolMessages(working);
+			if (action) {
+				if ("jumpTo" in action) return action;
+				if ("messages" in action) {
+					working.splice(0, working.length, ...action.messages);
+					modified = true;
+				}
+			}
+		}
+		if (checkInput) {
+			const action = await moderateUserMessage(working);
+			if (action) {
+				if ("jumpTo" in action) return action;
+				if ("messages" in action) {
+					working.splice(0, working.length, ...action.messages);
+					modified = true;
+				}
+			}
+		}
+		if (modified) return { messages: working };
+		return null;
+	};
+	return require_middleware.createMiddleware({
+		name: "OpenAIModerationMiddleware",
+		beforeModel: {
+			hook: async (state) => {
+				if (!checkInput && !checkToolResults) return void 0;
+				const messages = state.messages || [];
+				if (messages.length === 0) return void 0;
+				return await moderateInputs(messages) ?? void 0;
+			},
+			canJumpTo: ["end"]
+		},
+		afterModel: {
+			hook: async (state) => {
+				if (!checkOutput) return void 0;
+				const messages = state.messages || [];
+				if (messages.length === 0) return void 0;
+				return await moderateOutput(messages) ?? void 0;
+			},
+			canJumpTo: ["end"]
+		}
+	});
+}
+
+//#endregion
+exports.openAIModerationMiddleware = openAIModerationMiddleware;
+//# sourceMappingURL=moderation.cjs.map

package/dist/agents/middleware/provider/openai/moderation.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"moderation.cjs","names":["model: unknown","options: OpenAIModerationMiddlewareOptions","openaiModel: OpenAIModel | undefined","initChatModel","message: BaseMessage","messages: BaseMessage[]","messageType: typeof AIMessage | typeof HumanMessage | typeof ToolMessage","content: string","result: ModerationResult","categories: string[]","input: string | string[]","params?: { model?: ModerationModel; options?: unknown }","moderationModel","index: number | null","stage: ViolationStage","AIMessage","HumanMessage","ToolMessage","createMiddleware"],"sources":["../../../../../src/agents/middleware/provider/openai/moderation.ts"],"sourcesContent":["import type { BaseMessage } from \"@langchain/core/messages\";\nimport { AIMessage, HumanMessage, ToolMessage } from \"@langchain/core/messages\";\nimport type { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport { BaseLanguageModel } from \"@langchain/core/language_models/base\";\n\nimport { initChatModel } from \"../../../../chat_models/universal.js\";\nimport { createMiddleware } from \"../../../middleware.js\";\nimport type { MiddlewareResult, AgentMiddleware } from \"../../types.js\";\nimport type { AgentBuiltInState } from \"../../../runtime.js\";\n\n/**\n * OpenAI model interface.\n */\ninterface OpenAIModel extends BaseLanguageModel {\n  getName: () => string;\n  _getClientOptions: () => unknown;\n  client: {\n    moderations: {\n      create: (\n        input: {\n          input: string | string[];\n          model: string;\n        },\n        options?: unknown\n      ) => Promise<ModerationResponse>;\n    };\n  };\n}\n\n/**\n * Check if the model is an OpenAI model that supports moderation.\n * @param model - The model to check.\n * @returns Whether the model is an OpenAI model that supports moderation.\n */\nfunction isOpenAIModel(model: unknown): model is OpenAIModel {\n  if (\n    !model ||\n    typeof model !== \"object\" ||\n    model === null ||\n    !(\"client\" in model) ||\n    !(\"_getClientOptions\" in model) ||\n    typeof model._getClientOptions !== \"function\"\n  ) {\n    return false;\n  }\n\n  /**\n   * client may not yet be initialized, so we need to check if the model has a _getClientOptions method.\n   */\n  model._getClientOptions();\n  return (\n    typeof model.client === \"object\" &&\n    model.client !== null &&\n    \"moderations\" in model.client &&\n    typeof model.client.moderations === \"object\" &&\n    model.client.moderations !== null &&\n    \"create\" in model.client.moderations &&\n    typeof model.client.moderations.create === \"function\"\n  );\n}\n\n/**\n * Stage where a violation occurred.\n */\nexport type ViolationStage = \"input\" | \"output\" | \"tool\";\n\n/**\n * Default template for violation messages.\n */\nconst DEFAULT_VIOLATION_TEMPLATE =\n  \"I'm sorry, but I can't comply with that request. It was flagged for {categories}.\";\n\n/**\n * Result of moderation.\n * @see https://platform.openai.com/docs/api-reference/moderations/object\n */\ninterface ModerationResult {\n  flagged: boolean;\n  categories: Record<string, boolean>;\n  category_scores: Record<string, number>;\n  category_applied_input_types: Record<string, string[]>;\n}\n\n/**\n * Moderation response.\n * @see https://platform.openai.com/docs/api-reference/moderations/create\n */\ninterface ModerationResponse {\n  id: string;\n  model: string;\n  results: ModerationResult[];\n}\n\ntype ModerationModel =\n  | \"omni-moderation-latest\"\n  | \"omni-moderation-2024-09-26\"\n  | \"text-moderation-latest\"\n  | \"text-moderation-stable\";\n\n/**\n * Error raised when OpenAI flags content and `exitBehavior` is set to `\"error\"`.\n */\nexport class OpenAIModerationError extends Error {\n  content: string;\n  stage: ViolationStage;\n  result: ModerationResult;\n  originalMessage: string;\n\n  constructor({\n    content,\n    stage,\n    result,\n    message,\n  }: {\n    content: string;\n    stage: ViolationStage;\n    result: ModerationResult;\n    message: string;\n  }) {\n    super(message);\n    this.name = \"OpenAIModerationError\";\n    this.content = content;\n    this.stage = stage;\n    this.result = result;\n    this.originalMessage = message;\n  }\n}\n\n/**\n * Options for configuring the OpenAI Moderation middleware.\n */\nexport interface OpenAIModerationMiddlewareOptions {\n  /**\n   * OpenAI model to use for moderation. Can be either a model name or a BaseChatModel instance.\n   * @example\n   * ```ts\n   * const model = new ChatOpenAI({ model: \"gpt-4o-mini\" });\n   * const middleware = openAIModerationMiddleware({ model });\n   * const agent = createAgent({\n   *   model,\n   *   middleware: [middleware],\n   * });\n   * ```\n   * @example\n   * ```ts\n   * const middleware = openAIModerationMiddleware({ model: \"gpt-4o-mini\" });\n   * const agent = createAgent({\n   *   model: \"gpt-5\",\n   *   middleware: [middleware],\n   * });\n   * ```\n   */\n  model: string | BaseChatModel;\n\n  /**\n   * Moderation model to use.\n   * @default \"omni-moderation-latest\"\n   */\n  moderationModel?: ModerationModel;\n\n  /**\n   * Whether to check user input messages.\n   * @default true\n   */\n  checkInput?: boolean;\n\n  /**\n   * Whether to check model output messages.\n   * @default true\n   */\n  checkOutput?: boolean;\n\n  /**\n   * Whether to check tool result messages.\n   * @default false\n   */\n  checkToolResults?: boolean;\n\n  /**\n   * How to handle violations.\n   * - `\"error\"`: Throw an error when content is flagged\n   * - `\"end\"`: End the agent execution and return a violation message\n   * - `\"replace\"`: Replace the flagged content with a violation message\n   * @default \"end\"\n   */\n  exitBehavior?: \"error\" | \"end\" | \"replace\";\n\n  /**\n   * Custom template for violation messages.\n   * Available placeholders: `{categories}`, `{category_scores}`, `{original_content}`\n   */\n  violationMessage?: string;\n}\n\n/**\n * Middleware that moderates agent traffic using OpenAI's moderation endpoint.\n *\n * This middleware checks messages for content policy violations at different stages:\n * - Input: User messages before they reach the model\n * - Output: AI model responses\n * - Tool results: Results returned from tool executions\n *\n * @param options - Configuration options for the middleware\n * @param options.model - OpenAI model to use for moderation. Can be either a model name or a BaseChatModel instance.\n * @param options.moderationModel - Moderation model to use.\n * @param options.checkInput - Whether to check user input messages.\n * @param options.checkOutput - Whether to check model output messages.\n * @param options.checkToolResults - Whether to check tool result messages.\n * @param options.exitBehavior - How to handle violations.\n * @param options.violationMessage - Custom template for violation messages.\n * @returns Middleware function that can be used to moderate agent traffic.\n *\n * @example  Using model instance\n * ```ts\n * import { createAgent, openAIModerationMiddleware } from \"langchain\";\n *\n * const middleware = openAIModerationMiddleware({\n *   checkInput: true,\n *   checkOutput: true,\n *   exitBehavior: \"end\"\n * });\n *\n * const agent = createAgent({\n *   model: \"openai:gpt-4o\",\n *   tools: [...],\n *   middleware: [middleware],\n * });\n * ```\n *\n * @example Using model name\n * ```ts\n * import { createAgent, openAIModerationMiddleware } from \"langchain\";\n *\n * const middleware = openAIModerationMiddleware({\n *   model: \"gpt-4o-mini\",\n *   checkInput: true,\n *   checkOutput: true,\n *   exitBehavior: \"end\"\n * });\n *\n * const agent = createAgent({\n *   model: \"openai:gpt-4o\",\n *   tools: [...],\n *   middleware: [middleware],\n * });\n * ```\n *\n * @example Custom violation message\n * ```ts\n * const middleware = openAIModerationMiddleware({\n *   violationMessage: \"Content flagged: {categories}. Scores: {category_scores}\"\n * });\n * ```\n */\nexport function openAIModerationMiddleware(\n  options: OpenAIModerationMiddlewareOptions\n): AgentMiddleware {\n  const {\n    model,\n    moderationModel = \"omni-moderation-latest\",\n    checkInput = true,\n    checkOutput = true,\n    checkToolResults = false,\n    exitBehavior = \"end\",\n    violationMessage,\n  } = options;\n\n  let openaiModel: OpenAIModel | undefined;\n  const initModerationModel = async (): Promise<OpenAIModel> => {\n    if (openaiModel) {\n      return openaiModel;\n    }\n\n    const resolvedModel =\n      typeof model === \"string\" ? await initChatModel(model) : model;\n\n    /**\n     * Check if the model is an OpenAI model.\n     */\n    if (!resolvedModel.getName().includes(\"ChatOpenAI\")) {\n      throw new Error(\n        `Model must be an OpenAI model to use moderation middleware. Got: ${resolvedModel.getName()}`\n      );\n    }\n\n    /**\n     * check if OpenAI model package supports moderation.\n     */\n    if (!isOpenAIModel(resolvedModel)) {\n      throw new Error(\n        \"Model must support moderation to use moderation middleware.\"\n      );\n    }\n\n    openaiModel = resolvedModel as unknown as OpenAIModel;\n    return openaiModel;\n  };\n\n  /**\n   * Extract text content from a message.\n   */\n  const extractText = (message: BaseMessage): string | null => {\n    if (message.content == null) {\n      return null;\n    }\n    const text = message.text;\n    return text || null;\n  };\n\n  /**\n   * Find the last index of a message type in the messages array.\n   */\n  const findLastIndex = (\n    messages: BaseMessage[],\n    messageType: typeof AIMessage | typeof HumanMessage | typeof ToolMessage\n  ): number | null => {\n    for (let idx = messages.length - 1; idx >= 0; idx--) {\n      if (messageType.isInstance(messages[idx])) {\n        return idx;\n      }\n    }\n    return null;\n  };\n\n  /**\n   * Format violation message from moderation result.\n   */\n  const formatViolationMessage = (\n    content: string,\n    result: ModerationResult\n  ): string => {\n    // Convert categories to array of flagged category names\n    const categories: string[] = [];\n    const categoriesObj = result.categories as unknown as Record<\n      string,\n      boolean\n    >;\n    for (const [name, flagged] of Object.entries(categoriesObj)) {\n      if (flagged) {\n        categories.push(name.replace(/_/g, \" \"));\n      }\n    }\n\n    const categoryLabel =\n      categories.length > 0\n        ? categories.join(\", \")\n        : \"OpenAI's safety policies\";\n\n    const template = violationMessage || DEFAULT_VIOLATION_TEMPLATE;\n    const scoresJson = JSON.stringify(\n      result.category_scores as unknown as Record<string, number>,\n      null,\n      2\n    );\n\n    try {\n      return template\n        .replace(\"{categories}\", categoryLabel)\n        .replace(\"{category_scores}\", scoresJson)\n        .replace(\"{original_content}\", content);\n    } catch {\n      return template;\n    }\n  };\n\n  function moderateContent(\n    input: string | string[],\n    params?: { model?: ModerationModel; options?: unknown }\n  ): Promise<ModerationResponse> {\n    const clientOptions = openaiModel?._getClientOptions?.();\n    const moderationModel = params?.model ?? \"omni-moderation-latest\";\n    const moderationRequest = {\n      input,\n      model: moderationModel,\n    };\n    return openaiModel!.client.moderations.create(\n      moderationRequest,\n      clientOptions\n    );\n  }\n\n  /**\n   * Apply violation handling based on exit behavior.\n   */\n  const applyViolation = (\n    messages: BaseMessage[],\n    index: number | null,\n    stage: ViolationStage,\n    content: string,\n    result: ModerationResult\n  ): MiddlewareResult<Partial<AgentBuiltInState>> | undefined => {\n    const violationText = formatViolationMessage(content, result);\n\n    if (exitBehavior === \"error\") {\n      throw new OpenAIModerationError({\n        content,\n        stage,\n        result,\n        message: violationText,\n      });\n    }\n\n    if (exitBehavior === \"end\") {\n      return {\n        jumpTo: \"end\",\n        messages: [new AIMessage({ content: violationText })],\n      };\n    }\n\n    if (index == null) {\n      return undefined;\n    }\n\n    /**\n     * Replace the original message with a new message that contains the violation text.\n     */\n    const newMessages = [...messages];\n    const original = newMessages[index];\n    const MessageConstructor = Object.getPrototypeOf(original).constructor;\n    newMessages[index] = new MessageConstructor({\n      ...original,\n      content: violationText,\n    });\n\n    return { messages: newMessages };\n  };\n\n  /**\n   * Moderate user input messages.\n   */\n  const moderateUserMessage = async (\n    messages: BaseMessage[]\n  ): Promise<MiddlewareResult<Partial<AgentBuiltInState>> | null> => {\n    const idx = findLastIndex(messages, HumanMessage);\n    if (idx == null) {\n      return null;\n    }\n\n    const message = messages[idx];\n    const text = extractText(message);\n    if (!text) {\n      return null;\n    }\n\n    await initModerationModel();\n    const response = await moderateContent(text, {\n      model: moderationModel,\n    });\n\n    const flaggedResult = response.results.find((result) => result.flagged);\n    if (!flaggedResult) {\n      return null;\n    }\n\n    return applyViolation(messages, idx, \"input\", text, flaggedResult);\n  };\n\n  /**\n   * Moderate tool result messages.\n   */\n  const moderateToolMessages = async (\n    messages: BaseMessage[]\n  ): Promise<MiddlewareResult<Partial<AgentBuiltInState>> | null> => {\n    const lastAiIdx = findLastIndex(messages, AIMessage);\n    if (lastAiIdx == null) {\n      return null;\n    }\n\n    const working = [...messages];\n    let modified = false;\n\n    for (let idx = lastAiIdx + 1; idx < working.length; idx++) {\n      const msg = working[idx];\n      if (!ToolMessage.isInstance(msg)) {\n        continue;\n      }\n\n      const text = extractText(msg);\n      if (!text) {\n        continue;\n      }\n\n      await initModerationModel();\n      const response = await moderateContent(text, {\n        model: moderationModel,\n      });\n      const flaggedResult = response.results.find((result) => result.flagged);\n      if (!flaggedResult) {\n        continue;\n      }\n\n      const action = applyViolation(working, idx, \"tool\", text, flaggedResult);\n      if (action) {\n        if (\"jumpTo\" in action) {\n          return action;\n        }\n        if (\"messages\" in action) {\n          working.splice(\n            0,\n            working.length,\n            ...(action.messages as BaseMessage[])\n          );\n          modified = true;\n        }\n      }\n    }\n\n    if (modified) {\n      return { messages: working };\n    }\n\n    return null;\n  };\n\n  /**\n   * Moderate model output messages.\n   */\n  const moderateOutput = async (\n    messages: BaseMessage[]\n  ): Promise<MiddlewareResult<Partial<AgentBuiltInState>> | null> => {\n    const lastAiIdx = findLastIndex(messages, AIMessage);\n    if (lastAiIdx == null) {\n      return null;\n    }\n\n    const aiMessage = messages[lastAiIdx];\n    const text = extractText(aiMessage);\n    if (!text) {\n      return null;\n    }\n\n    await initModerationModel();\n    const response = await moderateContent(text, {\n      model: moderationModel,\n    });\n    const flaggedResult = response.results.find((result) => result.flagged);\n    if (!flaggedResult) {\n      return null;\n    }\n\n    return applyViolation(messages, lastAiIdx, \"output\", text, flaggedResult);\n  };\n\n  /**\n   * Moderate inputs (user messages and tool results) before model call.\n   */\n  const moderateInputs = async (\n    messages: BaseMessage[]\n  ): Promise<MiddlewareResult<Partial<AgentBuiltInState>> | null> => {\n    const working = [...messages];\n    let modified = false;\n\n    if (checkToolResults) {\n      const action = await moderateToolMessages(working);\n      if (action) {\n        if (\"jumpTo\" in action) {\n          return action;\n        }\n        if (\"messages\" in action) {\n          working.splice(\n            0,\n            working.length,\n            ...(action.messages as BaseMessage[])\n          );\n          modified = true;\n        }\n      }\n    }\n\n    if (checkInput) {\n      const action = await moderateUserMessage(working);\n      if (action) {\n        if (\"jumpTo\" in action) {\n          return action;\n        }\n        if (\"messages\" in action) {\n          working.splice(\n            0,\n            working.length,\n            ...(action.messages as BaseMessage[])\n          );\n          modified = true;\n        }\n      }\n    }\n\n    if (modified) {\n      return { messages: working };\n    }\n\n    return null;\n  };\n\n  return createMiddleware({\n    name: \"OpenAIModerationMiddleware\",\n    beforeModel: {\n      hook: async (\n        state\n      ): Promise<MiddlewareResult<Partial<AgentBuiltInState>> | undefined> => {\n        if (!checkInput && !checkToolResults) {\n          return undefined;\n        }\n\n        const messages = state.messages || [];\n        if (messages.length === 0) {\n          return undefined;\n        }\n\n        return (await moderateInputs(messages)) ?? undefined;\n      },\n      canJumpTo: [\"end\"],\n    },\n    afterModel: {\n      hook: async (\n        state\n      ): Promise<MiddlewareResult<Partial<AgentBuiltInState>> | undefined> => {\n        if (!checkOutput) {\n          return undefined;\n        }\n\n        const messages = state.messages || [];\n        if (messages.length === 0) {\n          return undefined;\n        }\n\n        return (await moderateOutput(messages)) ?? undefined;\n      },\n      canJumpTo: [\"end\"],\n    },\n  });\n}\n"],"mappings":";;;;;;;;;;;AAkCA,SAAS,cAAcA,OAAsC;AAC3D,KACE,CAAC,SACD,OAAO,UAAU,YACjB,UAAU,QACV,EAAE,YAAY,UACd,EAAE,uBAAuB,UACzB,OAAO,MAAM,sBAAsB,WAEnC,QAAO;;;;CAMT,MAAM,mBAAmB;AACzB,QACE,OAAO,MAAM,WAAW,YACxB,MAAM,WAAW,QACjB,iBAAiB,MAAM,UACvB,OAAO,MAAM,OAAO,gBAAgB,YACpC,MAAM,OAAO,gBAAgB,QAC7B,YAAY,MAAM,OAAO,eACzB,OAAO,MAAM,OAAO,YAAY,WAAW;AAE9C;;;;AAUD,MAAM,6BACJ;;;;AAgCF,IAAa,wBAAb,cAA2C,MAAM;CAC/C;CACA;CACA;CACA;CAEA,YAAY,EACV,SACA,OACA,QACA,SAMD,EAAE;EACD,MAAM,QAAQ;EACd,KAAK,OAAO;EACZ,KAAK,UAAU;EACf,KAAK,QAAQ;EACb,KAAK,SAAS;EACd,KAAK,kBAAkB;CACxB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgID,SAAgB,2BACdC,SACiB;CACjB,MAAM,EACJ,OACA,kBAAkB,0BAClB,aAAa,MACb,cAAc,MACd,mBAAmB,OACnB,eAAe,OACf,kBACD,GAAG;CAEJ,IAAIC;CACJ,MAAM,sBAAsB,YAAkC;AAC5D,MAAI,YACF,QAAO;EAGT,MAAM,gBACJ,OAAO,UAAU,WAAW,MAAMC,4CAAc,MAAM,GAAG;;;;AAK3D,MAAI,CAAC,cAAc,SAAS,CAAC,SAAS,aAAa,CACjD,OAAM,IAAI,MACR,CAAC,iEAAiE,EAAE,cAAc,SAAS,EAAE;;;;AAOjG,MAAI,CAAC,cAAc,cAAc,CAC/B,OAAM,IAAI,MACR;EAIJ,cAAc;AACd,SAAO;CACR;;;;CAKD,MAAM,cAAc,CAACC,YAAwC;AAC3D,MAAI,QAAQ,WAAW,KACrB,QAAO;EAET,MAAM,OAAO,QAAQ;AACrB,SAAO,QAAQ;CAChB;;;;CAKD,MAAM,gBAAgB,CACpBC,UACAC,gBACkB;AAClB,OAAK,IAAI,MAAM,SAAS,SAAS,GAAG,OAAO,GAAG,MAC5C,KAAI,YAAY,WAAW,SAAS,KAAK,CACvC,QAAO;AAGX,SAAO;CACR;;;;CAKD,MAAM,yBAAyB,CAC7BC,SACAC,WACW;EAEX,MAAMC,aAAuB,CAAE;EAC/B,MAAM,gBAAgB,OAAO;AAI7B,OAAK,MAAM,CAAC,MAAM,QAAQ,IAAI,OAAO,QAAQ,cAAc,CACzD,KAAI,SACF,WAAW,KAAK,KAAK,QAAQ,MAAM,IAAI,CAAC;EAI5C,MAAM,gBACJ,WAAW,SAAS,IAChB,WAAW,KAAK,KAAK,GACrB;EAEN,MAAM,WAAW,oBAAoB;EACrC,MAAM,aAAa,KAAK,UACtB,OAAO,iBACP,MACA,EACD;AAED,MAAI;AACF,UAAO,SACJ,QAAQ,gBAAgB,cAAc,CACtC,QAAQ,qBAAqB,WAAW,CACxC,QAAQ,sBAAsB,QAAQ;EAC1C,QAAO;AACN,UAAO;EACR;CACF;CAED,SAAS,gBACPC,OACAC,QAC6B;EAC7B,MAAM,gBAAgB,aAAa,qBAAqB;EACxD,MAAMC,oBAAkB,QAAQ,SAAS;EACzC,MAAM,oBAAoB;GACxB;GACA,OAAOA;EACR;AACD,SAAO,YAAa,OAAO,YAAY,OACrC,mBACA,cACD;CACF;;;;CAKD,MAAM,iBAAiB,CACrBP,UACAQ,OACAC,OACAP,SACAC,WAC6D;EAC7D,MAAM,gBAAgB,uBAAuB,SAAS,OAAO;AAE7D,MAAI,iBAAiB,QACnB,OAAM,IAAI,sBAAsB;GAC9B;GACA;GACA;GACA,SAAS;EACV;AAGH,MAAI,iBAAiB,MACnB,QAAO;GACL,QAAQ;GACR,UAAU,CAAC,IAAIO,oCAAU,EAAE,SAAS,cAAe,EAAE;EACtD;AAGH,MAAI,SAAS,KACX,QAAO;;;;EAMT,MAAM,cAAc,CAAC,GAAG,QAAS;EACjC,MAAM,WAAW,YAAY;EAC7B,MAAM,qBAAqB,OAAO,eAAe,SAAS,CAAC;EAC3D,YAAY,SAAS,IAAI,mBAAmB;GAC1C,GAAG;GACH,SAAS;EACV;AAED,SAAO,EAAE,UAAU,YAAa;CACjC;;;;CAKD,MAAM,sBAAsB,OAC1BV,aACiE;EACjE,MAAM,MAAM,cAAc,UAAUW,uCAAa;AACjD,MAAI,OAAO,KACT,QAAO;EAGT,MAAM,UAAU,SAAS;EACzB,MAAM,OAAO,YAAY,QAAQ;AACjC,MAAI,CAAC,KACH,QAAO;EAGT,MAAM,qBAAqB;EAC3B,MAAM,WAAW,MAAM,gBAAgB,MAAM,EAC3C,OAAO,gBACR,EAAC;EAEF,MAAM,gBAAgB,SAAS,QAAQ,KAAK,CAAC,WAAW,OAAO,QAAQ;AACvE,MAAI,CAAC,cACH,QAAO;AAGT,SAAO,eAAe,UAAU,KAAK,SAAS,MAAM,cAAc;CACnE;;;;CAKD,MAAM,uBAAuB,OAC3BX,aACiE;EACjE,MAAM,YAAY,cAAc,UAAUU,oCAAU;AACpD,MAAI,aAAa,KACf,QAAO;EAGT,MAAM,UAAU,CAAC,GAAG,QAAS;EAC7B,IAAI,WAAW;AAEf,OAAK,IAAI,MAAM,YAAY,GAAG,MAAM,QAAQ,QAAQ,OAAO;GACzD,MAAM,MAAM,QAAQ;AACpB,OAAI,CAACE,sCAAY,WAAW,IAAI,CAC9B;GAGF,MAAM,OAAO,YAAY,IAAI;AAC7B,OAAI,CAAC,KACH;GAGF,MAAM,qBAAqB;GAC3B,MAAM,WAAW,MAAM,gBAAgB,MAAM,EAC3C,OAAO,gBACR,EAAC;GACF,MAAM,gBAAgB,SAAS,QAAQ,KAAK,CAAC,WAAW,OAAO,QAAQ;AACvE,OAAI,CAAC,cACH;GAGF,MAAM,SAAS,eAAe,SAAS,KAAK,QAAQ,MAAM,cAAc;AACxE,OAAI,QAAQ;AACV,QAAI,YAAY,OACd,QAAO;AAET,QAAI,cAAc,QAAQ;KACxB,QAAQ,OACN,GACA,QAAQ,QACR,GAAI,OAAO,SACZ;KACD,WAAW;IACZ;GACF;EACF;AAED,MAAI,SACF,QAAO,EAAE,UAAU,QAAS;AAG9B,SAAO;CACR;;;;CAKD,MAAM,iBAAiB,OACrBZ,aACiE;EACjE,MAAM,YAAY,cAAc,UAAUU,oCAAU;AACpD,MAAI,aAAa,KACf,QAAO;EAGT,MAAM,YAAY,SAAS;EAC3B,MAAM,OAAO,YAAY,UAAU;AACnC,MAAI,CAAC,KACH,QAAO;EAGT,MAAM,qBAAqB;EAC3B,MAAM,WAAW,MAAM,gBAAgB,MAAM,EAC3C,OAAO,gBACR,EAAC;EACF,MAAM,gBAAgB,SAAS,QAAQ,KAAK,CAAC,WAAW,OAAO,QAAQ;AACvE,MAAI,CAAC,cACH,QAAO;AAGT,SAAO,eAAe,UAAU,WAAW,UAAU,MAAM,cAAc;CAC1E;;;;CAKD,MAAM,iBAAiB,OACrBV,aACiE;EACjE,MAAM,UAAU,CAAC,GAAG,QAAS;EAC7B,IAAI,WAAW;AAEf,MAAI,kBAAkB;GACpB,MAAM,SAAS,MAAM,qBAAqB,QAAQ;AAClD,OAAI,QAAQ;AACV,QAAI,YAAY,OACd,QAAO;AAET,QAAI,cAAc,QAAQ;KACxB,QAAQ,OACN,GACA,QAAQ,QACR,GAAI,OAAO,SACZ;KACD,WAAW;IACZ;GACF;EACF;AAED,MAAI,YAAY;GACd,MAAM,SAAS,MAAM,oBAAoB,QAAQ;AACjD,OAAI,QAAQ;AACV,QAAI,YAAY,OACd,QAAO;AAET,QAAI,cAAc,QAAQ;KACxB,QAAQ,OACN,GACA,QAAQ,QACR,GAAI,OAAO,SACZ;KACD,WAAW;IACZ;GACF;EACF;AAED,MAAI,SACF,QAAO,EAAE,UAAU,QAAS;AAG9B,SAAO;CACR;AAED,QAAOa,oCAAiB;EACtB,MAAM;EACN,aAAa;GACX,MAAM,OACJ,UACsE;AACtE,QAAI,CAAC,cAAc,CAAC,iBAClB,QAAO;IAGT,MAAM,WAAW,MAAM,YAAY,CAAE;AACrC,QAAI,SAAS,WAAW,EACtB,QAAO;AAGT,WAAQ,MAAM,eAAe,SAAS,IAAK;GAC5C;GACD,WAAW,CAAC,KAAM;EACnB;EACD,YAAY;GACV,MAAM,OACJ,UACsE;AACtE,QAAI,CAAC,YACH,QAAO;IAGT,MAAM,WAAW,MAAM,YAAY,CAAE;AACrC,QAAI,SAAS,WAAW,EACtB,QAAO;AAGT,WAAQ,MAAM,eAAe,SAAS,IAAK;GAC5C;GACD,WAAW,CAAC,KAAM;EACnB;CACF,EAAC;AACH"}

package/dist/agents/middleware/provider/openai/moderation.d.cts

@@ -0,0 +1,133 @@
+import { AgentMiddleware } from "../../types.cjs";
+import { BaseChatModel } from "@langchain/core/language_models/chat_models";
+
+//#region src/agents/middleware/provider/openai/moderation.d.ts
+
+type ModerationModel = "omni-moderation-latest" | "omni-moderation-2024-09-26" | "text-moderation-latest" | "text-moderation-stable";
+/**
+ * Error raised when OpenAI flags content and `exitBehavior` is set to `"error"`.
+ */
+
+/**
+ * Options for configuring the OpenAI Moderation middleware.
+ */
+interface OpenAIModerationMiddlewareOptions {
+  /**
+   * OpenAI model to use for moderation. Can be either a model name or a BaseChatModel instance.
+   * @example
+   * ```ts
+   * const model = new ChatOpenAI({ model: "gpt-4o-mini" });
+   * const middleware = openAIModerationMiddleware({ model });
+   * const agent = createAgent({
+   *   model,
+   *   middleware: [middleware],
+   * });
+   * ```
+   * @example
+   * ```ts
+   * const middleware = openAIModerationMiddleware({ model: "gpt-4o-mini" });
+   * const agent = createAgent({
+   *   model: "gpt-5",
+   *   middleware: [middleware],
+   * });
+   * ```
+   */
+  model: string | BaseChatModel;
+  /**
+   * Moderation model to use.
+   * @default "omni-moderation-latest"
+   */
+  moderationModel?: ModerationModel;
+  /**
+   * Whether to check user input messages.
+   * @default true
+   */
+  checkInput?: boolean;
+  /**
+   * Whether to check model output messages.
+   * @default true
+   */
+  checkOutput?: boolean;
+  /**
+   * Whether to check tool result messages.
+   * @default false
+   */
+  checkToolResults?: boolean;
+  /**
+   * How to handle violations.
+   * - `"error"`: Throw an error when content is flagged
+   * - `"end"`: End the agent execution and return a violation message
+   * - `"replace"`: Replace the flagged content with a violation message
+   * @default "end"
+   */
+  exitBehavior?: "error" | "end" | "replace";
+  /**
+   * Custom template for violation messages.
+   * Available placeholders: `{categories}`, `{category_scores}`, `{original_content}`
+   */
+  violationMessage?: string;
+}
+/**
+ * Middleware that moderates agent traffic using OpenAI's moderation endpoint.
+ *
+ * This middleware checks messages for content policy violations at different stages:
+ * - Input: User messages before they reach the model
+ * - Output: AI model responses
+ * - Tool results: Results returned from tool executions
+ *
+ * @param options - Configuration options for the middleware
+ * @param options.model - OpenAI model to use for moderation. Can be either a model name or a BaseChatModel instance.
+ * @param options.moderationModel - Moderation model to use.
+ * @param options.checkInput - Whether to check user input messages.
+ * @param options.checkOutput - Whether to check model output messages.
+ * @param options.checkToolResults - Whether to check tool result messages.
+ * @param options.exitBehavior - How to handle violations.
+ * @param options.violationMessage - Custom template for violation messages.
+ * @returns Middleware function that can be used to moderate agent traffic.
+ *
+ * @example  Using model instance
+ * ```ts
+ * import { createAgent, openAIModerationMiddleware } from "langchain";
+ *
+ * const middleware = openAIModerationMiddleware({
+ *   checkInput: true,
+ *   checkOutput: true,
+ *   exitBehavior: "end"
+ * });
+ *
+ * const agent = createAgent({
+ *   model: "openai:gpt-4o",
+ *   tools: [...],
+ *   middleware: [middleware],
+ * });
+ * ```
+ *
+ * @example Using model name
+ * ```ts
+ * import { createAgent, openAIModerationMiddleware } from "langchain";
+ *
+ * const middleware = openAIModerationMiddleware({
+ *   model: "gpt-4o-mini",
+ *   checkInput: true,
+ *   checkOutput: true,
+ *   exitBehavior: "end"
+ * });
+ *
+ * const agent = createAgent({
+ *   model: "openai:gpt-4o",
+ *   tools: [...],
+ *   middleware: [middleware],
+ * });
+ * ```
+ *
+ * @example Custom violation message
+ * ```ts
+ * const middleware = openAIModerationMiddleware({
+ *   violationMessage: "Content flagged: {categories}. Scores: {category_scores}"
+ * });
+ * ```
+ */
+declare function openAIModerationMiddleware(options: OpenAIModerationMiddlewareOptions): AgentMiddleware;
+//#endregion
+export { OpenAIModerationMiddlewareOptions, openAIModerationMiddleware };
+//# sourceMappingURL=moderation.d.cts.map

package/dist/agents/middleware/provider/openai/moderation.d.ts

@@ -0,0 +1,133 @@
+import { AgentMiddleware } from "../../types.js";
+import { BaseChatModel } from "@langchain/core/language_models/chat_models";
+
+//#region src/agents/middleware/provider/openai/moderation.d.ts
+
+type ModerationModel = "omni-moderation-latest" | "omni-moderation-2024-09-26" | "text-moderation-latest" | "text-moderation-stable";
+/**
+ * Error raised when OpenAI flags content and `exitBehavior` is set to `"error"`.
+ */
+
+/**
+ * Options for configuring the OpenAI Moderation middleware.
+ */
+interface OpenAIModerationMiddlewareOptions {
+  /**
+   * OpenAI model to use for moderation. Can be either a model name or a BaseChatModel instance.
+   * @example
+   * ```ts
+   * const model = new ChatOpenAI({ model: "gpt-4o-mini" });
+   * const middleware = openAIModerationMiddleware({ model });
+   * const agent = createAgent({
+   *   model,
+   *   middleware: [middleware],
+   * });
+   * ```
+   * @example
+   * ```ts
+   * const middleware = openAIModerationMiddleware({ model: "gpt-4o-mini" });
+   * const agent = createAgent({
+   *   model: "gpt-5",
+   *   middleware: [middleware],
+   * });
+   * ```
+   */
+  model: string | BaseChatModel;
+  /**
+   * Moderation model to use.
+   * @default "omni-moderation-latest"
+   */
+  moderationModel?: ModerationModel;
+  /**
+   * Whether to check user input messages.
+   * @default true
+   */
+  checkInput?: boolean;
+  /**
+   * Whether to check model output messages.
+   * @default true
+   */
+  checkOutput?: boolean;
+  /**
+   * Whether to check tool result messages.
+   * @default false
+   */
+  checkToolResults?: boolean;
+  /**
+   * How to handle violations.
+   * - `"error"`: Throw an error when content is flagged
+   * - `"end"`: End the agent execution and return a violation message
+   * - `"replace"`: Replace the flagged content with a violation message
+   * @default "end"
+   */
+  exitBehavior?: "error" | "end" | "replace";
+  /**
+   * Custom template for violation messages.
+   * Available placeholders: `{categories}`, `{category_scores}`, `{original_content}`
+   */
+  violationMessage?: string;
+}
+/**
+ * Middleware that moderates agent traffic using OpenAI's moderation endpoint.
+ *
+ * This middleware checks messages for content policy violations at different stages:
+ * - Input: User messages before they reach the model
+ * - Output: AI model responses
+ * - Tool results: Results returned from tool executions
+ *
+ * @param options - Configuration options for the middleware
+ * @param options.model - OpenAI model to use for moderation. Can be either a model name or a BaseChatModel instance.
+ * @param options.moderationModel - Moderation model to use.
+ * @param options.checkInput - Whether to check user input messages.
+ * @param options.checkOutput - Whether to check model output messages.
+ * @param options.checkToolResults - Whether to check tool result messages.
+ * @param options.exitBehavior - How to handle violations.
+ * @param options.violationMessage - Custom template for violation messages.
+ * @returns Middleware function that can be used to moderate agent traffic.
+ *
+ * @example  Using model instance
+ * ```ts
+ * import { createAgent, openAIModerationMiddleware } from "langchain";
+ *
+ * const middleware = openAIModerationMiddleware({
+ *   checkInput: true,
+ *   checkOutput: true,
+ *   exitBehavior: "end"
+ * });
+ *
+ * const agent = createAgent({
+ *   model: "openai:gpt-4o",
+ *   tools: [...],
+ *   middleware: [middleware],
+ * });
+ * ```
+ *
+ * @example Using model name
+ * ```ts
+ * import { createAgent, openAIModerationMiddleware } from "langchain";
+ *
+ * const middleware = openAIModerationMiddleware({
+ *   model: "gpt-4o-mini",
+ *   checkInput: true,
+ *   checkOutput: true,
+ *   exitBehavior: "end"
+ * });
+ *
+ * const agent = createAgent({
+ *   model: "openai:gpt-4o",
+ *   tools: [...],
+ *   middleware: [middleware],
+ * });
+ * ```
+ *
+ * @example Custom violation message
+ * ```ts
+ * const middleware = openAIModerationMiddleware({
+ *   violationMessage: "Content flagged: {categories}. Scores: {category_scores}"
+ * });
+ * ```
+ */
+declare function openAIModerationMiddleware(options: OpenAIModerationMiddlewareOptions): AgentMiddleware;
+//#endregion
+export { OpenAIModerationMiddlewareOptions, openAIModerationMiddleware };
+//# sourceMappingURL=moderation.d.ts.map