@dvina/agents 0.9.2 → 0.12.0

package/dist/eval/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/eval/index.ts","../../src/eval/config.ts","../../src/eval/suite.ts","../../src/eval/target.ts","../../src/runtime/langchain/model-resolver.ts","../../src/runtime/langchain/utils.ts","../../src/eval/expectations.ts","../../src/eval/evaluators/language.ts","../../src/eval/evaluators/response-content.ts","../../src/eval/evaluators/no-tool-calls.ts"],"sourcesContent":["// ── Configuration ───────────────────────────────────────────────────\nexport { configureEvals } from './config';\nexport type { EvalConfig, CreateTargetFn } from './config';\n\n// ── Suite API ────────────────────────────────────────────────────────\nexport { defineSuite, human, ai, toolResult, fromToolSpecs } from './suite';\nexport type { SuiteConfig, TestCase, ToolDef } from './suite';\n\n// ── Expectations ─────────────────────────────────────────────────────\nexport { toolsCalled, llmJudge, noTools, respondsInLanguage, contains, notContains } from './expectations';\nexport type { Expectation } from './expectations';\n","import type { Agent, Message, ToolDefinition } from '../core/agent.interface';\nimport type { LangchainModelConfig } from '../runtime/langchain/model-resolver';\n\n/** Factory that creates a fresh Agent per test case. Receives extra suite-level tools as ToolDefinition[]. */\nexport type CreateTargetFn = (extraTools: ToolDefinition[]) => Agent | Promise<Agent>;\n\nexport interface EvalConfig {\n\t/** Required for model-based target and LLM evaluators (respondsInLanguage, llmJudge). */\n\tmodelConfig: LangchainModelConfig;\n\t/** Required for model-based target. Also used as fallback for evaluatorModel. */\n\tmodel?: string;\n\t/** Model for evaluators needing LLM calls (language detection, LLM-as-judge). */\n\tevaluatorModel: string;\n\t/** System prompt for model-based target. Ignored when createTarget is used. Can be overridden per-suite or per-case. */\n\tsystemPrompt?: string;\n\t/** Factory that creates a fresh Agent per test case. When set, this is the default target. */\n\tcreateTarget?: CreateTargetFn;\n\t/** Transforms test case messages before sending to target. Simulates production preprocessing (e.g., message enrichment). */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n}\n\nlet _config: EvalConfig | null = null;\n\nexport function configureEvals(config: EvalConfig): void {\n\t_config = config;\n}\n\nexport function getEvalConfig(): EvalConfig {\n\tif (!_config) {\n\t\tthrow new Error('Evals not configured. Call configureEvals() in your vitest setupFiles.');\n\t}\n\treturn _config;\n}\n\n","import * as ls from 'langsmith/vitest';\nimport { BaseMessage } from '@langchain/core/messages';\nimport { createEvalTarget, runAgentTarget, type MockToolDef } from './target';\nimport { type Expectation } from './expectations';\nimport { getEvalConfig, type CreateTargetFn } from './config';\nimport {\n\ttype Message,\n\ttype HumanMessage,\n\ttype AiMessage,\n\ttype ToolMessage,\n\ttype ToolSpec,\n} from '../core/agent.interface';\n\n// ── Message builders ─────────────────────────────────────────────────\n\nexport function human(content: string): HumanMessage {\n\treturn { role: 'human', content: [{ type: 'text', text: content }] };\n}\n\nexport function ai(content: string, toolCalls?: string[]): AiMessage {\n\treturn { role: 'ai', content, ...(toolCalls ? { toolCalls: toolCalls.map((name) => ({ name })) } : {}) };\n}\n\nexport function toolResult(name: string, output: string): ToolMessage {\n\treturn { role: 'tool', name, output };\n}\n\nexport interface ToolDef {\n\tdescription: string;\n\t/** A plain key→description record, or a ZodObject passed through from a ToolSpec. */\n\tschema?: Record<string, string> | import('zod').ZodObject<any>;\n\t/** Auto-stringified if not a string or function. */\n\tresponse: unknown | ((input: Record<string, unknown>, callCount: number) => string);\n}\n\nexport interface TestCase {\n\t/** Test name. Defaults to the last human message content if omitted. */\n\tname?: string;\n\tmessages: Message[];\n\tsystemPrompt?: string;\n\t/** Override suite-level tools for this case. */\n\ttools?: Record<string, ToolDef>;\n\t/** Transforms messages before sending to target. Overrides suite-level and global hooks. */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n\texpect: Expectation[];\n}\n\ntype TargetFn = (inputs: { systemPrompt?: string; messages: Message[]; tools: MockToolDef[] }) => Promise<{ messages: BaseMessage[] }>;\n\nexport interface SuiteConfig {\n\t/** Custom target function, or model string override. Auto-created from global config if omitted. */\n\ttarget?: TargetFn | string;\n\t/** Factory that creates a fresh Agent per test case. Overrides global createTarget. */\n\tcreateTarget?: CreateTargetFn;\n\t/** System prompt for all cases in this suite. Overrides the global prompt; can be overridden per-case. */\n\tsystemPrompt?: string;\n\ttools?: Record<string, ToolDef>;\n\t/** Transforms messages before sending to target. Overrides global hook; can be overridden per-case. */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n\tcases: TestCase[];\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\n/**\n * Converts a `ToolSpec[]` (from a real tool provider) into the\n * `Record<string, ToolDef>` that `defineSuite` expects.\n *\n * `responses` maps tool names to canned mock responses. Tools without an\n * entry in `responses` default to `''`.\n */\nexport function fromToolSpecs(\n\tspecs: ToolSpec[],\n\tresponses: Record<string, ToolDef['response']> = {},\n): Record<string, ToolDef> {\n\treturn Object.fromEntries(\n\t\tspecs.map((spec) => [\n\t\t\tspec.name,\n\t\t\t{\n\t\t\t\tdescription: spec.description,\n\t\t\t\tschema: spec.inputSchema,\n\t\t\t\tresponse: responses[spec.name] ?? '',\n\t\t\t} satisfies ToolDef,\n\t\t]),\n\t);\n}\n\nfunction toMockTools(defs: Record<string, ToolDef>): MockToolDef[] {\n\treturn Object.entries(defs).map(([name, def]) => ({\n\t\tname,\n\t\tdescription: def.description,\n\t\tschema: def.schema ?? {},\n\t\tresponse:\n\t\t\ttypeof def.response === 'function'\n\t\t\t\t? (def.response as MockToolDef['response'])\n\t\t\t\t: typeof def.response === 'string'\n\t\t\t\t\t? def.response\n\t\t\t\t\t: JSON.stringify(def.response),\n\t}));\n}\n\n/** Strip function responses and ZodObjects so the object is JSON-serialisable for langsmith hashing. */\nfunction toSerializableTools(tools: MockToolDef[]): Record<string, unknown>[] {\n\treturn tools.map((t) => ({\n\t\t...t,\n\t\tschema: t.schema instanceof Object && 'shape' in t.schema ? '<ZodObject>' : t.schema,\n\t\tresponse: typeof t.response === 'function' ? '<function>' : t.response,\n\t}));\n}\n\nfunction lastHumanContent(messages: Message[]): string {\n\tfor (let i = messages.length - 1; i >= 0; i--) {\n\t\tconst msg = messages[i];\n\t\tif (msg.role === 'human') {\n\t\t\tconst textBlock = msg.content.find((c) => c.type === 'text');\n\t\t\treturn textBlock ? textBlock.text : '';\n\t\t}\n\t}\n\treturn '';\n}\n\n// ── Main entry point ─────────────────────────────────────────────────\n\n/**\n * Defines an eval suite. Internally registers `ls.describe` / `ls.test`\n * so vitest discovers the tests — eval files only need to call this function.\n */\nfunction resolveModelTarget(config: SuiteConfig): TargetFn {\n\tif (typeof config.target === 'function') return config.target;\n\tconst evalConfig = getEvalConfig();\n\tif (!evalConfig.model && typeof config.target !== 'string') {\n\t\tthrow new Error('model is required for model-based target. Add it to your configureEvals() call.');\n\t}\n\tconst model = typeof config.target === 'string' ? config.target : evalConfig.model!;\n\treturn createEvalTarget(evalConfig.modelConfig, model);\n}\n\nfunction resolveCreateTarget(config: SuiteConfig): CreateTargetFn | undefined {\n\treturn config.createTarget ?? getEvalConfig().createTarget;\n}\n\nexport function defineSuite(name: string, config: SuiteConfig): void {\n\tconst suiteTools = config.tools ?? {};\n\tconst createTarget = config.target ? undefined : resolveCreateTarget(config);\n\n\tls.describe(name, () => {\n\t\tfor (const tc of config.cases) {\n\t\t\tconst testName = tc.name ?? lastHumanContent(tc.messages);\n\t\t\tconst caseToolDefs = tc.tools ?? suiteTools;\n\t\t\tconst tools = toMockTools(caseToolDefs);\n\t\t\tconst ctx = { message: lastHumanContent(tc.messages) };\n\n\t\t\tconst resolved = tc.expect.map((exp) => exp(ctx));\n\t\t\tconst evaluators = resolved.map((r) => r.evaluator);\n\t\t\tconst referenceOutputs = Object.assign({}, ...resolved.map((r) => r.referenceOutputs));\n\n\t\t\tls.test(\n\t\t\t\ttestName,\n\t\t\t\t{\n\t\t\t\t\tinputs: {\n\t\t\t\t\t\tmessages: tc.messages,\n\t\t\t\t\t\ttools: toSerializableTools(tools),\n\t\t\t\t\t},\n\t\t\t\t\treferenceOutputs,\n\t\t\t\t},\n\t\t\t\tasync ({ referenceOutputs: refOut }) => {\n\t\t\t\t\tlet output: { messages: BaseMessage[] };\n\n\t\t\t\t\t// Resolution order: case > suite > global > identity\n\t\t\t\t\tconst prepareMessages =\n\t\t\t\t\t\ttc.prepareMessages ?? config.prepareMessages ?? getEvalConfig().prepareMessages;\n\t\t\t\t\tconst preparedMessages = prepareMessages\n\t\t\t\t\t\t? await prepareMessages(tc.messages)\n\t\t\t\t\t\t: tc.messages;\n\n\t\t\t\t\tif (createTarget) {\n\t\t\t\t\t\t// Agent mode: create a fresh agent per test, run it, convert result\n\t\t\t\t\t\toutput = await runAgentTarget(createTarget, preparedMessages, caseToolDefs);\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// Model mode: use model-based target with mock tools\n\t\t\t\t\t\tconst target = resolveModelTarget(config);\n\t\t\t\t\t\tconst globalPrompt = getEvalConfig().systemPrompt;\n\t\t\t\t\t\tconst systemPrompt = tc.systemPrompt ?? config.systemPrompt ?? globalPrompt;\n\t\t\t\t\t\toutput = await target({\n\t\t\t\t\t\t\tmessages: preparedMessages,\n\t\t\t\t\t\t\ttools,\n\t\t\t\t\t\t\t...(systemPrompt ? { systemPrompt } : {}),\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\n\t\t\t\t\tls.logOutputs(output);\n\t\t\t\t\tfor (const evaluator of evaluators) {\n\t\t\t\t\t\tawait evaluator({ outputs: output, referenceOutputs: refOut ?? {} });\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t);\n\t\t}\n\t});\n}\n","import { BaseChatModel } from '@langchain/core/language_models/chat_models';\nimport { tool } from '@langchain/core/tools';\nimport { AIMessage, BaseMessage, SystemMessage, ToolMessage } from '@langchain/core/messages';\nimport { z } from 'zod';\nimport { LangchainModelResolver, type LangchainModelConfig } from '../runtime/langchain/model-resolver';\nimport { type CreateTargetFn, getEvalConfig } from './config';\nimport type { AgentResult, Message as AgentMessage, ToolCallContentBlock, ToolDefinition } from '../core/agent.interface';\nimport { convertToLangchainMessages } from '../runtime/langchain/utils';\nimport type { ToolDef } from './suite';\n\nexport interface MockToolDef {\n\tname: string;\n\tdescription: string;\n\tschema: z.ZodObject<any> | Record<string, unknown>;\n\t/**\n\t * Canned response the mock tool returns.\n\t * Can be a static string, or a function that receives input and returns a response.\n\t * If a function is provided, it receives the full invocation count as a second arg\n\t * to support scenarios like \"first call fails, second call succeeds\".\n\t */\n\tresponse: string | ((input: Record<string, unknown>, callCount: number) => string);\n}\n\nexport interface EvalTargetInput {\n\tsystemPrompt?: string;\n\tmessages: AgentMessage[];\n\ttools: MockToolDef[];\n}\n\nconst MAX_AGENT_LOOPS = 10;\n\n/**\n * Creates a LangSmith-compatible target function that runs an agentic loop\n * with mock tools and returns the full message trajectory.\n */\nexport function createEvalTarget(modelConfig?: LangchainModelConfig, modelString?: string) {\n\treturn async (inputs: EvalTargetInput): Promise<{ messages: BaseMessage[] }> => {\n\t\tconst config = modelConfig && modelString ? { modelConfig, model: modelString } : getEvalConfig();\n\t\tif (!config.model) {\n\t\t\tthrow new Error('model is required for model-based target. Add it to your configureEvals() call.');\n\t\t}\n\t\tconst resolver = new LangchainModelResolver(config.modelConfig);\n\t\tconst model = resolver.resolve(config.model) as BaseChatModel;\n\n\t\t// Track invocation counts per tool for stateful mock responses\n\t\tconst toolCallCounts: Record<string, number> = {};\n\n\t\t// Create langchain tools from mock definitions\n\t\tconst langchainTools = inputs.tools.map((mockTool) => {\n\t\t\ttoolCallCounts[mockTool.name] = 0;\n\n\t\t\treturn tool(\n\t\t\t\tasync (toolInput: Record<string, unknown>) => {\n\t\t\t\t\ttoolCallCounts[mockTool.name]++;\n\t\t\t\t\tif (typeof mockTool.response === 'function') {\n\t\t\t\t\t\treturn mockTool.response(toolInput, toolCallCounts[mockTool.name]);\n\t\t\t\t\t}\n\t\t\t\t\treturn mockTool.response;\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tname: mockTool.name,\n\t\t\t\t\tdescription: mockTool.description,\n\t\t\t\t\tschema:\n\t\t\t\t\t\tmockTool.schema instanceof z.ZodObject\n\t\t\t\t\t\t\t? mockTool.schema\n\t\t\t\t\t\t\t: z.object(\n\t\t\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\t\t\tObject.entries(mockTool.schema).map(([key, val]) => {\n\t\t\t\t\t\t\t\t\t\t\tif (typeof val === 'string') return [key, z.string().describe(val)];\n\t\t\t\t\t\t\t\t\t\t\tif (typeof val === 'number') return [key, z.number().describe(String(val))];\n\t\t\t\t\t\t\t\t\t\t\treturn [key, z.any()];\n\t\t\t\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t\t\t\t),\n\t\t\t\t\t\t\t\t),\n\t\t\t\t},\n\t\t\t);\n\t\t});\n\n\t\tconst boundModel = langchainTools.length > 0 ? model.bindTools!(langchainTools) : model;\n\n\t\tconst messages: BaseMessage[] = [];\n\n\t\tif (inputs.systemPrompt) {\n\t\t\tmessages.push(new SystemMessage(inputs.systemPrompt));\n\t\t}\n\n\t\t// Convert and push all messages (history + final human)\n\t\tmessages.push(...convertToLangchainMessages(inputs.messages));\n\n\t\t// Agentic loop: keep calling model until it stops making tool calls\n\t\tlet loopCount = 0;\n\t\twhile (loopCount < MAX_AGENT_LOOPS) {\n\t\t\tloopCount++;\n\n\t\t\tconst response = await boundModel.invoke(messages);\n\t\t\tmessages.push(response as BaseMessage);\n\n\t\t\tconst aiMessage = response as AIMessage;\n\t\t\tif (!aiMessage.tool_calls || aiMessage.tool_calls.length === 0) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\t// Execute tool calls and add results\n\t\t\tfor (const tc of aiMessage.tool_calls) {\n\t\t\t\tconst mockTool = langchainTools.find((t) => t.name === tc.name);\n\t\t\t\tif (mockTool) {\n\t\t\t\t\tconst result = await mockTool.invoke(tc.args);\n\t\t\t\t\tmessages.push(\n\t\t\t\t\t\tnew ToolMessage({\n\t\t\t\t\t\t\tcontent: typeof result === 'string' ? result : JSON.stringify(result),\n\t\t\t\t\t\t\ttool_call_id: tc.id!,\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t} else {\n\t\t\t\t\tmessages.push(\n\t\t\t\t\t\tnew ToolMessage({\n\t\t\t\t\t\t\tcontent: `Tool \"${tc.name}\" not found`,\n\t\t\t\t\t\t\ttool_call_id: tc.id!,\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn { messages };\n\t};\n}\n\n// ── Agent-based target ──────────────────────────────────────────────\n\n/**\n * Converts an `AgentResult` (from `Agent.run()`) into LangChain `BaseMessage[]`\n * so existing evaluators (trajectory match, no-tool-calls, response-content, language) work unchanged.\n *\n * Consecutive `tool_call` content blocks are grouped into a single `AIMessage` with `tool_calls`,\n * followed by one `ToolMessage` per call.\n */\nexport function agentResultToMessages(inputMessages: AgentMessage[], result: AgentResult): BaseMessage[] {\n\t// Include input messages for trajectory context\n\tconst messages: BaseMessage[] = convertToLangchainMessages(inputMessages);\n\n\t// Group content blocks into BaseMessages\n\tlet pendingToolCalls: { id: string; name: string; args: Record<string, unknown>; output: string }[] = [];\n\n\tfor (const block of result.content) {\n\t\tif (block.type === 'tool_call') {\n\t\t\tconst tc = block as ToolCallContentBlock;\n\t\t\tpendingToolCalls.push({\n\t\t\t\tid: tc.toolCallId,\n\t\t\t\tname: tc.name,\n\t\t\t\targs: tc.input ? JSON.parse(tc.input) : {},\n\t\t\t\toutput: tc.output,\n\t\t\t});\n\t\t} else if (block.type === 'text') {\n\t\t\t// Flush any pending tool calls before the text block\n\t\t\tif (pendingToolCalls.length > 0) {\n\t\t\t\tmessages.push(\n\t\t\t\t\tnew AIMessage({\n\t\t\t\t\t\tcontent: '',\n\t\t\t\t\t\ttool_calls: pendingToolCalls.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })),\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t\tfor (const tc of pendingToolCalls) {\n\t\t\t\t\tmessages.push(new ToolMessage({ content: tc.output, tool_call_id: tc.id, name: tc.name }));\n\t\t\t\t}\n\t\t\t\tpendingToolCalls = [];\n\t\t\t}\n\t\t\tmessages.push(new AIMessage(block.output));\n\t\t}\n\t}\n\n\t// Flush remaining tool calls (agent ended mid-tool-use, unlikely but safe)\n\tif (pendingToolCalls.length > 0) {\n\t\tmessages.push(\n\t\t\tnew AIMessage({\n\t\t\t\tcontent: '',\n\t\t\t\ttool_calls: pendingToolCalls.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })),\n\t\t\t}),\n\t\t);\n\t\tfor (const tc of pendingToolCalls) {\n\t\t\tmessages.push(new ToolMessage({ content: tc.output, tool_call_id: tc.id, name: tc.name }));\n\t\t}\n\t}\n\n\treturn messages;\n}\n\n/**\n * Converts eval `Record<string, ToolDef>` into `ToolDefinition[]` with mock `exec` functions,\n * suitable for passing to an `AgentFactory.createAgent()` call.\n */\nexport function toolDefsToDefinitions(defs: Record<string, ToolDef>): ToolDefinition[] {\n\tconst callCounts: Record<string, number> = {};\n\n\treturn Object.entries(defs).map(([name, def]) => {\n\t\tcallCounts[name] = 0;\n\n\t\treturn {\n\t\t\tname,\n\t\t\ttoolKit: 'eval-mock',\n\t\t\tdescription: def.description,\n\t\t\tinputSchema:\n\t\t\t\tdef.schema instanceof z.ZodObject\n\t\t\t\t\t? def.schema\n\t\t\t\t\t: z.object(\n\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\tObject.entries(def.schema ?? {}).map(([key, val]) => {\n\t\t\t\t\t\t\t\t\tif (typeof val === 'string') return [key, z.string().describe(val)];\n\t\t\t\t\t\t\t\t\treturn [key, z.any()];\n\t\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t),\n\t\t\texec: async (input: Record<string, unknown>) => {\n\t\t\t\tcallCounts[name]++;\n\t\t\t\tif (typeof def.response === 'function') {\n\t\t\t\t\treturn (def.response as (input: Record<string, unknown>, callCount: number) => string)(\n\t\t\t\t\t\tinput,\n\t\t\t\t\t\tcallCounts[name],\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\treturn typeof def.response === 'string' ? def.response : JSON.stringify(def.response);\n\t\t\t},\n\t\t} satisfies ToolDefinition;\n\t});\n}\n\n/**\n * Runs a real `Agent` as the eval target. Creates a fresh agent per invocation via the factory,\n * sends human messages, and converts the `AgentResult` to `{ messages: BaseMessage[] }`.\n */\nexport async function runAgentTarget(\n\tcreateTarget: CreateTargetFn,\n\tevalMessages: AgentMessage[],\n\textraToolDefs: Record<string, ToolDef>,\n): Promise<{ messages: BaseMessage[] }> {\n\tconst extraTools = Object.keys(extraToolDefs).length > 0 ? toolDefsToDefinitions(extraToolDefs) : [];\n\tconst agent = await createTarget(extraTools);\n\n\tconst result = await agent.run({\n\t\tthreadId: `eval_${Date.now()}_${Math.random().toString(36).slice(2)}`,\n\t\tmessages: evalMessages,\n\t});\n\n\treturn { messages: agentResultToMessages(evalMessages, result) };\n}\n","import { BaseLanguageModel } from '@langchain/core/language_models/base';\nimport { ChatAnthropic } from '@langchain/anthropic';\nimport { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';\nimport { ReasoningEffort } from 'openai/resources';\n\nexport type LangchainOpenAIConfig = {\n\tapiKey: string;\n};\n\nexport type AzureModelProvider = 'openai' | 'anthropic';\n\nexport type LangchainAzureResourceConfig = {\n\tapiKey: string;\n\tmodels: {\n\t\tmodel: string;\n\t\tprovider: AzureModelProvider;\n\t\tendpoint: string;\n\t\tapiVersion: string;\n\t\tdeploymentName: string;\n\t}[];\n};\n\nexport type ResourceName = string;\n\nexport type LangchainModelConfig = {\n\topenai?: Record<string, LangchainOpenAIConfig>;\n\tazure?: Record<ResourceName, LangchainAzureResourceConfig>;\n};\n\nexport class LangchainModelResolver {\n\tconstructor(private config: LangchainModelConfig) {}\n\n\tresolve(modelString: string, tags?: string[], reasoningEffort?: ReasoningEffort): BaseLanguageModel {\n\t\tconst parts = modelString.split(':');\n\n\t\tif (parts.length === 1) {\n\t\t\tconst fullModelString = this.resolveFullModelString(modelString);\n\t\t\treturn this.resolve(fullModelString, tags, reasoningEffort);\n\t\t}\n\n\t\tif (parts.length === 2) {\n\t\t\tconst [provider, modelName] = parts;\n\t\t\treturn this.resolveByProvider(provider, modelName, modelName, tags, reasoningEffort);\n\t\t}\n\n\t\tif (parts.length === 3) {\n\t\t\tconst [provider, configName, modelName] = parts;\n\t\t\treturn this.resolveByProvider(provider, configName, modelName, tags, reasoningEffort);\n\t\t}\n\n\t\tthrow new Error(\n\t\t\t'Model string must follow format \"modelName\", \"provider:modelName\", or \"provider:configName:modelName\"',\n\t\t);\n\t}\n\n\tprivate resolveFullModelString(modelName: string): string {\n\t\tfor (const [provider, resources] of Object.entries(this.config)) {\n\t\t\tif (provider === 'openai') {\n\t\t\t\tif (modelName in (resources as Record<string, unknown>)) {\n\t\t\t\t\treturn `openai:${modelName}`;\n\t\t\t\t}\n\t\t\t} else if (provider === 'azure') {\n\t\t\t\tfor (const [resource, config] of Object.entries(\n\t\t\t\t\tresources as Record<string, { models: { model: string }[] }>,\n\t\t\t\t)) {\n\t\t\t\t\tif (config.models.some((m) => m.model === modelName)) {\n\t\t\t\t\t\treturn `azure:${resource}:${modelName}`;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tthrow new Error(`Model \"${modelName}\" not found in model config`);\n\t}\n\n\tprivate resolveByProvider(\n\t\tprovider: string,\n\t\tconfigName: string,\n\t\tmodelName: string,\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): BaseLanguageModel {\n\t\tswitch (provider) {\n\t\t\tcase 'openai':\n\t\t\t\treturn this.resolveOpenAI(configName, modelName, tags, reasoningEffort);\n\t\t\tcase 'azure':\n\t\t\t\treturn this.resolveAzure(configName, modelName, tags, reasoningEffort);\n\t\t\tdefault:\n\t\t\t\tthrow new Error(`Unsupported model provider: ${provider}`);\n\t\t}\n\t}\n\n\tprivate resolveOpenAI(\n\t\tconfigName: string,\n\t\tmodelName: string,\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): ChatOpenAI {\n\t\tconst providerConfig = this.config.openai?.[configName];\n\t\tif (!providerConfig) {\n\t\t\tthrow new Error(`Configuration \"${configName}\" for provider \"openai\" is missing`);\n\t\t}\n\n\t\treturn new ChatOpenAI({\n\t\t\tapiKey: providerConfig.apiKey,\n\t\t\tmodelName: modelName,\n\t\t\ttags: tags,\n\t\t\t...(reasoningEffort && {\n\t\t\t\treasoning: {\n\t\t\t\t\teffort: reasoningEffort,\n\t\t\t\t\tsummary: 'auto',\n\t\t\t\t},\n\t\t\t\tuseResponsesApi: true,\n\t\t\t}),\n\t\t});\n\t}\n\n\tprivate resolveAzure(\n\t\tresourceName: string,\n\t\tmodelName: string,\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): BaseLanguageModel {\n\t\tconst resource = this.config.azure?.[resourceName];\n\t\tif (!resource) {\n\t\t\tthrow new Error(`Resource \"${resourceName}\" for provider \"azure\" is missing`);\n\t\t}\n\n\t\tconst modelEntry = resource.models.find((m) => m.model === modelName);\n\t\tif (!modelEntry) {\n\t\t\tthrow new Error(`Model \"${modelName}\" not found in Azure resource \"${resourceName}\"`);\n\t\t}\n\n\t\tswitch (modelEntry.provider) {\n\t\t\tcase 'anthropic':\n\t\t\t\treturn this.resolveAzureAnthropic(resource, modelEntry, tags, reasoningEffort);\n\t\t\tcase 'openai':\n\t\t\t\treturn this.resolveAzureOpenAI(resource, modelEntry, tags, reasoningEffort);\n\t\t}\n\t}\n\n\tprivate resolveAzureOpenAI(\n\t\tresource: LangchainAzureResourceConfig,\n\t\tmodelEntry: LangchainAzureResourceConfig['models'][number],\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): AzureChatOpenAI {\n\t\t/**\n\t\t * OpenAI reasoning models require the Responses API which AzureChatOpenAI\n\t\t * does not support. We rewrite the endpoint to the Azure Responses API path.\n\t\t */\n\t\tconst endpoint = reasoningEffort\n\t\t\t? `${modelEntry.endpoint.replace(/\\/$/, '')}/openai/responses?api-version=${modelEntry.apiVersion}`\n\t\t\t: modelEntry.endpoint;\n\n\t\treturn new AzureChatOpenAI({\n\t\t\tmodel: modelEntry.model,\n\t\t\tazureOpenAIApiKey: resource.apiKey,\n\t\t\tazureOpenAIEndpoint: endpoint,\n\t\t\tazureOpenAIApiDeploymentName: modelEntry.deploymentName,\n\t\t\tazureOpenAIApiVersion: modelEntry.apiVersion,\n\t\t\ttags: tags,\n\t\t\t...(reasoningEffort && {\n\t\t\t\treasoning: {\n\t\t\t\t\teffort: reasoningEffort,\n\t\t\t\t\tsummary: 'auto',\n\t\t\t\t},\n\t\t\t}),\n\t\t});\n\t}\n\n\tprivate static readonly THINKING_BUDGET: Record<string, number> = {\n\t\tminimal: 1024,\n\t\tlow: 4096,\n\t\tmedium: 10000,\n\t\thigh: 16000,\n\t\txhigh: 32000,\n\t};\n\n\tprivate resolveAzureAnthropic(\n\t\tresource: LangchainAzureResourceConfig,\n\t\tmodelEntry: LangchainAzureResourceConfig['models'][number],\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): ChatAnthropic {\n\t\tconst budgetTokens = reasoningEffort ? LangchainModelResolver.THINKING_BUDGET[reasoningEffort] : undefined;\n\n\t\treturn new ChatAnthropic({\n\t\t\tmodel: modelEntry.model,\n\t\t\tapiKey: resource.apiKey,\n\t\t\tclientOptions: { baseURL: modelEntry.endpoint },\n\t\t\ttags: tags,\n\t\t\t...(budgetTokens && {\n\t\t\t\tmaxTokens: budgetTokens * 2,\n\t\t\t\tthinking: {\n\t\t\t\t\ttype: 'enabled',\n\t\t\t\t\tbudget_tokens: budgetTokens,\n\t\t\t\t},\n\t\t\t}),\n\t\t});\n\t}\n}\n","import { Message } from '@core/agent.interface';\nimport { AIMessage, BaseMessage, HumanMessage, ToolMessage } from 'langchain';\n\nexport function convertToLangchainMessages(messages: Message[]): BaseMessage[] {\n\tconst result: BaseMessage[] = [];\n\tlet tcIdx = 0;\n\tlet pendingToolCallIds: string[] = [];\n\n\tfor (const msg of messages) {\n\t\tif (msg.role === 'human') {\n\t\t\tresult.push(\n\t\t\t\tnew HumanMessage({\n\t\t\t\t\tcontent: msg.content.map((c) => {\n\t\t\t\t\t\tif (c.type === 'image') {\n\t\t\t\t\t\t\treturn { type: 'image_url', image_url: { url: c.url } };\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn c;\n\t\t\t\t\t}) as any,\n\t\t\t\t}),\n\t\t\t);\n\t\t} else if (msg.role === 'ai') {\n\t\t\tif (msg.toolCalls && msg.toolCalls.length > 0) {\n\t\t\t\tpendingToolCallIds = msg.toolCalls.map(() => `tc_${++tcIdx}`);\n\t\t\t\tresult.push(\n\t\t\t\t\tnew AIMessage({\n\t\t\t\t\t\tcontent: msg.content,\n\t\t\t\t\t\ttool_calls: msg.toolCalls.map((tc, i) => ({\n\t\t\t\t\t\t\tid: pendingToolCallIds[i],\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t\targs: tc.input ? JSON.parse(tc.input) : {},\n\t\t\t\t\t\t})),\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t} else {\n\t\t\t\tresult.push(new AIMessage(msg.content));\n\t\t\t}\n\t\t} else if (msg.role === 'tool') {\n\t\t\tconst toolCallId = pendingToolCallIds.shift();\n\t\t\tif (!toolCallId)\n\t\t\t\tthrow new Error(`ToolMessage for \"${msg.name}\" without a preceding AiMessage with toolCalls`);\n\t\t\tresult.push(\n\t\t\t\tnew ToolMessage({\n\t\t\t\t\tcontent: msg.output,\n\t\t\t\t\ttool_call_id: toolCallId,\n\t\t\t\t\tname: msg.name,\n\t\t\t\t}),\n\t\t\t);\n\t\t}\n\t}\n\n\treturn result;\n}\n","import * as ls from 'langsmith/vitest';\nimport {\n\tcreateTrajectoryMatchEvaluator,\n\tcreateTrajectoryLLMAsJudge,\n\tTRAJECTORY_ACCURACY_PROMPT,\n} from 'agentevals';\nimport { createLanguageEvaluator } from './evaluators/language';\nimport { createResponseContentEvaluator } from './evaluators/response-content';\nimport { createNoToolCallsEvaluator } from './evaluators/no-tool-calls';\nimport { getEvalConfig } from './config';\n\n// ── Types ────────────────────────────────────────────────────────────\n\ntype EvaluatorFn = (args: {\n\toutputs: Record<string, any>;\n\treferenceOutputs: Record<string, any>;\n}) => Promise<any>;\n\ninterface ResolvedExpectation {\n\tevaluator: EvaluatorFn;\n\treferenceOutputs: Record<string, unknown>;\n}\n\n/** A factory that receives test context and returns an evaluator + its referenceOutputs. */\nexport type Expectation = (ctx: { message: string }) => ResolvedExpectation;\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nfunction withTrajectoryGuard(evaluator: any, key: string): EvaluatorFn {\n\treturn async ({ outputs, referenceOutputs }) => {\n\t\tif (!referenceOutputs?.referenceTrajectory) {\n\t\t\treturn { key, score: true, comment: 'No referenceTrajectory specified, skipping' };\n\t\t}\n\t\treturn evaluator({ outputs, referenceOutputs: referenceOutputs.referenceTrajectory });\n\t};\n}\n\nfunction buildTrajectory(message: string, toolNames: string[]): Record<string, unknown>[] {\n\tconst trajectory: Record<string, unknown>[] = [];\n\tlet tcIdx = 0;\n\n\ttrajectory.push({ role: 'user', content: message });\n\n\tfor (const name of toolNames) {\n\t\tconst id = `tc${++tcIdx}`;\n\t\ttrajectory.push({\n\t\t\trole: 'assistant',\n\t\t\tcontent: '',\n\t\t\ttool_calls: [{ function: { name, arguments: '{}' }, id, type: 'function' }],\n\t\t});\n\t\ttrajectory.push({ role: 'tool', content: '...', tool_call_id: id });\n\t}\n\n\ttrajectory.push({ role: 'assistant', content: '...' });\n\n\treturn trajectory;\n}\n\n// ── Expectation functions ────────────────────────────────────────────\n\n/**\n * Expect the agent to call tools in order (superset trajectory match).\n * Empty `[]` means the agent should answer directly without calling any tools.\n */\nexport function toolsCalled(tools: string[]): Expectation {\n\treturn (ctx) => ({\n\t\tevaluator: ls.wrapEvaluator(\n\t\t\twithTrajectoryGuard(\n\t\t\t\tcreateTrajectoryMatchEvaluator({ trajectoryMatchMode: 'superset', toolArgsMatchMode: 'ignore' }) as any,\n\t\t\t\t'trajectory_match',\n\t\t\t),\n\t\t),\n\t\treferenceOutputs: { referenceTrajectory: buildTrajectory(ctx.message, tools) },\n\t});\n}\n\n/**\n * Run an LLM-as-judge evaluator on the trajectory.\n * Requires `toolsCalled` in the same expect array.\n * Uses the globally configured evaluator model.\n */\nexport function llmJudge(): Expectation {\n\treturn () => {\n\t\tconst config = getEvalConfig();\n\t\tconst model = config.evaluatorModel;\n\t\treturn {\n\t\t\tevaluator: ls.wrapEvaluator(\n\t\t\t\twithTrajectoryGuard(\n\t\t\t\t\tcreateTrajectoryLLMAsJudge({ prompt: TRAJECTORY_ACCURACY_PROMPT, model }) as any,\n\t\t\t\t\t'trajectory_llm_judge',\n\t\t\t\t),\n\t\t\t),\n\t\t\treferenceOutputs: {},\n\t\t};\n\t};\n}\n\n/** Assert the agent made zero tool calls. */\nexport function noTools(): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createNoToolCallsEvaluator()),\n\t\treferenceOutputs: { expectNoToolCalls: true },\n\t});\n}\n\n/**\n * Assert the response is in the given language (ISO 639-1 code).\n * Uses the globally configured evaluator model for language detection.\n * @param code - ISO 639-1 language code (e.g. 'en', 'tr', 'de').\n */\nexport function respondsInLanguage(code: string): Expectation {\n\treturn () => {\n\t\tconst config = getEvalConfig();\n\t\tconst model = config.evaluatorModel;\n\t\treturn {\n\t\t\tevaluator: ls.wrapEvaluator(createLanguageEvaluator(config.modelConfig, model)),\n\t\t\treferenceOutputs: { expectedLanguage: code },\n\t\t};\n\t};\n}\n\n/** Assert the response contains all given strings. */\nexport function contains(strings: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createResponseContentEvaluator()),\n\t\treferenceOutputs: { responseContains: strings },\n\t});\n}\n\n/** Assert the response does not contain any of the given strings. */\nexport function notContains(strings: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createResponseContentEvaluator()),\n\t\treferenceOutputs: { responseMustNotContain: strings },\n\t});\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\nimport { BaseChatModel } from '@langchain/core/language_models/chat_models';\nimport { LangchainModelResolver, type LangchainModelConfig } from '../../runtime/langchain/model-resolver';\n\n/**\n * Creates a custom evaluator that checks whether the agent's final response\n * is in the expected language. Uses a cheap LLM call for language detection.\n */\nexport function createLanguageEvaluator(modelConfig: LangchainModelConfig, model: string) {\n\tconst resolver = new LangchainModelResolver(modelConfig);\n\tconst judge = resolver.resolve(model) as BaseChatModel;\n\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tconst expectedLanguage = referenceOutputs?.expectedLanguage;\n\t\tif (!expectedLanguage) {\n\t\t\treturn { key: 'language_match', score: true, comment: 'No expected language specified, skipping' };\n\t\t}\n\n\t\t// Extract the last AI message text from the trajectory\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage);\n\n\t\tif (!lastAiMessage) {\n\t\t\treturn { key: 'language_match', score: false, comment: 'No AI message found in trajectory' };\n\t\t}\n\n\t\tconst responseText = typeof lastAiMessage.content === 'string' ? lastAiMessage.content : JSON.stringify(lastAiMessage.content);\n\n\t\t// Use LLM to detect the language\n\t\tconst detection = await judge.invoke([\n\t\t\t{\n\t\t\t\trole: 'system',\n\t\t\t\tcontent: 'You are a language detection tool. Respond with ONLY the ISO 639-1 language code (e.g., \"en\", \"tr\", \"de\", \"fr\") of the text provided. Nothing else.',\n\t\t\t},\n\t\t\t{\n\t\t\t\trole: 'user',\n\t\t\t\tcontent: responseText,\n\t\t\t},\n\t\t]);\n\n\t\tconst detectedLanguage = (typeof detection.content === 'string' ? detection.content : '').trim().toLowerCase();\n\n\t\tconst matches = detectedLanguage === expectedLanguage.toLowerCase();\n\n\t\treturn {\n\t\t\tkey: 'language_match',\n\t\t\tscore: matches,\n\t\t\tcomment: matches\n\t\t\t\t? `Response language matches expected: ${expectedLanguage}`\n\t\t\t\t: `Expected \"${expectedLanguage}\" but detected \"${detectedLanguage}\"`,\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that checks whether the agent's final response\n * contains expected strings and doesn't contain forbidden strings.\n */\nexport function createResponseContentEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tconst mustContain: string[] = referenceOutputs?.responseContains || [];\n\t\tconst mustNotContain: string[] = referenceOutputs?.responseMustNotContain || [];\n\n\t\tif (mustContain.length === 0 && mustNotContain.length === 0) {\n\t\t\treturn { key: 'response_content', score: true, comment: 'No content assertions specified, skipping' };\n\t\t}\n\n\t\t// Extract the last AI message text from the trajectory\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage);\n\n\t\tif (!lastAiMessage) {\n\t\t\treturn { key: 'response_content', score: false, comment: 'No AI message found in trajectory' };\n\t\t}\n\n\t\tconst responseText = (typeof lastAiMessage.content === 'string' ? lastAiMessage.content : JSON.stringify(lastAiMessage.content)).toLowerCase();\n\n\t\tconst failures: string[] = [];\n\n\t\tfor (const expected of mustContain) {\n\t\t\tif (!responseText.includes(expected.toLowerCase())) {\n\t\t\t\tfailures.push(`Missing expected text: \"${expected}\"`);\n\t\t\t}\n\t\t}\n\n\t\tfor (const forbidden of mustNotContain) {\n\t\t\tif (responseText.includes(forbidden.toLowerCase())) {\n\t\t\t\tfailures.push(`Contains forbidden text: \"${forbidden}\"`);\n\t\t\t}\n\t\t}\n\n\t\tconst passed = failures.length === 0;\n\n\t\treturn {\n\t\t\tkey: 'response_content',\n\t\t\tscore: passed,\n\t\t\tcomment: passed ? 'All content assertions passed' : failures.join('; '),\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that asserts the agent made zero tool calls.\n * Useful for scenarios like greetings where the agent should just respond with text.\n */\nexport function createNoToolCallsEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\t// Only run this evaluator if the reference explicitly expects no tool calls\n\t\tif (referenceOutputs?.maxToolCalls !== 0 && referenceOutputs?.expectNoToolCalls !== true) {\n\t\t\treturn { key: 'no_tool_calls', score: true, comment: 'No tool call restriction specified, skipping' };\n\t\t}\n\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\n\t\tconst toolCalls = messages\n\t\t\t.filter((m) => m instanceof AIMessage)\n\t\t\t.flatMap((m) => (m as AIMessage).tool_calls || []);\n\n\t\tconst passed = toolCalls.length === 0;\n\n\t\treturn {\n\t\t\tkey: 'no_tool_calls',\n\t\t\tscore: passed,\n\t\t\tcomment: passed\n\t\t\t\t? 'No tool calls made (as expected)'\n\t\t\t\t: `Agent made ${toolCalls.length} tool call(s): ${toolCalls.map((tc) => tc.name).join(', ')}`,\n\t\t};\n\t};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACqBA,IAAI,UAA6B;AAE1B,SAAS,eAAe,QAA0B;AACxD,YAAU;AACX;AAEO,SAAS,gBAA4B;AAC3C,MAAI,CAAC,SAAS;AACb,UAAM,IAAI,MAAM,wEAAwE;AAAA,EACzF;AACA,SAAO;AACR;;;AChCA,SAAoB;;;ACCpB,mBAAqB;AACrB,sBAAmE;AACnE,iBAAkB;;;ACFlB,uBAA8B;AAC9B,oBAA4C;AA2BrC,IAAM,yBAAN,MAAM,wBAAuB;AAAA,EACnC,YAAoB,QAA8B;AAA9B;AAAA,EAA+B;AAAA,EAEnD,QAAQ,aAAqB,MAAiB,iBAAsD;AACnG,UAAM,QAAQ,YAAY,MAAM,GAAG;AAEnC,QAAI,MAAM,WAAW,GAAG;AACvB,YAAM,kBAAkB,KAAK,uBAAuB,WAAW;AAC/D,aAAO,KAAK,QAAQ,iBAAiB,MAAM,eAAe;AAAA,IAC3D;AAEA,QAAI,MAAM,WAAW,GAAG;AACvB,YAAM,CAAC,UAAU,SAAS,IAAI;AAC9B,aAAO,KAAK,kBAAkB,UAAU,WAAW,WAAW,MAAM,eAAe;AAAA,IACpF;AAEA,QAAI,MAAM,WAAW,GAAG;AACvB,YAAM,CAAC,UAAU,YAAY,SAAS,IAAI;AAC1C,aAAO,KAAK,kBAAkB,UAAU,YAAY,WAAW,MAAM,eAAe;AAAA,IACrF;AAEA,UAAM,IAAI;AAAA,MACT;AAAA,IACD;AAAA,EACD;AAAA,EAEQ,uBAAuB,WAA2B;AACzD,eAAW,CAAC,UAAU,SAAS,KAAK,OAAO,QAAQ,KAAK,MAAM,GAAG;AAChE,UAAI,aAAa,UAAU;AAC1B,YAAI,aAAc,WAAuC;AACxD,iBAAO,UAAU,SAAS;AAAA,QAC3B;AAAA,MACD,WAAW,aAAa,SAAS;AAChC,mBAAW,CAAC,UAAU,MAAM,KAAK,OAAO;AAAA,UACvC;AAAA,QACD,GAAG;AACF,cAAI,OAAO,OAAO,KAAK,CAAC,MAAM,EAAE,UAAU,SAAS,GAAG;AACrD,mBAAO,SAAS,QAAQ,IAAI,SAAS;AAAA,UACtC;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAEA,UAAM,IAAI,MAAM,UAAU,SAAS,6BAA6B;AAAA,EACjE;AAAA,EAEQ,kBACP,UACA,YACA,WACA,MACA,iBACoB;AACpB,YAAQ,UAAU;AAAA,MACjB,KAAK;AACJ,eAAO,KAAK,cAAc,YAAY,WAAW,MAAM,eAAe;AAAA,MACvE,KAAK;AACJ,eAAO,KAAK,aAAa,YAAY,WAAW,MAAM,eAAe;AAAA,MACtE;AACC,cAAM,IAAI,MAAM,+BAA+B,QAAQ,EAAE;AAAA,IAC3D;AAAA,EACD;AAAA,EAEQ,cACP,YACA,WACA,MACA,iBACa;AACb,UAAM,iBAAiB,KAAK,OAAO,SAAS,UAAU;AACtD,QAAI,CAAC,gBAAgB;AACpB,YAAM,IAAI,MAAM,kBAAkB,UAAU,oCAAoC;AAAA,IACjF;AAEA,WAAO,IAAI,yBAAW;AAAA,MACrB,QAAQ,eAAe;AAAA,MACvB;AAAA,MACA;AAAA,MACA,GAAI,mBAAmB;AAAA,QACtB,WAAW;AAAA,UACV,QAAQ;AAAA,UACR,SAAS;AAAA,QACV;AAAA,QACA,iBAAiB;AAAA,MAClB;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EAEQ,aACP,cACA,WACA,MACA,iBACoB;AACpB,UAAM,WAAW,KAAK,OAAO,QAAQ,YAAY;AACjD,QAAI,CAAC,UAAU;AACd,YAAM,IAAI,MAAM,aAAa,YAAY,mCAAmC;AAAA,IAC7E;AAEA,UAAM,aAAa,SAAS,OAAO,KAAK,CAAC,MAAM,EAAE,UAAU,SAAS;AACpE,QAAI,CAAC,YAAY;AAChB,YAAM,IAAI,MAAM,UAAU,SAAS,kCAAkC,YAAY,GAAG;AAAA,IACrF;AAEA,YAAQ,WAAW,UAAU;AAAA,MAC5B,KAAK;AACJ,eAAO,KAAK,sBAAsB,UAAU,YAAY,MAAM,eAAe;AAAA,MAC9E,KAAK;AACJ,eAAO,KAAK,mBAAmB,UAAU,YAAY,MAAM,eAAe;AAAA,IAC5E;AAAA,EACD;AAAA,EAEQ,mBACP,UACA,YACA,MACA,iBACkB;AAKlB,UAAM,WAAW,kBACd,GAAG,WAAW,SAAS,QAAQ,OAAO,EAAE,CAAC,iCAAiC,WAAW,UAAU,KAC/F,WAAW;AAEd,WAAO,IAAI,8BAAgB;AAAA,MAC1B,OAAO,WAAW;AAAA,MAClB,mBAAmB,SAAS;AAAA,MAC5B,qBAAqB;AAAA,MACrB,8BAA8B,WAAW;AAAA,MACzC,uBAAuB,WAAW;AAAA,MAClC;AAAA,MACA,GAAI,mBAAmB;AAAA,QACtB,WAAW;AAAA,UACV,QAAQ;AAAA,UACR,SAAS;AAAA,QACV;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EAEA,OAAwB,kBAA0C;AAAA,IACjE,SAAS;AAAA,IACT,KAAK;AAAA,IACL,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,OAAO;AAAA,EACR;AAAA,EAEQ,sBACP,UACA,YACA,MACA,iBACgB;AAChB,UAAM,eAAe,kBAAkB,wBAAuB,gBAAgB,eAAe,IAAI;AAEjG,WAAO,IAAI,+BAAc;AAAA,MACxB,OAAO,WAAW;AAAA,MAClB,QAAQ,SAAS;AAAA,MACjB,eAAe,EAAE,SAAS,WAAW,SAAS;AAAA,MAC9C;AAAA,MACA,GAAI,gBAAgB;AAAA,QACnB,WAAW,eAAe;AAAA,QAC1B,UAAU;AAAA,UACT,MAAM;AAAA,UACN,eAAe;AAAA,QAChB;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AACD;;;ACxMA,uBAAkE;AAE3D,SAAS,2BAA2B,UAAoC;AAC9E,QAAM,SAAwB,CAAC;AAC/B,MAAI,QAAQ;AACZ,MAAI,qBAA+B,CAAC;AAEpC,aAAW,OAAO,UAAU;AAC3B,QAAI,IAAI,SAAS,SAAS;AACzB,aAAO;AAAA,QACN,IAAI,8BAAa;AAAA,UAChB,SAAS,IAAI,QAAQ,IAAI,CAAC,MAAM;AAC/B,gBAAI,EAAE,SAAS,SAAS;AACvB,qBAAO,EAAE,MAAM,aAAa,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE;AAAA,YACvD;AACA,mBAAO;AAAA,UACR,CAAC;AAAA,QACF,CAAC;AAAA,MACF;AAAA,IACD,WAAW,IAAI,SAAS,MAAM;AAC7B,UAAI,IAAI,aAAa,IAAI,UAAU,SAAS,GAAG;AAC9C,6BAAqB,IAAI,UAAU,IAAI,MAAM,MAAM,EAAE,KAAK,EAAE;AAC5D,eAAO;AAAA,UACN,IAAI,2BAAU;AAAA,YACb,SAAS,IAAI;AAAA,YACb,YAAY,IAAI,UAAU,IAAI,CAAC,IAAI,OAAO;AAAA,cACzC,IAAI,mBAAmB,CAAC;AAAA,cACxB,MAAM,GAAG;AAAA,cACT,MAAM,GAAG,QAAQ,KAAK,MAAM,GAAG,KAAK,IAAI,CAAC;AAAA,YAC1C,EAAE;AAAA,UACH,CAAC;AAAA,QACF;AAAA,MACD,OAAO;AACN,eAAO,KAAK,IAAI,2BAAU,IAAI,OAAO,CAAC;AAAA,MACvC;AAAA,IACD,WAAW,IAAI,SAAS,QAAQ;AAC/B,YAAM,aAAa,mBAAmB,MAAM;AAC5C,UAAI,CAAC;AACJ,cAAM,IAAI,MAAM,oBAAoB,IAAI,IAAI,gDAAgD;AAC7F,aAAO;AAAA,QACN,IAAI,6BAAY;AAAA,UACf,SAAS,IAAI;AAAA,UACb,cAAc;AAAA,UACd,MAAM,IAAI;AAAA,QACX,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AAEA,SAAO;AACR;;;AFtBA,IAAM,kBAAkB;AAMjB,SAAS,iBAAiB,aAAoC,aAAsB;AAC1F,SAAO,OAAO,WAAkE;AAC/E,UAAM,SAAS,eAAe,cAAc,EAAE,aAAa,OAAO,YAAY,IAAI,cAAc;AAChG,QAAI,CAAC,OAAO,OAAO;AAClB,YAAM,IAAI,MAAM,iFAAiF;AAAA,IAClG;AACA,UAAM,WAAW,IAAI,uBAAuB,OAAO,WAAW;AAC9D,UAAM,QAAQ,SAAS,QAAQ,OAAO,KAAK;AAG3C,UAAM,iBAAyC,CAAC;AAGhD,UAAM,iBAAiB,OAAO,MAAM,IAAI,CAAC,aAAa;AACrD,qBAAe,SAAS,IAAI,IAAI;AAEhC,iBAAO;AAAA,QACN,OAAO,cAAuC;AAC7C,yBAAe,SAAS,IAAI;AAC5B,cAAI,OAAO,SAAS,aAAa,YAAY;AAC5C,mBAAO,SAAS,SAAS,WAAW,eAAe,SAAS,IAAI,CAAC;AAAA,UAClE;AACA,iBAAO,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,UACC,MAAM,SAAS;AAAA,UACf,aAAa,SAAS;AAAA,UACtB,QACC,SAAS,kBAAkB,aAAE,YAC1B,SAAS,SACT,aAAE;AAAA,YACF,OAAO;AAAA,cACN,OAAO,QAAQ,SAAS,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM;AACnD,oBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,aAAE,OAAO,EAAE,SAAS,GAAG,CAAC;AAClE,oBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,aAAE,OAAO,EAAE,SAAS,OAAO,GAAG,CAAC,CAAC;AAC1E,uBAAO,CAAC,KAAK,aAAE,IAAI,CAAC;AAAA,cACrB,CAAC;AAAA,YACF;AAAA,UACD;AAAA,QACJ;AAAA,MACD;AAAA,IACD,CAAC;AAED,UAAM,aAAa,eAAe,SAAS,IAAI,MAAM,UAAW,cAAc,IAAI;AAElF,UAAM,WAA0B,CAAC;AAEjC,QAAI,OAAO,cAAc;AACxB,eAAS,KAAK,IAAI,8BAAc,OAAO,YAAY,CAAC;AAAA,IACrD;AAGA,aAAS,KAAK,GAAG,2BAA2B,OAAO,QAAQ,CAAC;AAG5D,QAAI,YAAY;AAChB,WAAO,YAAY,iBAAiB;AACnC;AAEA,YAAM,WAAW,MAAM,WAAW,OAAO,QAAQ;AACjD,eAAS,KAAK,QAAuB;AAErC,YAAM,YAAY;AAClB,UAAI,CAAC,UAAU,cAAc,UAAU,WAAW,WAAW,GAAG;AAC/D;AAAA,MACD;AAGA,iBAAW,MAAM,UAAU,YAAY;AACtC,cAAM,WAAW,eAAe,KAAK,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI;AAC9D,YAAI,UAAU;AACb,gBAAM,SAAS,MAAM,SAAS,OAAO,GAAG,IAAI;AAC5C,mBAAS;AAAA,YACR,IAAI,4BAAY;AAAA,cACf,SAAS,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,cACpE,cAAc,GAAG;AAAA,cACjB,MAAM,GAAG;AAAA,YACV,CAAC;AAAA,UACF;AAAA,QACD,OAAO;AACN,mBAAS;AAAA,YACR,IAAI,4BAAY;AAAA,cACf,SAAS,SAAS,GAAG,IAAI;AAAA,cACzB,cAAc,GAAG;AAAA,cACjB,MAAM,GAAG;AAAA,YACV,CAAC;AAAA,UACF;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAEA,WAAO,EAAE,SAAS;AAAA,EACnB;AACD;AAWO,SAAS,sBAAsB,eAA+B,QAAoC;AAExG,QAAM,WAA0B,2BAA2B,aAAa;AAGxE,MAAI,mBAAkG,CAAC;AAEvG,aAAW,SAAS,OAAO,SAAS;AACnC,QAAI,MAAM,SAAS,aAAa;AAC/B,YAAM,KAAK;AACX,uBAAiB,KAAK;AAAA,QACrB,IAAI,GAAG;AAAA,QACP,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,QAAQ,KAAK,MAAM,GAAG,KAAK,IAAI,CAAC;AAAA,QACzC,QAAQ,GAAG;AAAA,MACZ,CAAC;AAAA,IACF,WAAW,MAAM,SAAS,QAAQ;AAEjC,UAAI,iBAAiB,SAAS,GAAG;AAChC,iBAAS;AAAA,UACR,IAAI,0BAAU;AAAA,YACb,SAAS;AAAA,YACT,YAAY,iBAAiB,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,MAAM,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE;AAAA,UACvF,CAAC;AAAA,QACF;AACA,mBAAW,MAAM,kBAAkB;AAClC,mBAAS,KAAK,IAAI,4BAAY,EAAE,SAAS,GAAG,QAAQ,cAAc,GAAG,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,QAC1F;AACA,2BAAmB,CAAC;AAAA,MACrB;AACA,eAAS,KAAK,IAAI,0BAAU,MAAM,MAAM,CAAC;AAAA,IAC1C;AAAA,EACD;AAGA,MAAI,iBAAiB,SAAS,GAAG;AAChC,aAAS;AAAA,MACR,IAAI,0BAAU;AAAA,QACb,SAAS;AAAA,QACT,YAAY,iBAAiB,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,MAAM,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE;AAAA,MACvF,CAAC;AAAA,IACF;AACA,eAAW,MAAM,kBAAkB;AAClC,eAAS,KAAK,IAAI,4BAAY,EAAE,SAAS,GAAG,QAAQ,cAAc,GAAG,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,IAC1F;AAAA,EACD;AAEA,SAAO;AACR;AAMO,SAAS,sBAAsB,MAAiD;AACtF,QAAM,aAAqC,CAAC;AAE5C,SAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM;AAChD,eAAW,IAAI,IAAI;AAEnB,WAAO;AAAA,MACN;AAAA,MACA,SAAS;AAAA,MACT,aAAa,IAAI;AAAA,MACjB,aACC,IAAI,kBAAkB,aAAE,YACrB,IAAI,SACJ,aAAE;AAAA,QACF,OAAO;AAAA,UACN,OAAO,QAAQ,IAAI,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM;AACpD,gBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,aAAE,OAAO,EAAE,SAAS,GAAG,CAAC;AAClE,mBAAO,CAAC,KAAK,aAAE,IAAI,CAAC;AAAA,UACrB,CAAC;AAAA,QACF;AAAA,MACD;AAAA,MACH,MAAM,OAAO,UAAmC;AAC/C,mBAAW,IAAI;AACf,YAAI,OAAO,IAAI,aAAa,YAAY;AACvC,iBAAQ,IAAI;AAAA,YACX;AAAA,YACA,WAAW,IAAI;AAAA,UAChB;AAAA,QACD;AACA,eAAO,OAAO,IAAI,aAAa,WAAW,IAAI,WAAW,KAAK,UAAU,IAAI,QAAQ;AAAA,MACrF;AAAA,IACD;AAAA,EACD,CAAC;AACF;AAMA,eAAsB,eACrB,cACA,cACA,eACuC;AACvC,QAAM,aAAa,OAAO,KAAK,aAAa,EAAE,SAAS,IAAI,sBAAsB,aAAa,IAAI,CAAC;AACnG,QAAM,QAAQ,MAAM,aAAa,UAAU;AAE3C,QAAM,SAAS,MAAM,MAAM,IAAI;AAAA,IAC9B,UAAU,QAAQ,KAAK,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AAAA,IACnE,UAAU;AAAA,EACX,CAAC;AAED,SAAO,EAAE,UAAU,sBAAsB,cAAc,MAAM,EAAE;AAChE;;;ADvOO,SAAS,MAAM,SAA+B;AACpD,SAAO,EAAE,MAAM,SAAS,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,QAAQ,CAAC,EAAE;AACpE;AAEO,SAAS,GAAG,SAAiB,WAAiC;AACpE,SAAO,EAAE,MAAM,MAAM,SAAS,GAAI,YAAY,EAAE,WAAW,UAAU,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,IAAI,CAAC,EAAG;AACxG;AAEO,SAAS,WAAW,MAAc,QAA6B;AACrE,SAAO,EAAE,MAAM,QAAQ,MAAM,OAAO;AACrC;AA8CO,SAAS,cACf,OACA,YAAiD,CAAC,GACxB;AAC1B,SAAO,OAAO;AAAA,IACb,MAAM,IAAI,CAAC,SAAS;AAAA,MACnB,KAAK;AAAA,MACL;AAAA,QACC,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,UAAU,UAAU,KAAK,IAAI,KAAK;AAAA,MACnC;AAAA,IACD,CAAC;AAAA,EACF;AACD;AAEA,SAAS,YAAY,MAA8C;AAClE,SAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO;AAAA,IACjD;AAAA,IACA,aAAa,IAAI;AAAA,IACjB,QAAQ,IAAI,UAAU,CAAC;AAAA,IACvB,UACC,OAAO,IAAI,aAAa,aACpB,IAAI,WACL,OAAO,IAAI,aAAa,WACvB,IAAI,WACJ,KAAK,UAAU,IAAI,QAAQ;AAAA,EACjC,EAAE;AACH;AAGA,SAAS,oBAAoB,OAAiD;AAC7E,SAAO,MAAM,IAAI,CAAC,OAAO;AAAA,IACxB,GAAG;AAAA,IACH,QAAQ,EAAE,kBAAkB,UAAU,WAAW,EAAE,SAAS,gBAAgB,EAAE;AAAA,IAC9E,UAAU,OAAO,EAAE,aAAa,aAAa,eAAe,EAAE;AAAA,EAC/D,EAAE;AACH;AAEA,SAAS,iBAAiB,UAA6B;AACtD,WAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC9C,UAAM,MAAM,SAAS,CAAC;AACtB,QAAI,IAAI,SAAS,SAAS;AACzB,YAAM,YAAY,IAAI,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM;AAC3D,aAAO,YAAY,UAAU,OAAO;AAAA,IACrC;AAAA,EACD;AACA,SAAO;AACR;AAQA,SAAS,mBAAmB,QAA+B;AAC1D,MAAI,OAAO,OAAO,WAAW,WAAY,QAAO,OAAO;AACvD,QAAM,aAAa,cAAc;AACjC,MAAI,CAAC,WAAW,SAAS,OAAO,OAAO,WAAW,UAAU;AAC3D,UAAM,IAAI,MAAM,iFAAiF;AAAA,EAClG;AACA,QAAM,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS,WAAW;AAC7E,SAAO,iBAAiB,WAAW,aAAa,KAAK;AACtD;AAEA,SAAS,oBAAoB,QAAiD;AAC7E,SAAO,OAAO,gBAAgB,cAAc,EAAE;AAC/C;AAEO,SAAS,YAAY,MAAc,QAA2B;AACpE,QAAM,aAAa,OAAO,SAAS,CAAC;AACpC,QAAM,eAAe,OAAO,SAAS,SAAY,oBAAoB,MAAM;AAE3E,EAAG,YAAS,MAAM,MAAM;AACvB,eAAW,MAAM,OAAO,OAAO;AAC9B,YAAM,WAAW,GAAG,QAAQ,iBAAiB,GAAG,QAAQ;AACxD,YAAM,eAAe,GAAG,SAAS;AACjC,YAAM,QAAQ,YAAY,YAAY;AACtC,YAAM,MAAM,EAAE,SAAS,iBAAiB,GAAG,QAAQ,EAAE;AAErD,YAAM,WAAW,GAAG,OAAO,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC;AAChD,YAAM,aAAa,SAAS,IAAI,CAAC,MAAM,EAAE,SAAS;AAClD,YAAM,mBAAmB,OAAO,OAAO,CAAC,GAAG,GAAG,SAAS,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;AAErF,MAAG;AAAA,QACF;AAAA,QACA;AAAA,UACC,QAAQ;AAAA,YACP,UAAU,GAAG;AAAA,YACb,OAAO,oBAAoB,KAAK;AAAA,UACjC;AAAA,UACA;AAAA,QACD;AAAA,QACA,OAAO,EAAE,kBAAkB,OAAO,MAAM;AACvC,cAAI;AAGJ,gBAAM,kBACL,GAAG,mBAAmB,OAAO,mBAAmB,cAAc,EAAE;AACjE,gBAAM,mBAAmB,kBACtB,MAAM,gBAAgB,GAAG,QAAQ,IACjC,GAAG;AAEN,cAAI,cAAc;AAEjB,qBAAS,MAAM,eAAe,cAAc,kBAAkB,YAAY;AAAA,UAC3E,OAAO;AAEN,kBAAM,SAAS,mBAAmB,MAAM;AACxC,kBAAM,eAAe,cAAc,EAAE;AACrC,kBAAM,eAAe,GAAG,gBAAgB,OAAO,gBAAgB;AAC/D,qBAAS,MAAM,OAAO;AAAA,cACrB,UAAU;AAAA,cACV;AAAA,cACA,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;AAAA,YACxC,CAAC;AAAA,UACF;AAEA,UAAG,cAAW,MAAM;AACpB,qBAAW,aAAa,YAAY;AACnC,kBAAM,UAAU,EAAE,SAAS,QAAQ,kBAAkB,UAAU,CAAC,EAAE,CAAC;AAAA,UACpE;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AACF;;;AItMA,IAAAA,MAAoB;AACpB,wBAIO;;;ACLP,IAAAC,mBAAuC;AAQhC,SAAS,wBAAwB,aAAmC,OAAe;AACzF,QAAM,WAAW,IAAI,uBAAuB,WAAW;AACvD,QAAM,QAAQ,SAAS,QAAQ,KAAK;AAEpC,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,UAAM,mBAAmB,kBAAkB;AAC3C,QAAI,CAAC,kBAAkB;AACtB,aAAO,EAAE,KAAK,kBAAkB,OAAO,MAAM,SAAS,2CAA2C;AAAA,IAClG;AAGA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,gBAAgB,CAAC,GAAG,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,aAAa,0BAAS;AAEhF,QAAI,CAAC,eAAe;AACnB,aAAO,EAAE,KAAK,kBAAkB,OAAO,OAAO,SAAS,oCAAoC;AAAA,IAC5F;AAEA,UAAM,eAAe,OAAO,cAAc,YAAY,WAAW,cAAc,UAAU,KAAK,UAAU,cAAc,OAAO;AAG7H,UAAM,YAAY,MAAM,MAAM,OAAO;AAAA,MACpC;AAAA,QACC,MAAM;AAAA,QACN,SAAS;AAAA,MACV;AAAA,MACA;AAAA,QACC,MAAM;AAAA,QACN,SAAS;AAAA,MACV;AAAA,IACD,CAAC;AAED,UAAM,oBAAoB,OAAO,UAAU,YAAY,WAAW,UAAU,UAAU,IAAI,KAAK,EAAE,YAAY;AAE7G,UAAM,UAAU,qBAAqB,iBAAiB,YAAY;AAElE,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,UACN,uCAAuC,gBAAgB,KACvD,aAAa,gBAAgB,mBAAmB,gBAAgB;AAAA,IACpE;AAAA,EACD;AACD;;;AC1DA,IAAAC,mBAAuC;AAMhC,SAAS,iCAAiC;AAChD,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,UAAM,cAAwB,kBAAkB,oBAAoB,CAAC;AACrE,UAAM,iBAA2B,kBAAkB,0BAA0B,CAAC;AAE9E,QAAI,YAAY,WAAW,KAAK,eAAe,WAAW,GAAG;AAC5D,aAAO,EAAE,KAAK,oBAAoB,OAAO,MAAM,SAAS,4CAA4C;AAAA,IACrG;AAGA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,gBAAgB,CAAC,GAAG,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,aAAa,0BAAS;AAEhF,QAAI,CAAC,eAAe;AACnB,aAAO,EAAE,KAAK,oBAAoB,OAAO,OAAO,SAAS,oCAAoC;AAAA,IAC9F;AAEA,UAAM,gBAAgB,OAAO,cAAc,YAAY,WAAW,cAAc,UAAU,KAAK,UAAU,cAAc,OAAO,GAAG,YAAY;AAE7I,UAAM,WAAqB,CAAC;AAE5B,eAAW,YAAY,aAAa;AACnC,UAAI,CAAC,aAAa,SAAS,SAAS,YAAY,CAAC,GAAG;AACnD,iBAAS,KAAK,2BAA2B,QAAQ,GAAG;AAAA,MACrD;AAAA,IACD;AAEA,eAAW,aAAa,gBAAgB;AACvC,UAAI,aAAa,SAAS,UAAU,YAAY,CAAC,GAAG;AACnD,iBAAS,KAAK,6BAA6B,SAAS,GAAG;AAAA,MACxD;AAAA,IACD;AAEA,UAAM,SAAS,SAAS,WAAW;AAEnC,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SAAS,kCAAkC,SAAS,KAAK,IAAI;AAAA,IACvE;AAAA,EACD;AACD;;;ACrDA,IAAAC,mBAAuC;AAMhC,SAAS,6BAA6B;AAC5C,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AAEL,QAAI,kBAAkB,iBAAiB,KAAK,kBAAkB,sBAAsB,MAAM;AACzF,aAAO,EAAE,KAAK,iBAAiB,OAAO,MAAM,SAAS,+CAA+C;AAAA,IACrG;AAEA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AAErD,UAAM,YAAY,SAChB,OAAO,CAAC,MAAM,aAAa,0BAAS,EACpC,QAAQ,CAAC,MAAO,EAAgB,cAAc,CAAC,CAAC;AAElD,UAAM,SAAS,UAAU,WAAW;AAEpC,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SACN,qCACA,cAAc,UAAU,MAAM,kBAAkB,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,IAC7F;AAAA,EACD;AACD;;;AHPA,SAAS,oBAAoB,WAAgB,KAA0B;AACtE,SAAO,OAAO,EAAE,SAAS,iBAAiB,MAAM;AAC/C,QAAI,CAAC,kBAAkB,qBAAqB;AAC3C,aAAO,EAAE,KAAK,OAAO,MAAM,SAAS,6CAA6C;AAAA,IAClF;AACA,WAAO,UAAU,EAAE,SAAS,kBAAkB,iBAAiB,oBAAoB,CAAC;AAAA,EACrF;AACD;AAEA,SAAS,gBAAgB,SAAiB,WAAgD;AACzF,QAAM,aAAwC,CAAC;AAC/C,MAAI,QAAQ;AAEZ,aAAW,KAAK,EAAE,MAAM,QAAQ,SAAS,QAAQ,CAAC;AAElD,aAAW,QAAQ,WAAW;AAC7B,UAAM,KAAK,KAAK,EAAE,KAAK;AACvB,eAAW,KAAK;AAAA,MACf,MAAM;AAAA,MACN,SAAS;AAAA,MACT,YAAY,CAAC,EAAE,UAAU,EAAE,MAAM,WAAW,KAAK,GAAG,IAAI,MAAM,WAAW,CAAC;AAAA,IAC3E,CAAC;AACD,eAAW,KAAK,EAAE,MAAM,QAAQ,SAAS,OAAO,cAAc,GAAG,CAAC;AAAA,EACnE;AAEA,aAAW,KAAK,EAAE,MAAM,aAAa,SAAS,MAAM,CAAC;AAErD,SAAO;AACR;AAQO,SAAS,YAAY,OAA8B;AACzD,SAAO,CAAC,SAAS;AAAA,IAChB,WAAc;AAAA,MACb;AAAA,YACC,kDAA+B,EAAE,qBAAqB,YAAY,mBAAmB,SAAS,CAAC;AAAA,QAC/F;AAAA,MACD;AAAA,IACD;AAAA,IACA,kBAAkB,EAAE,qBAAqB,gBAAgB,IAAI,SAAS,KAAK,EAAE;AAAA,EAC9E;AACD;AAOO,SAAS,WAAwB;AACvC,SAAO,MAAM;AACZ,UAAM,SAAS,cAAc;AAC7B,UAAM,QAAQ,OAAO;AACrB,WAAO;AAAA,MACN,WAAc;AAAA,QACb;AAAA,cACC,8CAA2B,EAAE,QAAQ,8CAA4B,MAAM,CAAC;AAAA,UACxE;AAAA,QACD;AAAA,MACD;AAAA,MACA,kBAAkB,CAAC;AAAA,IACpB;AAAA,EACD;AACD;AAGO,SAAS,UAAuB;AACtC,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,2BAA2B,CAAC;AAAA,IACxD,kBAAkB,EAAE,mBAAmB,KAAK;AAAA,EAC7C;AACD;AAOO,SAAS,mBAAmB,MAA2B;AAC7D,SAAO,MAAM;AACZ,UAAM,SAAS,cAAc;AAC7B,UAAM,QAAQ,OAAO;AACrB,WAAO;AAAA,MACN,WAAc,kBAAc,wBAAwB,OAAO,aAAa,KAAK,CAAC;AAAA,MAC9E,kBAAkB,EAAE,kBAAkB,KAAK;AAAA,IAC5C;AAAA,EACD;AACD;AAGO,SAAS,SAAS,SAAgC;AACxD,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,+BAA+B,CAAC;AAAA,IAC5D,kBAAkB,EAAE,kBAAkB,QAAQ;AAAA,EAC/C;AACD;AAGO,SAAS,YAAY,SAAgC;AAC3D,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,+BAA+B,CAAC;AAAA,IAC5D,kBAAkB,EAAE,wBAAwB,QAAQ;AAAA,EACrD;AACD;","names":["ls","import_messages","import_messages","import_messages"]}
+{"version":3,"sources":["../../src/eval/index.ts","../../src/eval/config.ts","../../src/eval/suite.ts","../../src/eval/target.ts","../../src/runtime/langchain/model-resolver.ts","../../src/runtime/langchain/utils.ts","../../src/eval/expectations.ts","../../src/eval/evaluators/language.ts","../../src/eval/evaluators/response-content.ts","../../src/eval/evaluators/no-tool-calls.ts","../../src/eval/evaluators/any-tool-called.ts"],"sourcesContent":["// ── Configuration ───────────────────────────────────────────────────\nexport { configureEvals } from './config';\nexport type { EvalConfig, CreateTargetFn } from './config';\n\n// ── Suite API ────────────────────────────────────────────────────────\nexport { defineSuite, runEvals, human, ai, toolResult, fromToolSpecs } from './suite';\nexport type { SuiteConfig, TestCase, ToolDef } from './suite';\n\n// ── Expectations ─────────────────────────────────────────────────────\nexport { toolsCalled, llmJudge, noTools, anyToolCalled, respondsInLanguage, contains, notContains } from './expectations';\nexport type { Expectation } from './expectations';\n","import type { Agent, Message, ToolDefinition } from '../core/agent.interface';\nimport type { LangchainModelConfig } from '../runtime/langchain/model-resolver';\n\n/** Factory that creates a fresh Agent per test case. Receives extra suite-level tools as ToolDefinition[]. */\nexport type CreateTargetFn = (extraTools: ToolDefinition[]) => Agent | Promise<Agent>;\n\nexport interface EvalConfig {\n\t/** Required for model-based target and LLM evaluators (respondsInLanguage, llmJudge). */\n\tmodelConfig: LangchainModelConfig;\n\t/** Required for model-based target. Also used as fallback for evaluatorModel. */\n\tmodel?: string;\n\t/** Model for evaluators needing LLM calls (language detection, LLM-as-judge). */\n\tevaluatorModel: string;\n\t/** LangSmith experiment (dataset) name. All suites share this single experiment for easy comparison across runs. */\n\texperimentName: string;\n\t/** System prompt for model-based target. Ignored when createTarget is used. Can be overridden per-suite or per-case. */\n\tsystemPrompt?: string;\n\t/** Factory that creates a fresh Agent per test case. When set, this is the default target. */\n\tcreateTarget?: CreateTargetFn;\n\t/** Transforms test case messages before sending to target. Simulates production preprocessing (e.g., message enrichment). */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n}\n\nlet _config: EvalConfig | null = null;\n\nexport function configureEvals(config: EvalConfig): void {\n\t_config = config;\n}\n\nexport function getEvalConfig(): EvalConfig {\n\tif (!_config) {\n\t\tthrow new Error('Evals not configured. Call configureEvals() in your vitest setupFiles.');\n\t}\n\treturn _config;\n}\n\n","import * as ls from 'langsmith/vitest';\nimport { AIMessage, BaseMessage } from '@langchain/core/messages';\nimport { createEvalTarget, runAgentTarget, type MockToolDef } from './target';\nimport { type Expectation } from './expectations';\nimport { getEvalConfig, type CreateTargetFn } from './config';\nimport {\n\ttype Message,\n\ttype HumanMessage,\n\ttype AiMessage,\n\ttype ToolMessage,\n\ttype ToolSpec,\n} from '../core/agent.interface';\n\n// ── Message builders ─────────────────────────────────────────────────\n\nexport function human(content: string): HumanMessage {\n\treturn { role: 'human', content: [{ type: 'text', text: content }] };\n}\n\nexport function ai(content: string, toolCalls?: string[]): AiMessage {\n\treturn { role: 'ai', content, ...(toolCalls ? { toolCalls: toolCalls.map((name) => ({ name })) } : {}) };\n}\n\nexport function toolResult(name: string, output: string): ToolMessage {\n\treturn { role: 'tool', name, output };\n}\n\nexport interface ToolDef {\n\tdescription: string;\n\t/** A plain key→description record, or a ZodObject passed through from a ToolSpec. */\n\tschema?: Record<string, string> | import('zod').ZodObject<any>;\n\t/** Auto-stringified if not a string or function. */\n\tresponse: unknown | ((input: Record<string, unknown>, callCount: number) => string);\n}\n\nexport interface TestCase {\n\t/** Test name. Defaults to the last human message content if omitted. */\n\tname?: string;\n\tmessages: Message[];\n\tsystemPrompt?: string;\n\t/** Override suite-level tools for this case. */\n\ttools?: Record<string, ToolDef>;\n\t/** Transforms messages before sending to target. Overrides suite-level and global hooks. */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n\texpect: Expectation[];\n}\n\ntype TargetFn = (inputs: {\n\tsystemPrompt?: string;\n\tmessages: Message[];\n\ttools: MockToolDef[];\n}) => Promise<{ messages: BaseMessage[] }>;\n\nexport interface SuiteConfig {\n\t/** Custom target function, or model string override. Auto-created from global config if omitted. */\n\ttarget?: TargetFn | string;\n\t/** Factory that creates a fresh Agent per test case. Overrides global createTarget. */\n\tcreateTarget?: CreateTargetFn;\n\t/** System prompt for all cases in this suite. Overrides the global prompt; can be overridden per-case. */\n\tsystemPrompt?: string;\n\ttools?: Record<string, ToolDef>;\n\t/** Transforms messages before sending to target. Overrides global hook; can be overridden per-case. */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n\tcases: TestCase[];\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\n/**\n * Converts a `ToolSpec[]` (from a real tool provider) into the\n * `Record<string, ToolDef>` that `defineSuite` expects.\n *\n * `responses` maps tool names to canned mock responses. Tools without an\n * entry in `responses` default to `''`.\n */\nexport function fromToolSpecs(\n\tspecs: ToolSpec[],\n\tresponses: Record<string, ToolDef['response']> = {},\n): Record<string, ToolDef> {\n\treturn Object.fromEntries(\n\t\tspecs.map((spec) => [\n\t\t\tspec.name,\n\t\t\t{\n\t\t\t\tdescription: spec.description,\n\t\t\t\tschema: spec.inputSchema,\n\t\t\t\tresponse: responses[spec.name] ?? '',\n\t\t\t} satisfies ToolDef,\n\t\t]),\n\t);\n}\n\nfunction toMockTools(defs: Record<string, ToolDef>): MockToolDef[] {\n\treturn Object.entries(defs).map(([name, def]) => ({\n\t\tname,\n\t\tdescription: def.description,\n\t\tschema: def.schema ?? {},\n\t\tresponse:\n\t\t\ttypeof def.response === 'function'\n\t\t\t\t? (def.response as MockToolDef['response'])\n\t\t\t\t: typeof def.response === 'string'\n\t\t\t\t\t? def.response\n\t\t\t\t\t: JSON.stringify(def.response),\n\t}));\n}\n\n/** Strip function responses and ZodObjects so the object is JSON-serialisable for langsmith hashing. */\nfunction toSerializableTools(tools: MockToolDef[]): Record<string, unknown>[] {\n\treturn tools.map((t) => ({\n\t\t...t,\n\t\tschema: t.schema instanceof Object && 'shape' in t.schema ? '<ZodObject>' : t.schema,\n\t\tresponse: typeof t.response === 'function' ? '<function>' : t.response,\n\t}));\n}\n\nfunction lastHumanContent(messages: Message[]): string {\n\tfor (let i = messages.length - 1; i >= 0; i--) {\n\t\tconst msg = messages[i];\n\t\tif (msg.role === 'human') {\n\t\t\tconst textBlock = msg.content.find((c) => c.type === 'text');\n\t\t\treturn textBlock ? textBlock.text : '';\n\t\t}\n\t}\n\treturn '';\n}\n\nfunction resolveModelTarget(config: SuiteConfig): TargetFn {\n\tif (typeof config.target === 'function') return config.target;\n\tconst evalConfig = getEvalConfig();\n\tif (!evalConfig.model && typeof config.target !== 'string') {\n\t\tthrow new Error('model is required for model-based target. Add it to your configureEvals() call.');\n\t}\n\tconst model = typeof config.target === 'string' ? config.target : evalConfig.model!;\n\treturn createEvalTarget(evalConfig.modelConfig, model);\n}\n\nfunction resolveCreateTarget(config: SuiteConfig): CreateTargetFn | undefined {\n\treturn config.createTarget ?? getEvalConfig().createTarget;\n}\n\n// ── Suite registry ───────────────────────────────────────────────────\n\ninterface RegisteredSuite {\n\tname: string;\n\tconfig: SuiteConfig;\n}\n\nconst _suites: RegisteredSuite[] = [];\n\n/**\n * Registers an eval suite. Does not create tests on its own — call\n * `runEvals()` after all suites are registered to emit a single\n * LangSmith experiment containing every test case.\n */\nexport function defineSuite(name: string, config: SuiteConfig): void {\n\t_suites.push({ name, config });\n}\n\n/**\n * Emits all registered suites under a single `ls.describe` block so\n * every test case lands in one LangSmith experiment / dataset.\n *\n * Call this once, after importing all suite files.\n *\n * Individual suites are grouped with native `describe` blocks for\n * readability; test names are prefixed with the suite name\n * (e.g. \"discovery > should use search tool\").\n */\nexport function runEvals(): void {\n\tconst evalConfig = getEvalConfig();\n\n\tls.describe(evalConfig.experimentName, () => {\n\t\tfor (const { name: suiteName, config } of _suites) {\n\t\t\tconst suiteTools = config.tools ?? {};\n\t\t\tconst createTarget = config.target ? undefined : resolveCreateTarget(config);\n\n\t\t\tconst categoryLabel = suiteName.charAt(0).toUpperCase() + suiteName.slice(1);\n\t\t\tconst model = typeof config.target === 'string' ? config.target : (evalConfig.model ?? 'agent');\n\n\t\t\tfor (const tc of config.cases) {\n\t\t\t\tconst testName = tc.name ?? lastHumanContent(tc.messages);\n\t\t\t\tconst caseToolDefs = tc.tools ?? suiteTools;\n\t\t\t\tconst tools = toMockTools(caseToolDefs);\n\t\t\t\tconst ctx = { message: lastHumanContent(tc.messages) };\n\n\t\t\t\tconst resolved = tc.expect.map((exp) => exp(ctx));\n\t\t\t\tconst evaluators = resolved.map((r) => r.evaluator);\n\t\t\t\tconst referenceOutputs = Object.assign({}, ...resolved.map((r) => r.referenceOutputs));\n\n\t\t\t\tconst fullTestName = `[${categoryLabel}] > ${testName}`;\n\n\t\t\t\tls.test(\n\t\t\t\t\tfullTestName,\n\t\t\t\t\t{\n\t\t\t\t\t\tinputs: {\n\t\t\t\t\t\t\tname: fullTestName,\n\t\t\t\t\t\t\tcategory: categoryLabel,\n\t\t\t\t\t\t\tmodel,\n\t\t\t\t\t\t\ttools: tools.map((t) => t.name).join(' | ') || 'none',\n\t\t\t\t\t\t\tmessages: tc.messages,\n\t\t\t\t\t\t},\n\t\t\t\t\t\treferenceOutputs,\n\t\t\t\t\t},\n\t\t\t\t\tasync ({ referenceOutputs: refOut }) => {\n\t\t\t\t\t\tlet output: { messages: BaseMessage[] };\n\n\t\t\t\t\t\t// Resolution order: case > suite > global > identity\n\t\t\t\t\t\tconst prepareMessages =\n\t\t\t\t\t\t\ttc.prepareMessages ?? config.prepareMessages ?? getEvalConfig().prepareMessages;\n\t\t\t\t\t\tconst preparedMessages = prepareMessages ? await prepareMessages(tc.messages) : tc.messages;\n\n\t\t\t\t\t\tif (createTarget) {\n\t\t\t\t\t\t\toutput = await runAgentTarget(createTarget, preparedMessages, caseToolDefs);\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tconst target = resolveModelTarget(config);\n\t\t\t\t\t\t\tconst globalPrompt = getEvalConfig().systemPrompt;\n\t\t\t\t\t\t\tconst systemPrompt = tc.systemPrompt ?? config.systemPrompt ?? globalPrompt;\n\t\t\t\t\t\t\toutput = await target({\n\t\t\t\t\t\t\t\tmessages: preparedMessages,\n\t\t\t\t\t\t\t\ttools,\n\t\t\t\t\t\t\t\t...(systemPrompt ? { systemPrompt } : {}),\n\t\t\t\t\t\t\t});\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tconst calledTools = output.messages\n\t\t\t\t\t\t\t.filter((m) => m instanceof AIMessage)\n\t\t\t\t\t\t\t.flatMap((m) => (m as AIMessage).tool_calls ?? [])\n\t\t\t\t\t\t\t.map((tc) => tc.name);\n\n\t\t\t\t\t\tls.logOutputs({\n\t\t\t\t\t\t\ttools_called: calledTools.length > 0 ? calledTools.join(' | ') : 'none',\n\t\t\t\t\t\t});\n\t\t\t\t\t\tfor (const evaluator of evaluators) {\n\t\t\t\t\t\t\tawait evaluator({ outputs: output, referenceOutputs: refOut ?? {} });\n\t\t\t\t\t\t}\n\t\t\t\t\t},\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\t});\n}\n","import { BaseChatModel } from '@langchain/core/language_models/chat_models';\nimport { tool } from '@langchain/core/tools';\nimport { AIMessage, BaseMessage, SystemMessage, ToolMessage } from '@langchain/core/messages';\nimport { z } from 'zod';\nimport { LangchainModelResolver, type LangchainModelConfig } from '../runtime/langchain/model-resolver';\nimport { type CreateTargetFn, getEvalConfig } from './config';\nimport type { AgentResult, Message as AgentMessage, ToolCallContentBlock, ToolDefinition } from '../core/agent.interface';\nimport { convertToLangchainMessages } from '../runtime/langchain/utils';\nimport type { ToolDef } from './suite';\n\nexport interface MockToolDef {\n\tname: string;\n\tdescription: string;\n\tschema: z.ZodObject<any> | Record<string, unknown>;\n\t/**\n\t * Canned response the mock tool returns.\n\t * Can be a static string, or a function that receives input and returns a response.\n\t * If a function is provided, it receives the full invocation count as a second arg\n\t * to support scenarios like \"first call fails, second call succeeds\".\n\t */\n\tresponse: string | ((input: Record<string, unknown>, callCount: number) => string);\n}\n\nexport interface EvalTargetInput {\n\tsystemPrompt?: string;\n\tmessages: AgentMessage[];\n\ttools: MockToolDef[];\n}\n\nconst MAX_AGENT_LOOPS = 10;\n\n/**\n * Creates a LangSmith-compatible target function that runs an agentic loop\n * with mock tools and returns the full message trajectory.\n */\nexport function createEvalTarget(modelConfig?: LangchainModelConfig, modelString?: string) {\n\treturn async (inputs: EvalTargetInput): Promise<{ messages: BaseMessage[] }> => {\n\t\tconst config = modelConfig && modelString ? { modelConfig, model: modelString } : getEvalConfig();\n\t\tif (!config.model) {\n\t\t\tthrow new Error('model is required for model-based target. Add it to your configureEvals() call.');\n\t\t}\n\t\tconst resolver = new LangchainModelResolver(config.modelConfig);\n\t\tconst model = resolver.resolve(config.model) as BaseChatModel;\n\n\t\t// Track invocation counts per tool for stateful mock responses\n\t\tconst toolCallCounts: Record<string, number> = {};\n\n\t\t// Create langchain tools from mock definitions\n\t\tconst langchainTools = inputs.tools.map((mockTool) => {\n\t\t\ttoolCallCounts[mockTool.name] = 0;\n\n\t\t\treturn tool(\n\t\t\t\tasync (toolInput: Record<string, unknown>) => {\n\t\t\t\t\ttoolCallCounts[mockTool.name]++;\n\t\t\t\t\tif (typeof mockTool.response === 'function') {\n\t\t\t\t\t\treturn mockTool.response(toolInput, toolCallCounts[mockTool.name]);\n\t\t\t\t\t}\n\t\t\t\t\treturn mockTool.response;\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tname: mockTool.name,\n\t\t\t\t\tdescription: mockTool.description,\n\t\t\t\t\tschema:\n\t\t\t\t\t\tmockTool.schema instanceof z.ZodObject\n\t\t\t\t\t\t\t? mockTool.schema\n\t\t\t\t\t\t\t: z.object(\n\t\t\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\t\t\tObject.entries(mockTool.schema).map(([key, val]) => {\n\t\t\t\t\t\t\t\t\t\t\tif (typeof val === 'string') return [key, z.string().describe(val)];\n\t\t\t\t\t\t\t\t\t\t\tif (typeof val === 'number') return [key, z.number().describe(String(val))];\n\t\t\t\t\t\t\t\t\t\t\treturn [key, z.any()];\n\t\t\t\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t\t\t\t),\n\t\t\t\t\t\t\t\t),\n\t\t\t\t},\n\t\t\t);\n\t\t});\n\n\t\tconst boundModel = langchainTools.length > 0 ? model.bindTools!(langchainTools) : model;\n\n\t\tconst messages: BaseMessage[] = [];\n\n\t\tif (inputs.systemPrompt) {\n\t\t\tmessages.push(new SystemMessage(inputs.systemPrompt));\n\t\t}\n\n\t\t// Convert and push all messages (history + final human)\n\t\tmessages.push(...convertToLangchainMessages(inputs.messages));\n\n\t\t// Agentic loop: keep calling model until it stops making tool calls\n\t\tlet loopCount = 0;\n\t\twhile (loopCount < MAX_AGENT_LOOPS) {\n\t\t\tloopCount++;\n\n\t\t\tconst response = await boundModel.invoke(messages);\n\t\t\tmessages.push(response as BaseMessage);\n\n\t\t\tconst aiMessage = response as AIMessage;\n\t\t\tif (!aiMessage.tool_calls || aiMessage.tool_calls.length === 0) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\t// Execute tool calls and add results\n\t\t\tfor (const tc of aiMessage.tool_calls) {\n\t\t\t\tconst mockTool = langchainTools.find((t) => t.name === tc.name);\n\t\t\t\tif (mockTool) {\n\t\t\t\t\tconst result = await mockTool.invoke(tc.args);\n\t\t\t\t\tmessages.push(\n\t\t\t\t\t\tnew ToolMessage({\n\t\t\t\t\t\t\tcontent: typeof result === 'string' ? result : JSON.stringify(result),\n\t\t\t\t\t\t\ttool_call_id: tc.id!,\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t} else {\n\t\t\t\t\tmessages.push(\n\t\t\t\t\t\tnew ToolMessage({\n\t\t\t\t\t\t\tcontent: `Tool \"${tc.name}\" not found`,\n\t\t\t\t\t\t\ttool_call_id: tc.id!,\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn { messages };\n\t};\n}\n\n// ── Agent-based target ──────────────────────────────────────────────\n\n/**\n * Converts an `AgentResult` (from `Agent.run()`) into LangChain `BaseMessage[]`\n * so existing evaluators (trajectory match, no-tool-calls, response-content, language) work unchanged.\n *\n * Consecutive `tool_call` content blocks are grouped into a single `AIMessage` with `tool_calls`,\n * followed by one `ToolMessage` per call.\n */\nexport function agentResultToMessages(inputMessages: AgentMessage[], result: AgentResult): BaseMessage[] {\n\t// Include input messages for trajectory context\n\tconst messages: BaseMessage[] = convertToLangchainMessages(inputMessages);\n\n\t// Group content blocks into BaseMessages\n\tlet pendingToolCalls: { id: string; name: string; args: Record<string, unknown>; output: string }[] = [];\n\n\tfor (const block of result.content) {\n\t\tif (block.type === 'tool_call') {\n\t\t\tconst tc = block as ToolCallContentBlock;\n\t\t\tpendingToolCalls.push({\n\t\t\t\tid: tc.toolCallId,\n\t\t\t\tname: tc.name,\n\t\t\t\targs: tc.input ? JSON.parse(tc.input) : {},\n\t\t\t\toutput: tc.output,\n\t\t\t});\n\t\t} else if (block.type === 'text') {\n\t\t\t// Flush any pending tool calls before the text block\n\t\t\tif (pendingToolCalls.length > 0) {\n\t\t\t\tmessages.push(\n\t\t\t\t\tnew AIMessage({\n\t\t\t\t\t\tcontent: '',\n\t\t\t\t\t\ttool_calls: pendingToolCalls.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })),\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t\tfor (const tc of pendingToolCalls) {\n\t\t\t\t\tmessages.push(new ToolMessage({ content: tc.output, tool_call_id: tc.id, name: tc.name }));\n\t\t\t\t}\n\t\t\t\tpendingToolCalls = [];\n\t\t\t}\n\t\t\tmessages.push(new AIMessage(block.output));\n\t\t}\n\t}\n\n\t// Flush remaining tool calls (agent ended mid-tool-use, unlikely but safe)\n\tif (pendingToolCalls.length > 0) {\n\t\tmessages.push(\n\t\t\tnew AIMessage({\n\t\t\t\tcontent: '',\n\t\t\t\ttool_calls: pendingToolCalls.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })),\n\t\t\t}),\n\t\t);\n\t\tfor (const tc of pendingToolCalls) {\n\t\t\tmessages.push(new ToolMessage({ content: tc.output, tool_call_id: tc.id, name: tc.name }));\n\t\t}\n\t}\n\n\treturn messages;\n}\n\n/**\n * Converts eval `Record<string, ToolDef>` into `ToolDefinition[]` with mock `exec` functions,\n * suitable for passing to an `AgentFactory.createAgent()` call.\n */\nexport function toolDefsToDefinitions(defs: Record<string, ToolDef>): ToolDefinition[] {\n\tconst callCounts: Record<string, number> = {};\n\n\treturn Object.entries(defs).map(([name, def]) => {\n\t\tcallCounts[name] = 0;\n\n\t\treturn {\n\t\t\tname,\n\t\t\ttoolKit: 'eval-mock',\n\t\t\tdescription: def.description,\n\t\t\tinputSchema:\n\t\t\t\tdef.schema instanceof z.ZodObject\n\t\t\t\t\t? def.schema\n\t\t\t\t\t: z.object(\n\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\tObject.entries(def.schema ?? {}).map(([key, val]) => {\n\t\t\t\t\t\t\t\t\tif (typeof val === 'string') return [key, z.string().describe(val)];\n\t\t\t\t\t\t\t\t\treturn [key, z.any()];\n\t\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t),\n\t\t\texec: async (input: Record<string, unknown>) => {\n\t\t\t\tcallCounts[name]++;\n\t\t\t\tif (typeof def.response === 'function') {\n\t\t\t\t\treturn (def.response as (input: Record<string, unknown>, callCount: number) => string)(\n\t\t\t\t\t\tinput,\n\t\t\t\t\t\tcallCounts[name],\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\treturn typeof def.response === 'string' ? def.response : JSON.stringify(def.response);\n\t\t\t},\n\t\t} satisfies ToolDefinition;\n\t});\n}\n\n/**\n * Runs a real `Agent` as the eval target. Creates a fresh agent per invocation via the factory,\n * sends human messages, and converts the `AgentResult` to `{ messages: BaseMessage[] }`.\n */\nexport async function runAgentTarget(\n\tcreateTarget: CreateTargetFn,\n\tevalMessages: AgentMessage[],\n\textraToolDefs: Record<string, ToolDef>,\n): Promise<{ messages: BaseMessage[] }> {\n\tconst extraTools = Object.keys(extraToolDefs).length > 0 ? toolDefsToDefinitions(extraToolDefs) : [];\n\tconst agent = await createTarget(extraTools);\n\n\tconst result = await agent.run({\n\t\tthreadId: `eval_${Date.now()}_${Math.random().toString(36).slice(2)}`,\n\t\tmessages: evalMessages,\n\t});\n\n\treturn { messages: agentResultToMessages(evalMessages, result) };\n}\n","import { BaseLanguageModel } from '@langchain/core/language_models/base';\nimport { ChatAnthropic } from '@langchain/anthropic';\nimport { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';\nimport { ReasoningEffort } from 'openai/resources';\n\nexport type LangchainOpenAIConfig = {\n\tapiKey: string;\n};\n\nexport type AzureModelProvider = 'openai' | 'anthropic';\n\nexport type LangchainAzureResourceConfig = {\n\tapiKey: string;\n\tmodels: {\n\t\tmodel: string;\n\t\tprovider: AzureModelProvider;\n\t\tendpoint: string;\n\t\tapiVersion: string;\n\t\tdeploymentName: string;\n\t}[];\n};\n\nexport type ResourceName = string;\n\nexport type LangchainModelConfig = {\n\topenai?: Record<string, LangchainOpenAIConfig>;\n\tazure?: Record<ResourceName, LangchainAzureResourceConfig>;\n};\n\nexport class LangchainModelResolver {\n\tconstructor(private config: LangchainModelConfig) {}\n\n\tresolve(modelString: string, tags?: string[], reasoningEffort?: ReasoningEffort): BaseLanguageModel {\n\t\tconst parts = modelString.split(':');\n\n\t\tif (parts.length === 1) {\n\t\t\tconst fullModelString = this.resolveFullModelString(modelString);\n\t\t\treturn this.resolve(fullModelString, tags, reasoningEffort);\n\t\t}\n\n\t\tif (parts.length === 2) {\n\t\t\tconst [provider, modelName] = parts;\n\t\t\treturn this.resolveByProvider(provider, modelName, modelName, tags, reasoningEffort);\n\t\t}\n\n\t\tif (parts.length === 3) {\n\t\t\tconst [provider, configName, modelName] = parts;\n\t\t\treturn this.resolveByProvider(provider, configName, modelName, tags, reasoningEffort);\n\t\t}\n\n\t\tthrow new Error(\n\t\t\t'Model string must follow format \"modelName\", \"provider:modelName\", or \"provider:configName:modelName\"',\n\t\t);\n\t}\n\n\tprivate resolveFullModelString(modelName: string): string {\n\t\tfor (const [provider, resources] of Object.entries(this.config)) {\n\t\t\tif (provider === 'openai') {\n\t\t\t\tif (modelName in (resources as Record<string, unknown>)) {\n\t\t\t\t\treturn `openai:${modelName}`;\n\t\t\t\t}\n\t\t\t} else if (provider === 'azure') {\n\t\t\t\tfor (const [resource, config] of Object.entries(\n\t\t\t\t\tresources as Record<string, { models: { model: string }[] }>,\n\t\t\t\t)) {\n\t\t\t\t\tif (config.models.some((m) => m.model === modelName)) {\n\t\t\t\t\t\treturn `azure:${resource}:${modelName}`;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tthrow new Error(`Model \"${modelName}\" not found in model config`);\n\t}\n\n\tprivate resolveByProvider(\n\t\tprovider: string,\n\t\tconfigName: string,\n\t\tmodelName: string,\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): BaseLanguageModel {\n\t\tswitch (provider) {\n\t\t\tcase 'openai':\n\t\t\t\treturn this.resolveOpenAI(configName, modelName, tags, reasoningEffort);\n\t\t\tcase 'azure':\n\t\t\t\treturn this.resolveAzure(configName, modelName, tags, reasoningEffort);\n\t\t\tdefault:\n\t\t\t\tthrow new Error(`Unsupported model provider: ${provider}`);\n\t\t}\n\t}\n\n\tprivate resolveOpenAI(\n\t\tconfigName: string,\n\t\tmodelName: string,\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): ChatOpenAI {\n\t\tconst providerConfig = this.config.openai?.[configName];\n\t\tif (!providerConfig) {\n\t\t\tthrow new Error(`Configuration \"${configName}\" for provider \"openai\" is missing`);\n\t\t}\n\n\t\treturn new ChatOpenAI({\n\t\t\tapiKey: providerConfig.apiKey,\n\t\t\tmodelName: modelName,\n\t\t\ttags: tags,\n\t\t\t...(reasoningEffort && {\n\t\t\t\treasoning: {\n\t\t\t\t\teffort: reasoningEffort,\n\t\t\t\t\tsummary: 'auto',\n\t\t\t\t},\n\t\t\t\tuseResponsesApi: true,\n\t\t\t}),\n\t\t});\n\t}\n\n\tprivate resolveAzure(\n\t\tresourceName: string,\n\t\tmodelName: string,\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): BaseLanguageModel {\n\t\tconst resource = this.config.azure?.[resourceName];\n\t\tif (!resource) {\n\t\t\tthrow new Error(`Resource \"${resourceName}\" for provider \"azure\" is missing`);\n\t\t}\n\n\t\tconst modelEntry = resource.models.find((m) => m.model === modelName);\n\t\tif (!modelEntry) {\n\t\t\tthrow new Error(`Model \"${modelName}\" not found in Azure resource \"${resourceName}\"`);\n\t\t}\n\n\t\tswitch (modelEntry.provider) {\n\t\t\tcase 'anthropic':\n\t\t\t\treturn this.resolveAzureAnthropic(resource, modelEntry, tags, reasoningEffort);\n\t\t\tcase 'openai':\n\t\t\t\treturn this.resolveAzureOpenAI(resource, modelEntry, tags, reasoningEffort);\n\t\t}\n\t}\n\n\tprivate resolveAzureOpenAI(\n\t\tresource: LangchainAzureResourceConfig,\n\t\tmodelEntry: LangchainAzureResourceConfig['models'][number],\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): AzureChatOpenAI {\n\t\t/**\n\t\t * OpenAI reasoning models require the Responses API which AzureChatOpenAI\n\t\t * does not support. We rewrite the endpoint to the Azure Responses API path.\n\t\t */\n\t\tconst endpoint = reasoningEffort\n\t\t\t? `${modelEntry.endpoint.replace(/\\/$/, '')}/openai/responses?api-version=${modelEntry.apiVersion}`\n\t\t\t: modelEntry.endpoint;\n\n\t\treturn new AzureChatOpenAI({\n\t\t\tmodel: modelEntry.model,\n\t\t\tazureOpenAIApiKey: resource.apiKey,\n\t\t\tazureOpenAIEndpoint: endpoint,\n\t\t\tazureOpenAIApiDeploymentName: modelEntry.deploymentName,\n\t\t\tazureOpenAIApiVersion: modelEntry.apiVersion,\n\t\t\ttags: tags,\n\t\t\t...(reasoningEffort && {\n\t\t\t\treasoning: {\n\t\t\t\t\teffort: reasoningEffort,\n\t\t\t\t\tsummary: 'auto',\n\t\t\t\t},\n\t\t\t}),\n\t\t});\n\t}\n\n\tprivate static readonly THINKING_BUDGET: Record<string, number> = {\n\t\tminimal: 1024,\n\t\tlow: 4096,\n\t\tmedium: 10000,\n\t\thigh: 16000,\n\t\txhigh: 32000,\n\t};\n\n\tprivate resolveAzureAnthropic(\n\t\tresource: LangchainAzureResourceConfig,\n\t\tmodelEntry: LangchainAzureResourceConfig['models'][number],\n\t\ttags?: string[],\n\t\treasoningEffort?: ReasoningEffort,\n\t): ChatAnthropic {\n\t\tconst budgetTokens = reasoningEffort ? LangchainModelResolver.THINKING_BUDGET[reasoningEffort] : undefined;\n\n\t\treturn new ChatAnthropic({\n\t\t\tmodel: modelEntry.model,\n\t\t\tapiKey: resource.apiKey,\n\t\t\tclientOptions: { baseURL: modelEntry.endpoint },\n\t\t\ttags: tags,\n\t\t\t...(budgetTokens && {\n\t\t\t\tmaxTokens: budgetTokens * 2,\n\t\t\t\tthinking: {\n\t\t\t\t\ttype: 'enabled',\n\t\t\t\t\tbudget_tokens: budgetTokens,\n\t\t\t\t},\n\t\t\t}),\n\t\t});\n\t}\n}\n","import { Message } from '@core/agent.interface';\nimport { AIMessage, BaseMessage, HumanMessage, ToolMessage } from 'langchain';\n\nexport function convertToLangchainMessages(messages: Message[]): BaseMessage[] {\n\tconst result: BaseMessage[] = [];\n\tlet tcIdx = 0;\n\tlet pendingToolCallIds: string[] = [];\n\n\tfor (const msg of messages) {\n\t\tif (msg.role === 'human') {\n\t\t\tresult.push(\n\t\t\t\tnew HumanMessage({\n\t\t\t\t\tcontent: msg.content.map((c) => {\n\t\t\t\t\t\tif (c.type === 'image') {\n\t\t\t\t\t\t\treturn { type: 'image_url', image_url: { url: c.url } };\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn c;\n\t\t\t\t\t}) as any,\n\t\t\t\t}),\n\t\t\t);\n\t\t} else if (msg.role === 'ai') {\n\t\t\tif (msg.toolCalls && msg.toolCalls.length > 0) {\n\t\t\t\tpendingToolCallIds = msg.toolCalls.map(() => `tc_${++tcIdx}`);\n\t\t\t\tresult.push(\n\t\t\t\t\tnew AIMessage({\n\t\t\t\t\t\tcontent: msg.content,\n\t\t\t\t\t\ttool_calls: msg.toolCalls.map((tc, i) => ({\n\t\t\t\t\t\t\tid: pendingToolCallIds[i],\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t\targs: tc.input ? JSON.parse(tc.input) : {},\n\t\t\t\t\t\t})),\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t} else {\n\t\t\t\tresult.push(new AIMessage(msg.content));\n\t\t\t}\n\t\t} else if (msg.role === 'tool') {\n\t\t\tconst toolCallId = pendingToolCallIds.shift();\n\t\t\tif (!toolCallId)\n\t\t\t\tthrow new Error(`ToolMessage for \"${msg.name}\" without a preceding AiMessage with toolCalls`);\n\t\t\tresult.push(\n\t\t\t\tnew ToolMessage({\n\t\t\t\t\tcontent: msg.output,\n\t\t\t\t\ttool_call_id: toolCallId,\n\t\t\t\t\tname: msg.name,\n\t\t\t\t}),\n\t\t\t);\n\t\t}\n\t}\n\n\treturn result;\n}\n","import * as ls from 'langsmith/vitest';\nimport { createTrajectoryMatchEvaluator, createTrajectoryLLMAsJudge, TRAJECTORY_ACCURACY_PROMPT } from 'agentevals';\nimport { createLanguageEvaluator } from './evaluators/language';\nimport { createResponseContentEvaluator } from './evaluators/response-content';\nimport { createNoToolCallsEvaluator } from './evaluators/no-tool-calls';\nimport { createAnyToolCalledEvaluator } from './evaluators/any-tool-called';\nimport { getEvalConfig } from './config';\n\n// ── Types ────────────────────────────────────────────────────────────\n\ntype EvaluatorFn = (args: { outputs: Record<string, any>; referenceOutputs: Record<string, any> }) => Promise<any>;\n\ninterface ResolvedExpectation {\n\tevaluator: EvaluatorFn;\n\treferenceOutputs: Record<string, unknown>;\n}\n\n/** A factory that receives test context and returns an evaluator + its referenceOutputs. */\nexport type Expectation = (ctx: { message: string }) => ResolvedExpectation;\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nfunction withTrajectoryGuard(evaluator: any, key: string): EvaluatorFn {\n\treturn async ({ outputs, referenceOutputs }) => {\n\t\tif (!referenceOutputs?.referenceTrajectory) {\n\t\t\treturn { key, score: true, comment: 'No referenceTrajectory specified, skipping' };\n\t\t}\n\t\treturn evaluator({ outputs, referenceOutputs: referenceOutputs.referenceTrajectory });\n\t};\n}\n\nfunction buildTrajectory(message: string, toolNames: string[]): Record<string, unknown>[] {\n\tconst trajectory: Record<string, unknown>[] = [];\n\tlet tcIdx = 0;\n\n\ttrajectory.push({ role: 'user', content: message });\n\n\tfor (const name of toolNames) {\n\t\tconst id = `tc${++tcIdx}`;\n\t\ttrajectory.push({\n\t\t\trole: 'assistant',\n\t\t\tcontent: '',\n\t\t\ttool_calls: [{ function: { name, arguments: '{}' }, id, type: 'function' }],\n\t\t});\n\t\ttrajectory.push({ role: 'tool', content: '...', tool_call_id: id });\n\t}\n\n\ttrajectory.push({ role: 'assistant', content: '...' });\n\n\treturn trajectory;\n}\n\n// ── Expectation functions ────────────────────────────────────────────\n\n/**\n * Expect the agent to call tools in order (superset trajectory match).\n * Empty `[]` means the agent should answer directly without calling any tools.\n */\nexport function toolsCalled(tools: string[]): Expectation {\n\treturn (ctx) => ({\n\t\tevaluator: ls.wrapEvaluator(\n\t\t\twithTrajectoryGuard(\n\t\t\t\tcreateTrajectoryMatchEvaluator({ trajectoryMatchMode: 'superset', toolArgsMatchMode: 'ignore' }) as any,\n\t\t\t\t'trajectory_match',\n\t\t\t),\n\t\t),\n\t\treferenceOutputs: { referenceTrajectory: buildTrajectory(ctx.message, tools) },\n\t});\n}\n\n/**\n * Run an LLM-as-judge evaluator on the trajectory.\n * Requires `toolsCalled` in the same expect array.\n * Uses the globally configured evaluator model.\n */\nexport function llmJudge(): Expectation {\n\treturn () => {\n\t\tconst config = getEvalConfig();\n\t\tconst model = config.evaluatorModel;\n\t\treturn {\n\t\t\tevaluator: ls.wrapEvaluator(\n\t\t\t\twithTrajectoryGuard(\n\t\t\t\t\tcreateTrajectoryLLMAsJudge({ prompt: TRAJECTORY_ACCURACY_PROMPT, model }) as any,\n\t\t\t\t\t'trajectory_llm_judge',\n\t\t\t\t),\n\t\t\t),\n\t\t\treferenceOutputs: {},\n\t\t};\n\t};\n}\n\n/**\n * Assert the agent made zero tool calls.\n * Optionally allow specific tools via `except` — calls to those tools\n * are permitted (but not required), while any other tool call fails.\n */\nexport function noTools(options?: { except: string[] }): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createNoToolCallsEvaluator()),\n\t\treferenceOutputs: {\n\t\t\texpectNoToolCalls: true,\n\t\t\t...(options?.except?.length ? { exceptTools: options.except } : {}),\n\t\t},\n\t});\n}\n\n/**\n * Assert the response is in the given language (ISO 639-1 code).\n * Uses the globally configured evaluator model for language detection.\n * @param code - ISO 639-1 language code (e.g. 'en', 'tr', 'de').\n */\nexport function respondsInLanguage(code: string): Expectation {\n\treturn () => {\n\t\tconst config = getEvalConfig();\n\t\tconst model = config.evaluatorModel;\n\t\treturn {\n\t\t\tevaluator: ls.wrapEvaluator(createLanguageEvaluator(config.modelConfig, model)),\n\t\t\treferenceOutputs: { expectedLanguage: code },\n\t\t};\n\t};\n}\n\n/**\n * Assert that at least one tool call was made.\n * When `tools` is provided, at least one of those specific tools must\n * appear in the trajectory. When omitted, any tool call satisfies it.\n */\nexport function anyToolCalled(tools?: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createAnyToolCalledEvaluator()),\n\t\treferenceOutputs: {\n\t\t\texpectAnyToolCall: true,\n\t\t\t...(tools?.length ? { anyToolsExpected: tools } : {}),\n\t\t},\n\t});\n}\n\n/** Assert the response contains all given strings. */\nexport function contains(strings: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createResponseContentEvaluator()),\n\t\treferenceOutputs: { responseContains: strings },\n\t});\n}\n\n/** Assert the response does not contain any of the given strings. */\nexport function notContains(strings: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createResponseContentEvaluator()),\n\t\treferenceOutputs: { responseMustNotContain: strings },\n\t});\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\nimport { BaseChatModel } from '@langchain/core/language_models/chat_models';\nimport { LangchainModelResolver, type LangchainModelConfig } from '../../runtime/langchain/model-resolver';\n\n/**\n * Creates a custom evaluator that checks whether the agent's final response\n * is in the expected language. Uses a cheap LLM call for language detection.\n */\nexport function createLanguageEvaluator(modelConfig: LangchainModelConfig, model: string) {\n\tconst resolver = new LangchainModelResolver(modelConfig);\n\tconst judge = resolver.resolve(model) as BaseChatModel;\n\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tconst expectedLanguage = referenceOutputs?.expectedLanguage;\n\t\tif (!expectedLanguage) {\n\t\t\treturn { key: 'language_match', score: true, comment: 'No expected language specified, skipping' };\n\t\t}\n\n\t\t// Extract the last AI message text from the trajectory\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage);\n\n\t\tif (!lastAiMessage) {\n\t\t\treturn { key: 'language_match', score: false, comment: 'No AI message found in trajectory' };\n\t\t}\n\n\t\tconst responseText = typeof lastAiMessage.content === 'string' ? lastAiMessage.content : JSON.stringify(lastAiMessage.content);\n\n\t\t// Use LLM to detect the language\n\t\tconst detection = await judge.invoke([\n\t\t\t{\n\t\t\t\trole: 'system',\n\t\t\t\tcontent: 'You are a language detection tool. Respond with ONLY the ISO 639-1 language code (e.g., \"en\", \"tr\", \"de\", \"fr\") of the text provided. Nothing else.',\n\t\t\t},\n\t\t\t{\n\t\t\t\trole: 'user',\n\t\t\t\tcontent: responseText,\n\t\t\t},\n\t\t]);\n\n\t\tconst detectedLanguage = (typeof detection.content === 'string' ? detection.content : '').trim().toLowerCase();\n\n\t\tconst matches = detectedLanguage === expectedLanguage.toLowerCase();\n\n\t\treturn {\n\t\t\tkey: 'language_match',\n\t\t\tscore: matches,\n\t\t\tcomment: matches\n\t\t\t\t? `Response language matches expected: ${expectedLanguage}`\n\t\t\t\t: `Expected \"${expectedLanguage}\" but detected \"${detectedLanguage}\"`,\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that checks whether the agent's final response\n * contains expected strings and doesn't contain forbidden strings.\n */\nexport function createResponseContentEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tconst mustContain: string[] = referenceOutputs?.responseContains || [];\n\t\tconst mustNotContain: string[] = referenceOutputs?.responseMustNotContain || [];\n\n\t\tif (mustContain.length === 0 && mustNotContain.length === 0) {\n\t\t\treturn { key: 'response_content', score: true, comment: 'No content assertions specified, skipping' };\n\t\t}\n\n\t\t// Extract the last AI message text from the trajectory\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage);\n\n\t\tif (!lastAiMessage) {\n\t\t\treturn { key: 'response_content', score: false, comment: 'No AI message found in trajectory' };\n\t\t}\n\n\t\tconst responseText = (typeof lastAiMessage.content === 'string' ? lastAiMessage.content : JSON.stringify(lastAiMessage.content)).toLowerCase();\n\n\t\tconst failures: string[] = [];\n\n\t\tfor (const expected of mustContain) {\n\t\t\tif (!responseText.includes(expected.toLowerCase())) {\n\t\t\t\tfailures.push(`Missing expected text: \"${expected}\"`);\n\t\t\t}\n\t\t}\n\n\t\tfor (const forbidden of mustNotContain) {\n\t\t\tif (responseText.includes(forbidden.toLowerCase())) {\n\t\t\t\tfailures.push(`Contains forbidden text: \"${forbidden}\"`);\n\t\t\t}\n\t\t}\n\n\t\tconst passed = failures.length === 0;\n\n\t\treturn {\n\t\t\tkey: 'response_content',\n\t\t\tscore: passed,\n\t\t\tcomment: passed ? 'All content assertions passed' : failures.join('; '),\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that asserts the agent made zero tool calls.\n * When `referenceOutputs.exceptTools` is set, calls to those tools are\n * allowed (but not required) — only calls to non-excepted tools cause failure.\n */\nexport function createNoToolCallsEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\t// Only run this evaluator if the reference explicitly expects no tool calls\n\t\tif (referenceOutputs?.maxToolCalls !== 0 && referenceOutputs?.expectNoToolCalls !== true) {\n\t\t\treturn { key: 'no_tool_calls', score: true, comment: 'No tool call restriction specified, skipping' };\n\t\t}\n\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst exceptTools: string[] = referenceOutputs?.exceptTools ?? [];\n\n\t\tconst toolCalls = messages\n\t\t\t.filter((m) => m instanceof AIMessage)\n\t\t\t.flatMap((m) => (m as AIMessage).tool_calls || []);\n\n\t\tconst disallowedCalls = exceptTools.length > 0 ? toolCalls.filter((tc) => !exceptTools.includes(tc.name)) : toolCalls;\n\n\t\tconst passed = disallowedCalls.length === 0;\n\n\t\tif (exceptTools.length > 0) {\n\t\t\treturn {\n\t\t\t\tkey: 'no_tool_calls',\n\t\t\t\tscore: passed,\n\t\t\t\tcomment: passed\n\t\t\t\t\t? `No disallowed tool calls made (allowed: ${exceptTools.join(', ')})`\n\t\t\t\t\t: `Agent made ${disallowedCalls.length} disallowed tool call(s): ${disallowedCalls.map((tc) => tc.name).join(', ')}`,\n\t\t\t};\n\t\t}\n\n\t\treturn {\n\t\t\tkey: 'no_tool_calls',\n\t\t\tscore: passed,\n\t\t\tcomment: passed\n\t\t\t\t? 'No tool calls made (as expected)'\n\t\t\t\t: `Agent made ${toolCalls.length} tool call(s): ${toolCalls.map((tc) => tc.name).join(', ')}`,\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that asserts at least one tool was called.\n * When `referenceOutputs.anyToolsExpected` contains tool names, at least\n * one of those specific tools must appear. When the list is empty, any\n * tool call satisfies the expectation.\n */\nexport function createAnyToolCalledEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tif (referenceOutputs?.expectAnyToolCall !== true) {\n\t\t\treturn { key: 'any_tool_called', score: true, comment: 'No any-tool-call expectation specified, skipping' };\n\t\t}\n\n\t\tconst expectedTools: string[] = referenceOutputs?.anyToolsExpected ?? [];\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\n\t\tconst calledToolNames = messages\n\t\t\t.filter((m) => m instanceof AIMessage)\n\t\t\t.flatMap((m) => (m as AIMessage).tool_calls || [])\n\t\t\t.map((tc) => tc.name);\n\n\t\t// No specific tools requested — any tool call satisfies the expectation\n\t\tif (expectedTools.length === 0) {\n\t\t\tconst passed = calledToolNames.length > 0;\n\t\t\treturn {\n\t\t\t\tkey: 'any_tool_called',\n\t\t\t\tscore: passed,\n\t\t\t\tcomment: passed\n\t\t\t\t\t? `Agent called tool(s): ${calledToolNames.join(', ')}`\n\t\t\t\t\t: 'Agent made no tool calls (expected at least one)',\n\t\t\t};\n\t\t}\n\n\t\tconst matchedTools = expectedTools.filter((name) => calledToolNames.includes(name));\n\t\tconst passed = matchedTools.length > 0;\n\n\t\treturn {\n\t\t\tkey: 'any_tool_called',\n\t\t\tscore: passed,\n\t\t\tcomment: passed\n\t\t\t\t? `Called expected tool(s): ${matchedTools.join(', ')}`\n\t\t\t\t: `None of the expected tools were called (expected one of: ${expectedTools.join(', ')}; actual: ${calledToolNames.length > 0 ? calledToolNames.join(', ') : 'none'})`,\n\t\t};\n\t};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACuBA,IAAI,UAA6B;AAE1B,SAAS,eAAe,QAA0B;AACxD,YAAU;AACX;AAEO,SAAS,gBAA4B;AAC3C,MAAI,CAAC,SAAS;AACb,UAAM,IAAI,MAAM,wEAAwE;AAAA,EACzF;AACA,SAAO;AACR;;;AClCA,SAAoB;AACpB,IAAAA,mBAAuC;;;ACAvC,mBAAqB;AACrB,sBAAmE;AACnE,iBAAkB;;;ACFlB,uBAA8B;AAC9B,oBAA4C;AA2BrC,IAAM,yBAAN,MAAM,wBAAuB;AAAA,EACnC,YAAoB,QAA8B;AAA9B;AAAA,EAA+B;AAAA,EAEnD,QAAQ,aAAqB,MAAiB,iBAAsD;AACnG,UAAM,QAAQ,YAAY,MAAM,GAAG;AAEnC,QAAI,MAAM,WAAW,GAAG;AACvB,YAAM,kBAAkB,KAAK,uBAAuB,WAAW;AAC/D,aAAO,KAAK,QAAQ,iBAAiB,MAAM,eAAe;AAAA,IAC3D;AAEA,QAAI,MAAM,WAAW,GAAG;AACvB,YAAM,CAAC,UAAU,SAAS,IAAI;AAC9B,aAAO,KAAK,kBAAkB,UAAU,WAAW,WAAW,MAAM,eAAe;AAAA,IACpF;AAEA,QAAI,MAAM,WAAW,GAAG;AACvB,YAAM,CAAC,UAAU,YAAY,SAAS,IAAI;AAC1C,aAAO,KAAK,kBAAkB,UAAU,YAAY,WAAW,MAAM,eAAe;AAAA,IACrF;AAEA,UAAM,IAAI;AAAA,MACT;AAAA,IACD;AAAA,EACD;AAAA,EAEQ,uBAAuB,WAA2B;AACzD,eAAW,CAAC,UAAU,SAAS,KAAK,OAAO,QAAQ,KAAK,MAAM,GAAG;AAChE,UAAI,aAAa,UAAU;AAC1B,YAAI,aAAc,WAAuC;AACxD,iBAAO,UAAU,SAAS;AAAA,QAC3B;AAAA,MACD,WAAW,aAAa,SAAS;AAChC,mBAAW,CAAC,UAAU,MAAM,KAAK,OAAO;AAAA,UACvC;AAAA,QACD,GAAG;AACF,cAAI,OAAO,OAAO,KAAK,CAAC,MAAM,EAAE,UAAU,SAAS,GAAG;AACrD,mBAAO,SAAS,QAAQ,IAAI,SAAS;AAAA,UACtC;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAEA,UAAM,IAAI,MAAM,UAAU,SAAS,6BAA6B;AAAA,EACjE;AAAA,EAEQ,kBACP,UACA,YACA,WACA,MACA,iBACoB;AACpB,YAAQ,UAAU;AAAA,MACjB,KAAK;AACJ,eAAO,KAAK,cAAc,YAAY,WAAW,MAAM,eAAe;AAAA,MACvE,KAAK;AACJ,eAAO,KAAK,aAAa,YAAY,WAAW,MAAM,eAAe;AAAA,MACtE;AACC,cAAM,IAAI,MAAM,+BAA+B,QAAQ,EAAE;AAAA,IAC3D;AAAA,EACD;AAAA,EAEQ,cACP,YACA,WACA,MACA,iBACa;AACb,UAAM,iBAAiB,KAAK,OAAO,SAAS,UAAU;AACtD,QAAI,CAAC,gBAAgB;AACpB,YAAM,IAAI,MAAM,kBAAkB,UAAU,oCAAoC;AAAA,IACjF;AAEA,WAAO,IAAI,yBAAW;AAAA,MACrB,QAAQ,eAAe;AAAA,MACvB;AAAA,MACA;AAAA,MACA,GAAI,mBAAmB;AAAA,QACtB,WAAW;AAAA,UACV,QAAQ;AAAA,UACR,SAAS;AAAA,QACV;AAAA,QACA,iBAAiB;AAAA,MAClB;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EAEQ,aACP,cACA,WACA,MACA,iBACoB;AACpB,UAAM,WAAW,KAAK,OAAO,QAAQ,YAAY;AACjD,QAAI,CAAC,UAAU;AACd,YAAM,IAAI,MAAM,aAAa,YAAY,mCAAmC;AAAA,IAC7E;AAEA,UAAM,aAAa,SAAS,OAAO,KAAK,CAAC,MAAM,EAAE,UAAU,SAAS;AACpE,QAAI,CAAC,YAAY;AAChB,YAAM,IAAI,MAAM,UAAU,SAAS,kCAAkC,YAAY,GAAG;AAAA,IACrF;AAEA,YAAQ,WAAW,UAAU;AAAA,MAC5B,KAAK;AACJ,eAAO,KAAK,sBAAsB,UAAU,YAAY,MAAM,eAAe;AAAA,MAC9E,KAAK;AACJ,eAAO,KAAK,mBAAmB,UAAU,YAAY,MAAM,eAAe;AAAA,IAC5E;AAAA,EACD;AAAA,EAEQ,mBACP,UACA,YACA,MACA,iBACkB;AAKlB,UAAM,WAAW,kBACd,GAAG,WAAW,SAAS,QAAQ,OAAO,EAAE,CAAC,iCAAiC,WAAW,UAAU,KAC/F,WAAW;AAEd,WAAO,IAAI,8BAAgB;AAAA,MAC1B,OAAO,WAAW;AAAA,MAClB,mBAAmB,SAAS;AAAA,MAC5B,qBAAqB;AAAA,MACrB,8BAA8B,WAAW;AAAA,MACzC,uBAAuB,WAAW;AAAA,MAClC;AAAA,MACA,GAAI,mBAAmB;AAAA,QACtB,WAAW;AAAA,UACV,QAAQ;AAAA,UACR,SAAS;AAAA,QACV;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AAAA,EAEA,OAAwB,kBAA0C;AAAA,IACjE,SAAS;AAAA,IACT,KAAK;AAAA,IACL,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,OAAO;AAAA,EACR;AAAA,EAEQ,sBACP,UACA,YACA,MACA,iBACgB;AAChB,UAAM,eAAe,kBAAkB,wBAAuB,gBAAgB,eAAe,IAAI;AAEjG,WAAO,IAAI,+BAAc;AAAA,MACxB,OAAO,WAAW;AAAA,MAClB,QAAQ,SAAS;AAAA,MACjB,eAAe,EAAE,SAAS,WAAW,SAAS;AAAA,MAC9C;AAAA,MACA,GAAI,gBAAgB;AAAA,QACnB,WAAW,eAAe;AAAA,QAC1B,UAAU;AAAA,UACT,MAAM;AAAA,UACN,eAAe;AAAA,QAChB;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AACD;;;ACxMA,uBAAkE;AAE3D,SAAS,2BAA2B,UAAoC;AAC9E,QAAM,SAAwB,CAAC;AAC/B,MAAI,QAAQ;AACZ,MAAI,qBAA+B,CAAC;AAEpC,aAAW,OAAO,UAAU;AAC3B,QAAI,IAAI,SAAS,SAAS;AACzB,aAAO;AAAA,QACN,IAAI,8BAAa;AAAA,UAChB,SAAS,IAAI,QAAQ,IAAI,CAAC,MAAM;AAC/B,gBAAI,EAAE,SAAS,SAAS;AACvB,qBAAO,EAAE,MAAM,aAAa,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE;AAAA,YACvD;AACA,mBAAO;AAAA,UACR,CAAC;AAAA,QACF,CAAC;AAAA,MACF;AAAA,IACD,WAAW,IAAI,SAAS,MAAM;AAC7B,UAAI,IAAI,aAAa,IAAI,UAAU,SAAS,GAAG;AAC9C,6BAAqB,IAAI,UAAU,IAAI,MAAM,MAAM,EAAE,KAAK,EAAE;AAC5D,eAAO;AAAA,UACN,IAAI,2BAAU;AAAA,YACb,SAAS,IAAI;AAAA,YACb,YAAY,IAAI,UAAU,IAAI,CAAC,IAAI,OAAO;AAAA,cACzC,IAAI,mBAAmB,CAAC;AAAA,cACxB,MAAM,GAAG;AAAA,cACT,MAAM,GAAG,QAAQ,KAAK,MAAM,GAAG,KAAK,IAAI,CAAC;AAAA,YAC1C,EAAE;AAAA,UACH,CAAC;AAAA,QACF;AAAA,MACD,OAAO;AACN,eAAO,KAAK,IAAI,2BAAU,IAAI,OAAO,CAAC;AAAA,MACvC;AAAA,IACD,WAAW,IAAI,SAAS,QAAQ;AAC/B,YAAM,aAAa,mBAAmB,MAAM;AAC5C,UAAI,CAAC;AACJ,cAAM,IAAI,MAAM,oBAAoB,IAAI,IAAI,gDAAgD;AAC7F,aAAO;AAAA,QACN,IAAI,6BAAY;AAAA,UACf,SAAS,IAAI;AAAA,UACb,cAAc;AAAA,UACd,MAAM,IAAI;AAAA,QACX,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AAEA,SAAO;AACR;;;AFtBA,IAAM,kBAAkB;AAMjB,SAAS,iBAAiB,aAAoC,aAAsB;AAC1F,SAAO,OAAO,WAAkE;AAC/E,UAAM,SAAS,eAAe,cAAc,EAAE,aAAa,OAAO,YAAY,IAAI,cAAc;AAChG,QAAI,CAAC,OAAO,OAAO;AAClB,YAAM,IAAI,MAAM,iFAAiF;AAAA,IAClG;AACA,UAAM,WAAW,IAAI,uBAAuB,OAAO,WAAW;AAC9D,UAAM,QAAQ,SAAS,QAAQ,OAAO,KAAK;AAG3C,UAAM,iBAAyC,CAAC;AAGhD,UAAM,iBAAiB,OAAO,MAAM,IAAI,CAAC,aAAa;AACrD,qBAAe,SAAS,IAAI,IAAI;AAEhC,iBAAO;AAAA,QACN,OAAO,cAAuC;AAC7C,yBAAe,SAAS,IAAI;AAC5B,cAAI,OAAO,SAAS,aAAa,YAAY;AAC5C,mBAAO,SAAS,SAAS,WAAW,eAAe,SAAS,IAAI,CAAC;AAAA,UAClE;AACA,iBAAO,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,UACC,MAAM,SAAS;AAAA,UACf,aAAa,SAAS;AAAA,UACtB,QACC,SAAS,kBAAkB,aAAE,YAC1B,SAAS,SACT,aAAE;AAAA,YACF,OAAO;AAAA,cACN,OAAO,QAAQ,SAAS,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM;AACnD,oBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,aAAE,OAAO,EAAE,SAAS,GAAG,CAAC;AAClE,oBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,aAAE,OAAO,EAAE,SAAS,OAAO,GAAG,CAAC,CAAC;AAC1E,uBAAO,CAAC,KAAK,aAAE,IAAI,CAAC;AAAA,cACrB,CAAC;AAAA,YACF;AAAA,UACD;AAAA,QACJ;AAAA,MACD;AAAA,IACD,CAAC;AAED,UAAM,aAAa,eAAe,SAAS,IAAI,MAAM,UAAW,cAAc,IAAI;AAElF,UAAM,WAA0B,CAAC;AAEjC,QAAI,OAAO,cAAc;AACxB,eAAS,KAAK,IAAI,8BAAc,OAAO,YAAY,CAAC;AAAA,IACrD;AAGA,aAAS,KAAK,GAAG,2BAA2B,OAAO,QAAQ,CAAC;AAG5D,QAAI,YAAY;AAChB,WAAO,YAAY,iBAAiB;AACnC;AAEA,YAAM,WAAW,MAAM,WAAW,OAAO,QAAQ;AACjD,eAAS,KAAK,QAAuB;AAErC,YAAM,YAAY;AAClB,UAAI,CAAC,UAAU,cAAc,UAAU,WAAW,WAAW,GAAG;AAC/D;AAAA,MACD;AAGA,iBAAW,MAAM,UAAU,YAAY;AACtC,cAAM,WAAW,eAAe,KAAK,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI;AAC9D,YAAI,UAAU;AACb,gBAAM,SAAS,MAAM,SAAS,OAAO,GAAG,IAAI;AAC5C,mBAAS;AAAA,YACR,IAAI,4BAAY;AAAA,cACf,SAAS,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,cACpE,cAAc,GAAG;AAAA,cACjB,MAAM,GAAG;AAAA,YACV,CAAC;AAAA,UACF;AAAA,QACD,OAAO;AACN,mBAAS;AAAA,YACR,IAAI,4BAAY;AAAA,cACf,SAAS,SAAS,GAAG,IAAI;AAAA,cACzB,cAAc,GAAG;AAAA,cACjB,MAAM,GAAG;AAAA,YACV,CAAC;AAAA,UACF;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAEA,WAAO,EAAE,SAAS;AAAA,EACnB;AACD;AAWO,SAAS,sBAAsB,eAA+B,QAAoC;AAExG,QAAM,WAA0B,2BAA2B,aAAa;AAGxE,MAAI,mBAAkG,CAAC;AAEvG,aAAW,SAAS,OAAO,SAAS;AACnC,QAAI,MAAM,SAAS,aAAa;AAC/B,YAAM,KAAK;AACX,uBAAiB,KAAK;AAAA,QACrB,IAAI,GAAG;AAAA,QACP,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,QAAQ,KAAK,MAAM,GAAG,KAAK,IAAI,CAAC;AAAA,QACzC,QAAQ,GAAG;AAAA,MACZ,CAAC;AAAA,IACF,WAAW,MAAM,SAAS,QAAQ;AAEjC,UAAI,iBAAiB,SAAS,GAAG;AAChC,iBAAS;AAAA,UACR,IAAI,0BAAU;AAAA,YACb,SAAS;AAAA,YACT,YAAY,iBAAiB,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,MAAM,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE;AAAA,UACvF,CAAC;AAAA,QACF;AACA,mBAAW,MAAM,kBAAkB;AAClC,mBAAS,KAAK,IAAI,4BAAY,EAAE,SAAS,GAAG,QAAQ,cAAc,GAAG,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,QAC1F;AACA,2BAAmB,CAAC;AAAA,MACrB;AACA,eAAS,KAAK,IAAI,0BAAU,MAAM,MAAM,CAAC;AAAA,IAC1C;AAAA,EACD;AAGA,MAAI,iBAAiB,SAAS,GAAG;AAChC,aAAS;AAAA,MACR,IAAI,0BAAU;AAAA,QACb,SAAS;AAAA,QACT,YAAY,iBAAiB,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,MAAM,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE;AAAA,MACvF,CAAC;AAAA,IACF;AACA,eAAW,MAAM,kBAAkB;AAClC,eAAS,KAAK,IAAI,4BAAY,EAAE,SAAS,GAAG,QAAQ,cAAc,GAAG,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,IAC1F;AAAA,EACD;AAEA,SAAO;AACR;AAMO,SAAS,sBAAsB,MAAiD;AACtF,QAAM,aAAqC,CAAC;AAE5C,SAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM;AAChD,eAAW,IAAI,IAAI;AAEnB,WAAO;AAAA,MACN;AAAA,MACA,SAAS;AAAA,MACT,aAAa,IAAI;AAAA,MACjB,aACC,IAAI,kBAAkB,aAAE,YACrB,IAAI,SACJ,aAAE;AAAA,QACF,OAAO;AAAA,UACN,OAAO,QAAQ,IAAI,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM;AACpD,gBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,aAAE,OAAO,EAAE,SAAS,GAAG,CAAC;AAClE,mBAAO,CAAC,KAAK,aAAE,IAAI,CAAC;AAAA,UACrB,CAAC;AAAA,QACF;AAAA,MACD;AAAA,MACH,MAAM,OAAO,UAAmC;AAC/C,mBAAW,IAAI;AACf,YAAI,OAAO,IAAI,aAAa,YAAY;AACvC,iBAAQ,IAAI;AAAA,YACX;AAAA,YACA,WAAW,IAAI;AAAA,UAChB;AAAA,QACD;AACA,eAAO,OAAO,IAAI,aAAa,WAAW,IAAI,WAAW,KAAK,UAAU,IAAI,QAAQ;AAAA,MACrF;AAAA,IACD;AAAA,EACD,CAAC;AACF;AAMA,eAAsB,eACrB,cACA,cACA,eACuC;AACvC,QAAM,aAAa,OAAO,KAAK,aAAa,EAAE,SAAS,IAAI,sBAAsB,aAAa,IAAI,CAAC;AACnG,QAAM,QAAQ,MAAM,aAAa,UAAU;AAE3C,QAAM,SAAS,MAAM,MAAM,IAAI;AAAA,IAC9B,UAAU,QAAQ,KAAK,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AAAA,IACnE,UAAU;AAAA,EACX,CAAC;AAED,SAAO,EAAE,UAAU,sBAAsB,cAAc,MAAM,EAAE;AAChE;;;ADvOO,SAAS,MAAM,SAA+B;AACpD,SAAO,EAAE,MAAM,SAAS,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,QAAQ,CAAC,EAAE;AACpE;AAEO,SAAS,GAAG,SAAiB,WAAiC;AACpE,SAAO,EAAE,MAAM,MAAM,SAAS,GAAI,YAAY,EAAE,WAAW,UAAU,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,IAAI,CAAC,EAAG;AACxG;AAEO,SAAS,WAAW,MAAc,QAA6B;AACrE,SAAO,EAAE,MAAM,QAAQ,MAAM,OAAO;AACrC;AAkDO,SAAS,cACf,OACA,YAAiD,CAAC,GACxB;AAC1B,SAAO,OAAO;AAAA,IACb,MAAM,IAAI,CAAC,SAAS;AAAA,MACnB,KAAK;AAAA,MACL;AAAA,QACC,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,UAAU,UAAU,KAAK,IAAI,KAAK;AAAA,MACnC;AAAA,IACD,CAAC;AAAA,EACF;AACD;AAEA,SAAS,YAAY,MAA8C;AAClE,SAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO;AAAA,IACjD;AAAA,IACA,aAAa,IAAI;AAAA,IACjB,QAAQ,IAAI,UAAU,CAAC;AAAA,IACvB,UACC,OAAO,IAAI,aAAa,aACpB,IAAI,WACL,OAAO,IAAI,aAAa,WACvB,IAAI,WACJ,KAAK,UAAU,IAAI,QAAQ;AAAA,EACjC,EAAE;AACH;AAWA,SAAS,iBAAiB,UAA6B;AACtD,WAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC9C,UAAM,MAAM,SAAS,CAAC;AACtB,QAAI,IAAI,SAAS,SAAS;AACzB,YAAM,YAAY,IAAI,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM;AAC3D,aAAO,YAAY,UAAU,OAAO;AAAA,IACrC;AAAA,EACD;AACA,SAAO;AACR;AAEA,SAAS,mBAAmB,QAA+B;AAC1D,MAAI,OAAO,OAAO,WAAW,WAAY,QAAO,OAAO;AACvD,QAAM,aAAa,cAAc;AACjC,MAAI,CAAC,WAAW,SAAS,OAAO,OAAO,WAAW,UAAU;AAC3D,UAAM,IAAI,MAAM,iFAAiF;AAAA,EAClG;AACA,QAAM,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS,WAAW;AAC7E,SAAO,iBAAiB,WAAW,aAAa,KAAK;AACtD;AAEA,SAAS,oBAAoB,QAAiD;AAC7E,SAAO,OAAO,gBAAgB,cAAc,EAAE;AAC/C;AASA,IAAM,UAA6B,CAAC;AAO7B,SAAS,YAAY,MAAc,QAA2B;AACpE,UAAQ,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B;AAYO,SAAS,WAAiB;AAChC,QAAM,aAAa,cAAc;AAEjC,EAAG,YAAS,WAAW,gBAAgB,MAAM;AAC5C,eAAW,EAAE,MAAM,WAAW,OAAO,KAAK,SAAS;AAClD,YAAM,aAAa,OAAO,SAAS,CAAC;AACpC,YAAM,eAAe,OAAO,SAAS,SAAY,oBAAoB,MAAM;AAE3E,YAAM,gBAAgB,UAAU,OAAO,CAAC,EAAE,YAAY,IAAI,UAAU,MAAM,CAAC;AAC3E,YAAM,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAU,WAAW,SAAS;AAEvF,iBAAW,MAAM,OAAO,OAAO;AAC9B,cAAM,WAAW,GAAG,QAAQ,iBAAiB,GAAG,QAAQ;AACxD,cAAM,eAAe,GAAG,SAAS;AACjC,cAAM,QAAQ,YAAY,YAAY;AACtC,cAAM,MAAM,EAAE,SAAS,iBAAiB,GAAG,QAAQ,EAAE;AAErD,cAAM,WAAW,GAAG,OAAO,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC;AAChD,cAAM,aAAa,SAAS,IAAI,CAAC,MAAM,EAAE,SAAS;AAClD,cAAM,mBAAmB,OAAO,OAAO,CAAC,GAAG,GAAG,SAAS,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;AAErF,cAAM,eAAe,IAAI,aAAa,OAAO,QAAQ;AAErD,QAAG;AAAA,UACF;AAAA,UACA;AAAA,YACC,QAAQ;AAAA,cACP,MAAM;AAAA,cACN,UAAU;AAAA,cACV;AAAA,cACA,OAAO,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,KAAK,KAAK;AAAA,cAC/C,UAAU,GAAG;AAAA,YACd;AAAA,YACA;AAAA,UACD;AAAA,UACA,OAAO,EAAE,kBAAkB,OAAO,MAAM;AACvC,gBAAI;AAGJ,kBAAM,kBACL,GAAG,mBAAmB,OAAO,mBAAmB,cAAc,EAAE;AACjE,kBAAM,mBAAmB,kBAAkB,MAAM,gBAAgB,GAAG,QAAQ,IAAI,GAAG;AAEnF,gBAAI,cAAc;AACjB,uBAAS,MAAM,eAAe,cAAc,kBAAkB,YAAY;AAAA,YAC3E,OAAO;AACN,oBAAM,SAAS,mBAAmB,MAAM;AACxC,oBAAM,eAAe,cAAc,EAAE;AACrC,oBAAM,eAAe,GAAG,gBAAgB,OAAO,gBAAgB;AAC/D,uBAAS,MAAM,OAAO;AAAA,gBACrB,UAAU;AAAA,gBACV;AAAA,gBACA,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;AAAA,cACxC,CAAC;AAAA,YACF;AAEA,kBAAM,cAAc,OAAO,SACzB,OAAO,CAAC,MAAM,aAAa,0BAAS,EACpC,QAAQ,CAAC,MAAO,EAAgB,cAAc,CAAC,CAAC,EAChD,IAAI,CAACC,QAAOA,IAAG,IAAI;AAErB,YAAG,cAAW;AAAA,cACb,cAAc,YAAY,SAAS,IAAI,YAAY,KAAK,KAAK,IAAI;AAAA,YAClE,CAAC;AACD,uBAAW,aAAa,YAAY;AACnC,oBAAM,UAAU,EAAE,SAAS,QAAQ,kBAAkB,UAAU,CAAC,EAAE,CAAC;AAAA,YACpE;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AACF;;;AI/OA,IAAAC,MAAoB;AACpB,wBAAuG;;;ACDvG,IAAAC,mBAAuC;AAQhC,SAAS,wBAAwB,aAAmC,OAAe;AACzF,QAAM,WAAW,IAAI,uBAAuB,WAAW;AACvD,QAAM,QAAQ,SAAS,QAAQ,KAAK;AAEpC,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,UAAM,mBAAmB,kBAAkB;AAC3C,QAAI,CAAC,kBAAkB;AACtB,aAAO,EAAE,KAAK,kBAAkB,OAAO,MAAM,SAAS,2CAA2C;AAAA,IAClG;AAGA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,gBAAgB,CAAC,GAAG,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,aAAa,0BAAS;AAEhF,QAAI,CAAC,eAAe;AACnB,aAAO,EAAE,KAAK,kBAAkB,OAAO,OAAO,SAAS,oCAAoC;AAAA,IAC5F;AAEA,UAAM,eAAe,OAAO,cAAc,YAAY,WAAW,cAAc,UAAU,KAAK,UAAU,cAAc,OAAO;AAG7H,UAAM,YAAY,MAAM,MAAM,OAAO;AAAA,MACpC;AAAA,QACC,MAAM;AAAA,QACN,SAAS;AAAA,MACV;AAAA,MACA;AAAA,QACC,MAAM;AAAA,QACN,SAAS;AAAA,MACV;AAAA,IACD,CAAC;AAED,UAAM,oBAAoB,OAAO,UAAU,YAAY,WAAW,UAAU,UAAU,IAAI,KAAK,EAAE,YAAY;AAE7G,UAAM,UAAU,qBAAqB,iBAAiB,YAAY;AAElE,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,UACN,uCAAuC,gBAAgB,KACvD,aAAa,gBAAgB,mBAAmB,gBAAgB;AAAA,IACpE;AAAA,EACD;AACD;;;AC1DA,IAAAC,mBAAuC;AAMhC,SAAS,iCAAiC;AAChD,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,UAAM,cAAwB,kBAAkB,oBAAoB,CAAC;AACrE,UAAM,iBAA2B,kBAAkB,0BAA0B,CAAC;AAE9E,QAAI,YAAY,WAAW,KAAK,eAAe,WAAW,GAAG;AAC5D,aAAO,EAAE,KAAK,oBAAoB,OAAO,MAAM,SAAS,4CAA4C;AAAA,IACrG;AAGA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,gBAAgB,CAAC,GAAG,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,aAAa,0BAAS;AAEhF,QAAI,CAAC,eAAe;AACnB,aAAO,EAAE,KAAK,oBAAoB,OAAO,OAAO,SAAS,oCAAoC;AAAA,IAC9F;AAEA,UAAM,gBAAgB,OAAO,cAAc,YAAY,WAAW,cAAc,UAAU,KAAK,UAAU,cAAc,OAAO,GAAG,YAAY;AAE7I,UAAM,WAAqB,CAAC;AAE5B,eAAW,YAAY,aAAa;AACnC,UAAI,CAAC,aAAa,SAAS,SAAS,YAAY,CAAC,GAAG;AACnD,iBAAS,KAAK,2BAA2B,QAAQ,GAAG;AAAA,MACrD;AAAA,IACD;AAEA,eAAW,aAAa,gBAAgB;AACvC,UAAI,aAAa,SAAS,UAAU,YAAY,CAAC,GAAG;AACnD,iBAAS,KAAK,6BAA6B,SAAS,GAAG;AAAA,MACxD;AAAA,IACD;AAEA,UAAM,SAAS,SAAS,WAAW;AAEnC,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SAAS,kCAAkC,SAAS,KAAK,IAAI;AAAA,IACvE;AAAA,EACD;AACD;;;ACrDA,IAAAC,mBAAuC;AAOhC,SAAS,6BAA6B;AAC5C,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AAEL,QAAI,kBAAkB,iBAAiB,KAAK,kBAAkB,sBAAsB,MAAM;AACzF,aAAO,EAAE,KAAK,iBAAiB,OAAO,MAAM,SAAS,+CAA+C;AAAA,IACrG;AAEA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,cAAwB,kBAAkB,eAAe,CAAC;AAEhE,UAAM,YAAY,SAChB,OAAO,CAAC,MAAM,aAAa,0BAAS,EACpC,QAAQ,CAAC,MAAO,EAAgB,cAAc,CAAC,CAAC;AAElD,UAAM,kBAAkB,YAAY,SAAS,IAAI,UAAU,OAAO,CAAC,OAAO,CAAC,YAAY,SAAS,GAAG,IAAI,CAAC,IAAI;AAE5G,UAAM,SAAS,gBAAgB,WAAW;AAE1C,QAAI,YAAY,SAAS,GAAG;AAC3B,aAAO;AAAA,QACN,KAAK;AAAA,QACL,OAAO;AAAA,QACP,SAAS,SACN,2CAA2C,YAAY,KAAK,IAAI,CAAC,MACjE,cAAc,gBAAgB,MAAM,6BAA6B,gBAAgB,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,MACpH;AAAA,IACD;AAEA,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SACN,qCACA,cAAc,UAAU,MAAM,kBAAkB,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,IAC7F;AAAA,EACD;AACD;;;ACjDA,IAAAC,mBAAuC;AAQhC,SAAS,+BAA+B;AAC9C,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,QAAI,kBAAkB,sBAAsB,MAAM;AACjD,aAAO,EAAE,KAAK,mBAAmB,OAAO,MAAM,SAAS,mDAAmD;AAAA,IAC3G;AAEA,UAAM,gBAA0B,kBAAkB,oBAAoB,CAAC;AACvE,UAAM,WAA0B,QAAQ,YAAY,CAAC;AAErD,UAAM,kBAAkB,SACtB,OAAO,CAAC,MAAM,aAAa,0BAAS,EACpC,QAAQ,CAAC,MAAO,EAAgB,cAAc,CAAC,CAAC,EAChD,IAAI,CAAC,OAAO,GAAG,IAAI;AAGrB,QAAI,cAAc,WAAW,GAAG;AAC/B,YAAMC,UAAS,gBAAgB,SAAS;AACxC,aAAO;AAAA,QACN,KAAK;AAAA,QACL,OAAOA;AAAA,QACP,SAASA,UACN,yBAAyB,gBAAgB,KAAK,IAAI,CAAC,KACnD;AAAA,MACJ;AAAA,IACD;AAEA,UAAM,eAAe,cAAc,OAAO,CAAC,SAAS,gBAAgB,SAAS,IAAI,CAAC;AAClF,UAAM,SAAS,aAAa,SAAS;AAErC,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SACN,4BAA4B,aAAa,KAAK,IAAI,CAAC,KACnD,4DAA4D,cAAc,KAAK,IAAI,CAAC,aAAa,gBAAgB,SAAS,IAAI,gBAAgB,KAAK,IAAI,IAAI,MAAM;AAAA,IACrK;AAAA,EACD;AACD;;;AJ7BA,SAAS,oBAAoB,WAAgB,KAA0B;AACtE,SAAO,OAAO,EAAE,SAAS,iBAAiB,MAAM;AAC/C,QAAI,CAAC,kBAAkB,qBAAqB;AAC3C,aAAO,EAAE,KAAK,OAAO,MAAM,SAAS,6CAA6C;AAAA,IAClF;AACA,WAAO,UAAU,EAAE,SAAS,kBAAkB,iBAAiB,oBAAoB,CAAC;AAAA,EACrF;AACD;AAEA,SAAS,gBAAgB,SAAiB,WAAgD;AACzF,QAAM,aAAwC,CAAC;AAC/C,MAAI,QAAQ;AAEZ,aAAW,KAAK,EAAE,MAAM,QAAQ,SAAS,QAAQ,CAAC;AAElD,aAAW,QAAQ,WAAW;AAC7B,UAAM,KAAK,KAAK,EAAE,KAAK;AACvB,eAAW,KAAK;AAAA,MACf,MAAM;AAAA,MACN,SAAS;AAAA,MACT,YAAY,CAAC,EAAE,UAAU,EAAE,MAAM,WAAW,KAAK,GAAG,IAAI,MAAM,WAAW,CAAC;AAAA,IAC3E,CAAC;AACD,eAAW,KAAK,EAAE,MAAM,QAAQ,SAAS,OAAO,cAAc,GAAG,CAAC;AAAA,EACnE;AAEA,aAAW,KAAK,EAAE,MAAM,aAAa,SAAS,MAAM,CAAC;AAErD,SAAO;AACR;AAQO,SAAS,YAAY,OAA8B;AACzD,SAAO,CAAC,SAAS;AAAA,IAChB,WAAc;AAAA,MACb;AAAA,YACC,kDAA+B,EAAE,qBAAqB,YAAY,mBAAmB,SAAS,CAAC;AAAA,QAC/F;AAAA,MACD;AAAA,IACD;AAAA,IACA,kBAAkB,EAAE,qBAAqB,gBAAgB,IAAI,SAAS,KAAK,EAAE;AAAA,EAC9E;AACD;AAOO,SAAS,WAAwB;AACvC,SAAO,MAAM;AACZ,UAAM,SAAS,cAAc;AAC7B,UAAM,QAAQ,OAAO;AACrB,WAAO;AAAA,MACN,WAAc;AAAA,QACb;AAAA,cACC,8CAA2B,EAAE,QAAQ,8CAA4B,MAAM,CAAC;AAAA,UACxE;AAAA,QACD;AAAA,MACD;AAAA,MACA,kBAAkB,CAAC;AAAA,IACpB;AAAA,EACD;AACD;AAOO,SAAS,QAAQ,SAA6C;AACpE,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,2BAA2B,CAAC;AAAA,IACxD,kBAAkB;AAAA,MACjB,mBAAmB;AAAA,MACnB,GAAI,SAAS,QAAQ,SAAS,EAAE,aAAa,QAAQ,OAAO,IAAI,CAAC;AAAA,IAClE;AAAA,EACD;AACD;AAOO,SAAS,mBAAmB,MAA2B;AAC7D,SAAO,MAAM;AACZ,UAAM,SAAS,cAAc;AAC7B,UAAM,QAAQ,OAAO;AACrB,WAAO;AAAA,MACN,WAAc,kBAAc,wBAAwB,OAAO,aAAa,KAAK,CAAC;AAAA,MAC9E,kBAAkB,EAAE,kBAAkB,KAAK;AAAA,IAC5C;AAAA,EACD;AACD;AAOO,SAAS,cAAc,OAA+B;AAC5D,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,6BAA6B,CAAC;AAAA,IAC1D,kBAAkB;AAAA,MACjB,mBAAmB;AAAA,MACnB,GAAI,OAAO,SAAS,EAAE,kBAAkB,MAAM,IAAI,CAAC;AAAA,IACpD;AAAA,EACD;AACD;AAGO,SAAS,SAAS,SAAgC;AACxD,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,+BAA+B,CAAC;AAAA,IAC5D,kBAAkB,EAAE,kBAAkB,QAAQ;AAAA,EAC/C;AACD;AAGO,SAAS,YAAY,SAAgC;AAC3D,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,+BAA+B,CAAC;AAAA,IAC5D,kBAAkB,EAAE,wBAAwB,QAAQ;AAAA,EACrD;AACD;","names":["import_messages","tc","ls","import_messages","import_messages","import_messages","import_messages","passed"]}

package/dist/eval/index.mjs

@@ -17,6 +17,7 @@ function getEvalConfig() {
 
 // src/eval/suite.ts
 import * as ls from "langsmith/vitest";
+import { AIMessage as AIMessage2 } from "@langchain/core/messages";
 
 // src/eval/target.ts
 import { tool } from "@langchain/core/tools";
@@ -207,13 +208,6 @@ function toMockTools(defs) {
     response: typeof def.response === "function" ? def.response : typeof def.response === "string" ? def.response : JSON.stringify(def.response)
   }));
 }
-function toSerializableTools(tools) {
-  return tools.map((t) => ({
-    ...t,
-    schema: t.schema instanceof Object && "shape" in t.schema ? "<ZodObject>" : t.schema,
-    response: typeof t.response === "function" ? "<function>" : t.response
-  }));
-}
 function lastHumanContent(messages) {
   for (let i = messages.length - 1; i >= 0; i--) {
     const msg = messages[i];
@@ -236,63 +230,75 @@ function resolveModelTarget(config) {
 function resolveCreateTarget(config) {
   return config.createTarget ?? getEvalConfig().createTarget;
 }
+var _suites = [];
 function defineSuite(name, config) {
-  const suiteTools = config.tools ?? {};
-  const createTarget = config.target ? void 0 : resolveCreateTarget(config);
-  ls.describe(name, () => {
-    for (const tc of config.cases) {
-      const testName = tc.name ?? lastHumanContent(tc.messages);
-      const caseToolDefs = tc.tools ?? suiteTools;
-      const tools = toMockTools(caseToolDefs);
-      const ctx = { message: lastHumanContent(tc.messages) };
-      const resolved = tc.expect.map((exp) => exp(ctx));
-      const evaluators = resolved.map((r) => r.evaluator);
-      const referenceOutputs = Object.assign({}, ...resolved.map((r) => r.referenceOutputs));
-      ls.test(
-        testName,
-        {
-          inputs: {
-            messages: tc.messages,
-            tools: toSerializableTools(tools)
+  _suites.push({ name, config });
+}
+function runEvals() {
+  const evalConfig = getEvalConfig();
+  ls.describe(evalConfig.experimentName, () => {
+    for (const { name: suiteName, config } of _suites) {
+      const suiteTools = config.tools ?? {};
+      const createTarget = config.target ? void 0 : resolveCreateTarget(config);
+      const categoryLabel = suiteName.charAt(0).toUpperCase() + suiteName.slice(1);
+      const model = typeof config.target === "string" ? config.target : evalConfig.model ?? "agent";
+      for (const tc of config.cases) {
+        const testName = tc.name ?? lastHumanContent(tc.messages);
+        const caseToolDefs = tc.tools ?? suiteTools;
+        const tools = toMockTools(caseToolDefs);
+        const ctx = { message: lastHumanContent(tc.messages) };
+        const resolved = tc.expect.map((exp) => exp(ctx));
+        const evaluators = resolved.map((r) => r.evaluator);
+        const referenceOutputs = Object.assign({}, ...resolved.map((r) => r.referenceOutputs));
+        const fullTestName = `[${categoryLabel}] > ${testName}`;
+        ls.test(
+          fullTestName,
+          {
+            inputs: {
+              name: fullTestName,
+              category: categoryLabel,
+              model,
+              tools: tools.map((t) => t.name).join(" | ") || "none",
+              messages: tc.messages
+            },
+            referenceOutputs
           },
-          referenceOutputs
-        },
-        async ({ referenceOutputs: refOut }) => {
-          let output;
-          const prepareMessages = tc.prepareMessages ?? config.prepareMessages ?? getEvalConfig().prepareMessages;
-          const preparedMessages = prepareMessages ? await prepareMessages(tc.messages) : tc.messages;
-          if (createTarget) {
-            output = await runAgentTarget(createTarget, preparedMessages, caseToolDefs);
-          } else {
-            const target = resolveModelTarget(config);
-            const globalPrompt = getEvalConfig().systemPrompt;
-            const systemPrompt = tc.systemPrompt ?? config.systemPrompt ?? globalPrompt;
-            output = await target({
-              messages: preparedMessages,
-              tools,
-              ...systemPrompt ? { systemPrompt } : {}
+          async ({ referenceOutputs: refOut }) => {
+            let output;
+            const prepareMessages = tc.prepareMessages ?? config.prepareMessages ?? getEvalConfig().prepareMessages;
+            const preparedMessages = prepareMessages ? await prepareMessages(tc.messages) : tc.messages;
+            if (createTarget) {
+              output = await runAgentTarget(createTarget, preparedMessages, caseToolDefs);
+            } else {
+              const target = resolveModelTarget(config);
+              const globalPrompt = getEvalConfig().systemPrompt;
+              const systemPrompt = tc.systemPrompt ?? config.systemPrompt ?? globalPrompt;
+              output = await target({
+                messages: preparedMessages,
+                tools,
+                ...systemPrompt ? { systemPrompt } : {}
+              });
+            }
+            const calledTools = output.messages.filter((m) => m instanceof AIMessage2).flatMap((m) => m.tool_calls ?? []).map((tc2) => tc2.name);
+            ls.logOutputs({
+              tools_called: calledTools.length > 0 ? calledTools.join(" | ") : "none"
             });
+            for (const evaluator of evaluators) {
+              await evaluator({ outputs: output, referenceOutputs: refOut ?? {} });
+            }
           }
-          ls.logOutputs(output);
-          for (const evaluator of evaluators) {
-            await evaluator({ outputs: output, referenceOutputs: refOut ?? {} });
-          }
-        }
-      );
+        );
+      }
     }
   });
 }
 
 // src/eval/expectations.ts
 import * as ls2 from "langsmith/vitest";
-import {
-  createTrajectoryMatchEvaluator,
-  createTrajectoryLLMAsJudge,
-  TRAJECTORY_ACCURACY_PROMPT
-} from "agentevals";
+import { createTrajectoryMatchEvaluator, createTrajectoryLLMAsJudge, TRAJECTORY_ACCURACY_PROMPT } from "agentevals";
 
 // src/eval/evaluators/language.ts
-import { AIMessage as AIMessage2 } from "@langchain/core/messages";
+import { AIMessage as AIMessage3 } from "@langchain/core/messages";
 function createLanguageEvaluator(modelConfig, model) {
   const resolver = new LangchainModelResolver(modelConfig);
   const judge = resolver.resolve(model);
@@ -305,7 +311,7 @@ function createLanguageEvaluator(modelConfig, model) {
       return { key: "language_match", score: true, comment: "No expected language specified, skipping" };
     }
     const messages = outputs.messages || [];
-    const lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage2);
+    const lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage3);
     if (!lastAiMessage) {
       return { key: "language_match", score: false, comment: "No AI message found in trajectory" };
     }
@@ -331,7 +337,7 @@ function createLanguageEvaluator(modelConfig, model) {
 }
 
 // src/eval/evaluators/response-content.ts
-import { AIMessage as AIMessage3 } from "@langchain/core/messages";
+import { AIMessage as AIMessage4 } from "@langchain/core/messages";
 function createResponseContentEvaluator() {
   return async ({
     outputs,
@@ -343,7 +349,7 @@ function createResponseContentEvaluator() {
       return { key: "response_content", score: true, comment: "No content assertions specified, skipping" };
     }
     const messages = outputs.messages || [];
-    const lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage3);
+    const lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage4);
     if (!lastAiMessage) {
       return { key: "response_content", score: false, comment: "No AI message found in trajectory" };
     }
@@ -369,7 +375,7 @@ function createResponseContentEvaluator() {
 }
 
 // src/eval/evaluators/no-tool-calls.ts
-import { AIMessage as AIMessage4 } from "@langchain/core/messages";
+import { AIMessage as AIMessage5 } from "@langchain/core/messages";
 function createNoToolCallsEvaluator() {
   return async ({
     outputs,
@@ -379,8 +385,17 @@ function createNoToolCallsEvaluator() {
       return { key: "no_tool_calls", score: true, comment: "No tool call restriction specified, skipping" };
     }
     const messages = outputs.messages || [];
-    const toolCalls = messages.filter((m) => m instanceof AIMessage4).flatMap((m) => m.tool_calls || []);
-    const passed = toolCalls.length === 0;
+    const exceptTools = referenceOutputs?.exceptTools ?? [];
+    const toolCalls = messages.filter((m) => m instanceof AIMessage5).flatMap((m) => m.tool_calls || []);
+    const disallowedCalls = exceptTools.length > 0 ? toolCalls.filter((tc) => !exceptTools.includes(tc.name)) : toolCalls;
+    const passed = disallowedCalls.length === 0;
+    if (exceptTools.length > 0) {
+      return {
+        key: "no_tool_calls",
+        score: passed,
+        comment: passed ? `No disallowed tool calls made (allowed: ${exceptTools.join(", ")})` : `Agent made ${disallowedCalls.length} disallowed tool call(s): ${disallowedCalls.map((tc) => tc.name).join(", ")}`
+      };
+    }
     return {
       key: "no_tool_calls",
       score: passed,
@@ -389,6 +404,37 @@ function createNoToolCallsEvaluator() {
   };
 }
 
+// src/eval/evaluators/any-tool-called.ts
+import { AIMessage as AIMessage6 } from "@langchain/core/messages";
+function createAnyToolCalledEvaluator() {
+  return async ({
+    outputs,
+    referenceOutputs
+  }) => {
+    if (referenceOutputs?.expectAnyToolCall !== true) {
+      return { key: "any_tool_called", score: true, comment: "No any-tool-call expectation specified, skipping" };
+    }
+    const expectedTools = referenceOutputs?.anyToolsExpected ?? [];
+    const messages = outputs.messages || [];
+    const calledToolNames = messages.filter((m) => m instanceof AIMessage6).flatMap((m) => m.tool_calls || []).map((tc) => tc.name);
+    if (expectedTools.length === 0) {
+      const passed2 = calledToolNames.length > 0;
+      return {
+        key: "any_tool_called",
+        score: passed2,
+        comment: passed2 ? `Agent called tool(s): ${calledToolNames.join(", ")}` : "Agent made no tool calls (expected at least one)"
+      };
+    }
+    const matchedTools = expectedTools.filter((name) => calledToolNames.includes(name));
+    const passed = matchedTools.length > 0;
+    return {
+      key: "any_tool_called",
+      score: passed,
+      comment: passed ? `Called expected tool(s): ${matchedTools.join(", ")}` : `None of the expected tools were called (expected one of: ${expectedTools.join(", ")}; actual: ${calledToolNames.length > 0 ? calledToolNames.join(", ") : "none"})`
+    };
+  };
+}
+
 // src/eval/expectations.ts
 function withTrajectoryGuard(evaluator, key) {
   return async ({ outputs, referenceOutputs }) => {
@@ -440,10 +486,13 @@ function llmJudge() {
     };
   };
 }
-function noTools() {
+function noTools(options) {
   return () => ({
     evaluator: ls2.wrapEvaluator(createNoToolCallsEvaluator()),
-    referenceOutputs: { expectNoToolCalls: true }
+    referenceOutputs: {
+      expectNoToolCalls: true,
+      ...options?.except?.length ? { exceptTools: options.except } : {}
+    }
   });
 }
 function respondsInLanguage(code) {
@@ -456,6 +505,15 @@ function respondsInLanguage(code) {
     };
   };
 }
+function anyToolCalled(tools) {
+  return () => ({
+    evaluator: ls2.wrapEvaluator(createAnyToolCalledEvaluator()),
+    referenceOutputs: {
+      expectAnyToolCall: true,
+      ...tools?.length ? { anyToolsExpected: tools } : {}
+    }
+  });
+}
 function contains(strings) {
   return () => ({
     evaluator: ls2.wrapEvaluator(createResponseContentEvaluator()),
@@ -470,6 +528,7 @@ function notContains(strings) {
 }
 export {
   ai,
+  anyToolCalled,
   configureEvals,
   contains,
   defineSuite,
@@ -479,6 +538,7 @@ export {
   noTools,
   notContains,
   respondsInLanguage,
+  runEvals,
   toolResult,
   toolsCalled
 };