@dvina/agents 0.9.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. package/dist/eval/index.d.mts +34 -4
  2. package/dist/eval/index.d.ts +34 -4
  3. package/dist/eval/index.js +121 -55
  4. package/dist/eval/index.js.map +1 -1
  5. package/dist/eval/index.mjs +120 -60
  6. package/dist/eval/index.mjs.map +1 -1
  7. package/dist/index.d.mts +15 -12
  8. package/dist/index.d.ts +15 -12
  9. package/dist/index.js +42 -17
  10. package/dist/index.js.map +1 -1
  11. package/dist/index.mjs +42 -17
  12. package/dist/index.mjs.map +1 -1
  13. package/dist/{model-resolver-lIpXv0Pc.d.mts → model-resolver-DjKRXKtu.d.mts} +7 -1
  14. package/dist/{model-resolver-lIpXv0Pc.d.ts → model-resolver-DjKRXKtu.d.ts} +7 -1
  15. package/package.json +1 -1
package/dist/eval/index.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/eval/config.ts","../../src/eval/suite.ts","../../src/eval/target.ts","../../src/eval/expectations.ts","../../src/eval/evaluators/language.ts","../../src/eval/evaluators/response-content.ts","../../src/eval/evaluators/no-tool-calls.ts"],"sourcesContent":["import type { Agent, Message, ToolDefinition } from '../core/agent.interface';\nimport type { LangchainModelConfig } from '../runtime/langchain/model-resolver';\n\n/** Factory that creates a fresh Agent per test case. Receives extra suite-level tools as ToolDefinition[]. */\nexport type CreateTargetFn = (extraTools: ToolDefinition[]) => Agent | Promise<Agent>;\n\nexport interface EvalConfig {\n\t/** Required for model-based target and LLM evaluators (respondsInLanguage, llmJudge). */\n\tmodelConfig: LangchainModelConfig;\n\t/** Required for model-based target. Also used as fallback for evaluatorModel. */\n\tmodel?: string;\n\t/** Model for evaluators needing LLM calls (language detection, LLM-as-judge). */\n\tevaluatorModel: string;\n\t/** System prompt for model-based target. Ignored when createTarget is used. Can be overridden per-suite or per-case. */\n\tsystemPrompt?: string;\n\t/** Factory that creates a fresh Agent per test case. When set, this is the default target. */\n\tcreateTarget?: CreateTargetFn;\n\t/** Transforms test case messages before sending to target. Simulates production preprocessing (e.g., message enrichment). */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n}\n\nlet _config: EvalConfig | null = null;\n\nexport function configureEvals(config: EvalConfig): void {\n\t_config = config;\n}\n\nexport function getEvalConfig(): EvalConfig {\n\tif (!_config) {\n\t\tthrow new Error('Evals not configured. Call configureEvals() in your vitest setupFiles.');\n\t}\n\treturn _config;\n}\n\n","import * as ls from 'langsmith/vitest';\nimport { BaseMessage } from '@langchain/core/messages';\nimport { createEvalTarget, runAgentTarget, type MockToolDef } from './target';\nimport { type Expectation } from './expectations';\nimport { getEvalConfig, type CreateTargetFn } from './config';\nimport {\n\ttype Message,\n\ttype HumanMessage,\n\ttype AiMessage,\n\ttype ToolMessage,\n\ttype ToolSpec,\n} from '../core/agent.interface';\n\n// ── Message builders ─────────────────────────────────────────────────\n\nexport function human(content: string): HumanMessage {\n\treturn { role: 'human', content: [{ type: 'text', text: content }] };\n}\n\nexport function ai(content: string, toolCalls?: string[]): AiMessage {\n\treturn { role: 'ai', content, ...(toolCalls ? { toolCalls: toolCalls.map((name) => ({ name })) } : {}) };\n}\n\nexport function toolResult(name: string, output: string): ToolMessage {\n\treturn { role: 'tool', name, output };\n}\n\nexport interface ToolDef {\n\tdescription: string;\n\t/** A plain key→description record, or a ZodObject passed through from a ToolSpec. */\n\tschema?: Record<string, string> | import('zod').ZodObject<any>;\n\t/** Auto-stringified if not a string or function. */\n\tresponse: unknown | ((input: Record<string, unknown>, callCount: number) => string);\n}\n\nexport interface TestCase {\n\t/** Test name. Defaults to the last human message content if omitted. */\n\tname?: string;\n\tmessages: Message[];\n\tsystemPrompt?: string;\n\t/** Override suite-level tools for this case. */\n\ttools?: Record<string, ToolDef>;\n\t/** Transforms messages before sending to target. Overrides suite-level and global hooks. */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n\texpect: Expectation[];\n}\n\ntype TargetFn = (inputs: { systemPrompt?: string; messages: Message[]; tools: MockToolDef[] }) => Promise<{ messages: BaseMessage[] }>;\n\nexport interface SuiteConfig {\n\t/** Custom target function, or model string override. Auto-created from global config if omitted. */\n\ttarget?: TargetFn | string;\n\t/** Factory that creates a fresh Agent per test case. Overrides global createTarget. */\n\tcreateTarget?: CreateTargetFn;\n\t/** System prompt for all cases in this suite. Overrides the global prompt; can be overridden per-case. */\n\tsystemPrompt?: string;\n\ttools?: Record<string, ToolDef>;\n\t/** Transforms messages before sending to target. Overrides global hook; can be overridden per-case. */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n\tcases: TestCase[];\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\n/**\n * Converts a `ToolSpec[]` (from a real tool provider) into the\n * `Record<string, ToolDef>` that `defineSuite` expects.\n *\n * `responses` maps tool names to canned mock responses. Tools without an\n * entry in `responses` default to `''`.\n */\nexport function fromToolSpecs(\n\tspecs: ToolSpec[],\n\tresponses: Record<string, ToolDef['response']> = {},\n): Record<string, ToolDef> {\n\treturn Object.fromEntries(\n\t\tspecs.map((spec) => [\n\t\t\tspec.name,\n\t\t\t{\n\t\t\t\tdescription: spec.description,\n\t\t\t\tschema: spec.inputSchema,\n\t\t\t\tresponse: responses[spec.name] ?? '',\n\t\t\t} satisfies ToolDef,\n\t\t]),\n\t);\n}\n\nfunction toMockTools(defs: Record<string, ToolDef>): MockToolDef[] {\n\treturn Object.entries(defs).map(([name, def]) => ({\n\t\tname,\n\t\tdescription: def.description,\n\t\tschema: def.schema ?? {},\n\t\tresponse:\n\t\t\ttypeof def.response === 'function'\n\t\t\t\t? (def.response as MockToolDef['response'])\n\t\t\t\t: typeof def.response === 'string'\n\t\t\t\t\t? def.response\n\t\t\t\t\t: JSON.stringify(def.response),\n\t}));\n}\n\n/** Strip function responses and ZodObjects so the object is JSON-serialisable for langsmith hashing. */\nfunction toSerializableTools(tools: MockToolDef[]): Record<string, unknown>[] {\n\treturn tools.map((t) => ({\n\t\t...t,\n\t\tschema: t.schema instanceof Object && 'shape' in t.schema ? '<ZodObject>' : t.schema,\n\t\tresponse: typeof t.response === 'function' ? '<function>' : t.response,\n\t}));\n}\n\nfunction lastHumanContent(messages: Message[]): string {\n\tfor (let i = messages.length - 1; i >= 0; i--) {\n\t\tconst msg = messages[i];\n\t\tif (msg.role === 'human') {\n\t\t\tconst textBlock = msg.content.find((c) => c.type === 'text');\n\t\t\treturn textBlock ? textBlock.text : '';\n\t\t}\n\t}\n\treturn '';\n}\n\n// ── Main entry point ─────────────────────────────────────────────────\n\n/**\n * Defines an eval suite. Internally registers `ls.describe` / `ls.test`\n * so vitest discovers the tests — eval files only need to call this function.\n */\nfunction resolveModelTarget(config: SuiteConfig): TargetFn {\n\tif (typeof config.target === 'function') return config.target;\n\tconst evalConfig = getEvalConfig();\n\tif (!evalConfig.model && typeof config.target !== 'string') {\n\t\tthrow new Error('model is required for model-based target. Add it to your configureEvals() call.');\n\t}\n\tconst model = typeof config.target === 'string' ? config.target : evalConfig.model!;\n\treturn createEvalTarget(evalConfig.modelConfig, model);\n}\n\nfunction resolveCreateTarget(config: SuiteConfig): CreateTargetFn | undefined {\n\treturn config.createTarget ?? getEvalConfig().createTarget;\n}\n\nexport function defineSuite(name: string, config: SuiteConfig): void {\n\tconst suiteTools = config.tools ?? {};\n\tconst createTarget = config.target ? undefined : resolveCreateTarget(config);\n\n\tls.describe(name, () => {\n\t\tfor (const tc of config.cases) {\n\t\t\tconst testName = tc.name ?? lastHumanContent(tc.messages);\n\t\t\tconst caseToolDefs = tc.tools ?? suiteTools;\n\t\t\tconst tools = toMockTools(caseToolDefs);\n\t\t\tconst ctx = { message: lastHumanContent(tc.messages) };\n\n\t\t\tconst resolved = tc.expect.map((exp) => exp(ctx));\n\t\t\tconst evaluators = resolved.map((r) => r.evaluator);\n\t\t\tconst referenceOutputs = Object.assign({}, ...resolved.map((r) => r.referenceOutputs));\n\n\t\t\tls.test(\n\t\t\t\ttestName,\n\t\t\t\t{\n\t\t\t\t\tinputs: {\n\t\t\t\t\t\tmessages: tc.messages,\n\t\t\t\t\t\ttools: toSerializableTools(tools),\n\t\t\t\t\t},\n\t\t\t\t\treferenceOutputs,\n\t\t\t\t},\n\t\t\t\tasync ({ referenceOutputs: refOut }) => {\n\t\t\t\t\tlet output: { messages: BaseMessage[] };\n\n\t\t\t\t\t// Resolution order: case > suite > global > identity\n\t\t\t\t\tconst prepareMessages =\n\t\t\t\t\t\ttc.prepareMessages ?? config.prepareMessages ?? getEvalConfig().prepareMessages;\n\t\t\t\t\tconst preparedMessages = prepareMessages\n\t\t\t\t\t\t? await prepareMessages(tc.messages)\n\t\t\t\t\t\t: tc.messages;\n\n\t\t\t\t\tif (createTarget) {\n\t\t\t\t\t\t// Agent mode: create a fresh agent per test, run it, convert result\n\t\t\t\t\t\toutput = await runAgentTarget(createTarget, preparedMessages, caseToolDefs);\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// Model mode: use model-based target with mock tools\n\t\t\t\t\t\tconst target = resolveModelTarget(config);\n\t\t\t\t\t\tconst globalPrompt = getEvalConfig().systemPrompt;\n\t\t\t\t\t\tconst systemPrompt = tc.systemPrompt ?? config.systemPrompt ?? globalPrompt;\n\t\t\t\t\t\toutput = await target({\n\t\t\t\t\t\t\tmessages: preparedMessages,\n\t\t\t\t\t\t\ttools,\n\t\t\t\t\t\t\t...(systemPrompt ? { systemPrompt } : {}),\n\t\t\t\t\t\t});\n\t\t\t\t\t}\n\n\t\t\t\t\tls.logOutputs(output);\n\t\t\t\t\tfor (const evaluator of evaluators) {\n\t\t\t\t\t\tawait evaluator({ outputs: output, referenceOutputs: refOut ?? {} });\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t);\n\t\t}\n\t});\n}\n","import { BaseChatModel } from '@langchain/core/language_models/chat_models';\nimport { tool } from '@langchain/core/tools';\nimport { AIMessage, BaseMessage, SystemMessage, ToolMessage } from '@langchain/core/messages';\nimport { z } from 'zod';\nimport { LangchainModelResolver, type LangchainModelConfig } from '../runtime/langchain/model-resolver';\nimport { type CreateTargetFn, getEvalConfig } from './config';\nimport type { AgentResult, Message as AgentMessage, ToolCallContentBlock, ToolDefinition } from '../core/agent.interface';\nimport { convertToLangchainMessages } from '../runtime/langchain/utils';\nimport type { ToolDef } from './suite';\n\nexport interface MockToolDef {\n\tname: string;\n\tdescription: string;\n\tschema: z.ZodObject<any> | Record<string, unknown>;\n\t/**\n\t * Canned response the mock tool returns.\n\t * Can be a static string, or a function that receives input and returns a response.\n\t * If a function is provided, it receives the full invocation count as a second arg\n\t * to support scenarios like \"first call fails, second call succeeds\".\n\t */\n\tresponse: string | ((input: Record<string, unknown>, callCount: number) => string);\n}\n\nexport interface EvalTargetInput {\n\tsystemPrompt?: string;\n\tmessages: AgentMessage[];\n\ttools: MockToolDef[];\n}\n\nconst MAX_AGENT_LOOPS = 10;\n\n/**\n * Creates a LangSmith-compatible target function that runs an agentic loop\n * with mock tools and returns the full message trajectory.\n */\nexport function createEvalTarget(modelConfig?: LangchainModelConfig, modelString?: string) {\n\treturn async (inputs: EvalTargetInput): Promise<{ messages: BaseMessage[] }> => {\n\t\tconst config = modelConfig && modelString ? { modelConfig, model: modelString } : getEvalConfig();\n\t\tif (!config.model) {\n\t\t\tthrow new Error('model is required for model-based target. Add it to your configureEvals() call.');\n\t\t}\n\t\tconst resolver = new LangchainModelResolver(config.modelConfig);\n\t\tconst model = resolver.resolve(config.model) as BaseChatModel;\n\n\t\t// Track invocation counts per tool for stateful mock responses\n\t\tconst toolCallCounts: Record<string, number> = {};\n\n\t\t// Create langchain tools from mock definitions\n\t\tconst langchainTools = inputs.tools.map((mockTool) => {\n\t\t\ttoolCallCounts[mockTool.name] = 0;\n\n\t\t\treturn tool(\n\t\t\t\tasync (toolInput: Record<string, unknown>) => {\n\t\t\t\t\ttoolCallCounts[mockTool.name]++;\n\t\t\t\t\tif (typeof mockTool.response === 'function') {\n\t\t\t\t\t\treturn mockTool.response(toolInput, toolCallCounts[mockTool.name]);\n\t\t\t\t\t}\n\t\t\t\t\treturn mockTool.response;\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tname: mockTool.name,\n\t\t\t\t\tdescription: mockTool.description,\n\t\t\t\t\tschema:\n\t\t\t\t\t\tmockTool.schema instanceof z.ZodObject\n\t\t\t\t\t\t\t? mockTool.schema\n\t\t\t\t\t\t\t: z.object(\n\t\t\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\t\t\tObject.entries(mockTool.schema).map(([key, val]) => {\n\t\t\t\t\t\t\t\t\t\t\tif (typeof val === 'string') return [key, z.string().describe(val)];\n\t\t\t\t\t\t\t\t\t\t\tif (typeof val === 'number') return [key, z.number().describe(String(val))];\n\t\t\t\t\t\t\t\t\t\t\treturn [key, z.any()];\n\t\t\t\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t\t\t\t),\n\t\t\t\t\t\t\t\t),\n\t\t\t\t},\n\t\t\t);\n\t\t});\n\n\t\tconst boundModel = langchainTools.length > 0 ? model.bindTools!(langchainTools) : model;\n\n\t\tconst messages: BaseMessage[] = [];\n\n\t\tif (inputs.systemPrompt) {\n\t\t\tmessages.push(new SystemMessage(inputs.systemPrompt));\n\t\t}\n\n\t\t// Convert and push all messages (history + final human)\n\t\tmessages.push(...convertToLangchainMessages(inputs.messages));\n\n\t\t// Agentic loop: keep calling model until it stops making tool calls\n\t\tlet loopCount = 0;\n\t\twhile (loopCount < MAX_AGENT_LOOPS) {\n\t\t\tloopCount++;\n\n\t\t\tconst response = await boundModel.invoke(messages);\n\t\t\tmessages.push(response as BaseMessage);\n\n\t\t\tconst aiMessage = response as AIMessage;\n\t\t\tif (!aiMessage.tool_calls || aiMessage.tool_calls.length === 0) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\t// Execute tool calls and add results\n\t\t\tfor (const tc of aiMessage.tool_calls) {\n\t\t\t\tconst mockTool = langchainTools.find((t) => t.name === tc.name);\n\t\t\t\tif (mockTool) {\n\t\t\t\t\tconst result = await mockTool.invoke(tc.args);\n\t\t\t\t\tmessages.push(\n\t\t\t\t\t\tnew ToolMessage({\n\t\t\t\t\t\t\tcontent: typeof result === 'string' ? result : JSON.stringify(result),\n\t\t\t\t\t\t\ttool_call_id: tc.id!,\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t} else {\n\t\t\t\t\tmessages.push(\n\t\t\t\t\t\tnew ToolMessage({\n\t\t\t\t\t\t\tcontent: `Tool \"${tc.name}\" not found`,\n\t\t\t\t\t\t\ttool_call_id: tc.id!,\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn { messages };\n\t};\n}\n\n// ── Agent-based target ──────────────────────────────────────────────\n\n/**\n * Converts an `AgentResult` (from `Agent.run()`) into LangChain `BaseMessage[]`\n * so existing evaluators (trajectory match, no-tool-calls, response-content, language) work unchanged.\n *\n * Consecutive `tool_call` content blocks are grouped into a single `AIMessage` with `tool_calls`,\n * followed by one `ToolMessage` per call.\n */\nexport function agentResultToMessages(inputMessages: AgentMessage[], result: AgentResult): BaseMessage[] {\n\t// Include input messages for trajectory context\n\tconst messages: BaseMessage[] = convertToLangchainMessages(inputMessages);\n\n\t// Group content blocks into BaseMessages\n\tlet pendingToolCalls: { id: string; name: string; args: Record<string, unknown>; output: string }[] = [];\n\n\tfor (const block of result.content) {\n\t\tif (block.type === 'tool_call') {\n\t\t\tconst tc = block as ToolCallContentBlock;\n\t\t\tpendingToolCalls.push({\n\t\t\t\tid: tc.toolCallId,\n\t\t\t\tname: tc.name,\n\t\t\t\targs: tc.input ? JSON.parse(tc.input) : {},\n\t\t\t\toutput: tc.output,\n\t\t\t});\n\t\t} else if (block.type === 'text') {\n\t\t\t// Flush any pending tool calls before the text block\n\t\t\tif (pendingToolCalls.length > 0) {\n\t\t\t\tmessages.push(\n\t\t\t\t\tnew AIMessage({\n\t\t\t\t\t\tcontent: '',\n\t\t\t\t\t\ttool_calls: pendingToolCalls.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })),\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t\tfor (const tc of pendingToolCalls) {\n\t\t\t\t\tmessages.push(new ToolMessage({ content: tc.output, tool_call_id: tc.id, name: tc.name }));\n\t\t\t\t}\n\t\t\t\tpendingToolCalls = [];\n\t\t\t}\n\t\t\tmessages.push(new AIMessage(block.output));\n\t\t}\n\t}\n\n\t// Flush remaining tool calls (agent ended mid-tool-use, unlikely but safe)\n\tif (pendingToolCalls.length > 0) {\n\t\tmessages.push(\n\t\t\tnew AIMessage({\n\t\t\t\tcontent: '',\n\t\t\t\ttool_calls: pendingToolCalls.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })),\n\t\t\t}),\n\t\t);\n\t\tfor (const tc of pendingToolCalls) {\n\t\t\tmessages.push(new ToolMessage({ content: tc.output, tool_call_id: tc.id, name: tc.name }));\n\t\t}\n\t}\n\n\treturn messages;\n}\n\n/**\n * Converts eval `Record<string, ToolDef>` into `ToolDefinition[]` with mock `exec` functions,\n * suitable for passing to an `AgentFactory.createAgent()` call.\n */\nexport function toolDefsToDefinitions(defs: Record<string, ToolDef>): ToolDefinition[] {\n\tconst callCounts: Record<string, number> = {};\n\n\treturn Object.entries(defs).map(([name, def]) => {\n\t\tcallCounts[name] = 0;\n\n\t\treturn {\n\t\t\tname,\n\t\t\ttoolKit: 'eval-mock',\n\t\t\tdescription: def.description,\n\t\t\tinputSchema:\n\t\t\t\tdef.schema instanceof z.ZodObject\n\t\t\t\t\t? def.schema\n\t\t\t\t\t: z.object(\n\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\tObject.entries(def.schema ?? {}).map(([key, val]) => {\n\t\t\t\t\t\t\t\t\tif (typeof val === 'string') return [key, z.string().describe(val)];\n\t\t\t\t\t\t\t\t\treturn [key, z.any()];\n\t\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t),\n\t\t\texec: async (input: Record<string, unknown>) => {\n\t\t\t\tcallCounts[name]++;\n\t\t\t\tif (typeof def.response === 'function') {\n\t\t\t\t\treturn (def.response as (input: Record<string, unknown>, callCount: number) => string)(\n\t\t\t\t\t\tinput,\n\t\t\t\t\t\tcallCounts[name],\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\treturn typeof def.response === 'string' ? def.response : JSON.stringify(def.response);\n\t\t\t},\n\t\t} satisfies ToolDefinition;\n\t});\n}\n\n/**\n * Runs a real `Agent` as the eval target. Creates a fresh agent per invocation via the factory,\n * sends human messages, and converts the `AgentResult` to `{ messages: BaseMessage[] }`.\n */\nexport async function runAgentTarget(\n\tcreateTarget: CreateTargetFn,\n\tevalMessages: AgentMessage[],\n\textraToolDefs: Record<string, ToolDef>,\n): Promise<{ messages: BaseMessage[] }> {\n\tconst extraTools = Object.keys(extraToolDefs).length > 0 ? toolDefsToDefinitions(extraToolDefs) : [];\n\tconst agent = await createTarget(extraTools);\n\n\tconst result = await agent.run({\n\t\tthreadId: `eval_${Date.now()}_${Math.random().toString(36).slice(2)}`,\n\t\tmessages: evalMessages,\n\t});\n\n\treturn { messages: agentResultToMessages(evalMessages, result) };\n}\n","import * as ls from 'langsmith/vitest';\nimport {\n\tcreateTrajectoryMatchEvaluator,\n\tcreateTrajectoryLLMAsJudge,\n\tTRAJECTORY_ACCURACY_PROMPT,\n} from 'agentevals';\nimport { createLanguageEvaluator } from './evaluators/language';\nimport { createResponseContentEvaluator } from './evaluators/response-content';\nimport { createNoToolCallsEvaluator } from './evaluators/no-tool-calls';\nimport { getEvalConfig } from './config';\n\n// ── Types ────────────────────────────────────────────────────────────\n\ntype EvaluatorFn = (args: {\n\toutputs: Record<string, any>;\n\treferenceOutputs: Record<string, any>;\n}) => Promise<any>;\n\ninterface ResolvedExpectation {\n\tevaluator: EvaluatorFn;\n\treferenceOutputs: Record<string, unknown>;\n}\n\n/** A factory that receives test context and returns an evaluator + its referenceOutputs. */\nexport type Expectation = (ctx: { message: string }) => ResolvedExpectation;\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nfunction withTrajectoryGuard(evaluator: any, key: string): EvaluatorFn {\n\treturn async ({ outputs, referenceOutputs }) => {\n\t\tif (!referenceOutputs?.referenceTrajectory) {\n\t\t\treturn { key, score: true, comment: 'No referenceTrajectory specified, skipping' };\n\t\t}\n\t\treturn evaluator({ outputs, referenceOutputs: referenceOutputs.referenceTrajectory });\n\t};\n}\n\nfunction buildTrajectory(message: string, toolNames: string[]): Record<string, unknown>[] {\n\tconst trajectory: Record<string, unknown>[] = [];\n\tlet tcIdx = 0;\n\n\ttrajectory.push({ role: 'user', content: message });\n\n\tfor (const name of toolNames) {\n\t\tconst id = `tc${++tcIdx}`;\n\t\ttrajectory.push({\n\t\t\trole: 'assistant',\n\t\t\tcontent: '',\n\t\t\ttool_calls: [{ function: { name, arguments: '{}' }, id, type: 'function' }],\n\t\t});\n\t\ttrajectory.push({ role: 'tool', content: '...', tool_call_id: id });\n\t}\n\n\ttrajectory.push({ role: 'assistant', content: '...' });\n\n\treturn trajectory;\n}\n\n// ── Expectation functions ────────────────────────────────────────────\n\n/**\n * Expect the agent to call tools in order (superset trajectory match).\n * Empty `[]` means the agent should answer directly without calling any tools.\n */\nexport function toolsCalled(tools: string[]): Expectation {\n\treturn (ctx) => ({\n\t\tevaluator: ls.wrapEvaluator(\n\t\t\twithTrajectoryGuard(\n\t\t\t\tcreateTrajectoryMatchEvaluator({ trajectoryMatchMode: 'superset', toolArgsMatchMode: 'ignore' }) as any,\n\t\t\t\t'trajectory_match',\n\t\t\t),\n\t\t),\n\t\treferenceOutputs: { referenceTrajectory: buildTrajectory(ctx.message, tools) },\n\t});\n}\n\n/**\n * Run an LLM-as-judge evaluator on the trajectory.\n * Requires `toolsCalled` in the same expect array.\n * Uses the globally configured evaluator model.\n */\nexport function llmJudge(): Expectation {\n\treturn () => {\n\t\tconst config = getEvalConfig();\n\t\tconst model = config.evaluatorModel;\n\t\treturn {\n\t\t\tevaluator: ls.wrapEvaluator(\n\t\t\t\twithTrajectoryGuard(\n\t\t\t\t\tcreateTrajectoryLLMAsJudge({ prompt: TRAJECTORY_ACCURACY_PROMPT, model }) as any,\n\t\t\t\t\t'trajectory_llm_judge',\n\t\t\t\t),\n\t\t\t),\n\t\t\treferenceOutputs: {},\n\t\t};\n\t};\n}\n\n/** Assert the agent made zero tool calls. */\nexport function noTools(): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createNoToolCallsEvaluator()),\n\t\treferenceOutputs: { expectNoToolCalls: true },\n\t});\n}\n\n/**\n * Assert the response is in the given language (ISO 639-1 code).\n * Uses the globally configured evaluator model for language detection.\n * @param code - ISO 639-1 language code (e.g. 'en', 'tr', 'de').\n */\nexport function respondsInLanguage(code: string): Expectation {\n\treturn () => {\n\t\tconst config = getEvalConfig();\n\t\tconst model = config.evaluatorModel;\n\t\treturn {\n\t\t\tevaluator: ls.wrapEvaluator(createLanguageEvaluator(config.modelConfig, model)),\n\t\t\treferenceOutputs: { expectedLanguage: code },\n\t\t};\n\t};\n}\n\n/** Assert the response contains all given strings. */\nexport function contains(strings: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createResponseContentEvaluator()),\n\t\treferenceOutputs: { responseContains: strings },\n\t});\n}\n\n/** Assert the response does not contain any of the given strings. */\nexport function notContains(strings: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createResponseContentEvaluator()),\n\t\treferenceOutputs: { responseMustNotContain: strings },\n\t});\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\nimport { BaseChatModel } from '@langchain/core/language_models/chat_models';\nimport { LangchainModelResolver, type LangchainModelConfig } from '../../runtime/langchain/model-resolver';\n\n/**\n * Creates a custom evaluator that checks whether the agent's final response\n * is in the expected language. Uses a cheap LLM call for language detection.\n */\nexport function createLanguageEvaluator(modelConfig: LangchainModelConfig, model: string) {\n\tconst resolver = new LangchainModelResolver(modelConfig);\n\tconst judge = resolver.resolve(model) as BaseChatModel;\n\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tconst expectedLanguage = referenceOutputs?.expectedLanguage;\n\t\tif (!expectedLanguage) {\n\t\t\treturn { key: 'language_match', score: true, comment: 'No expected language specified, skipping' };\n\t\t}\n\n\t\t// Extract the last AI message text from the trajectory\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage);\n\n\t\tif (!lastAiMessage) {\n\t\t\treturn { key: 'language_match', score: false, comment: 'No AI message found in trajectory' };\n\t\t}\n\n\t\tconst responseText = typeof lastAiMessage.content === 'string' ? lastAiMessage.content : JSON.stringify(lastAiMessage.content);\n\n\t\t// Use LLM to detect the language\n\t\tconst detection = await judge.invoke([\n\t\t\t{\n\t\t\t\trole: 'system',\n\t\t\t\tcontent: 'You are a language detection tool. Respond with ONLY the ISO 639-1 language code (e.g., \"en\", \"tr\", \"de\", \"fr\") of the text provided. Nothing else.',\n\t\t\t},\n\t\t\t{\n\t\t\t\trole: 'user',\n\t\t\t\tcontent: responseText,\n\t\t\t},\n\t\t]);\n\n\t\tconst detectedLanguage = (typeof detection.content === 'string' ? detection.content : '').trim().toLowerCase();\n\n\t\tconst matches = detectedLanguage === expectedLanguage.toLowerCase();\n\n\t\treturn {\n\t\t\tkey: 'language_match',\n\t\t\tscore: matches,\n\t\t\tcomment: matches\n\t\t\t\t? `Response language matches expected: ${expectedLanguage}`\n\t\t\t\t: `Expected \"${expectedLanguage}\" but detected \"${detectedLanguage}\"`,\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that checks whether the agent's final response\n * contains expected strings and doesn't contain forbidden strings.\n */\nexport function createResponseContentEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tconst mustContain: string[] = referenceOutputs?.responseContains || [];\n\t\tconst mustNotContain: string[] = referenceOutputs?.responseMustNotContain || [];\n\n\t\tif (mustContain.length === 0 && mustNotContain.length === 0) {\n\t\t\treturn { key: 'response_content', score: true, comment: 'No content assertions specified, skipping' };\n\t\t}\n\n\t\t// Extract the last AI message text from the trajectory\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage);\n\n\t\tif (!lastAiMessage) {\n\t\t\treturn { key: 'response_content', score: false, comment: 'No AI message found in trajectory' };\n\t\t}\n\n\t\tconst responseText = (typeof lastAiMessage.content === 'string' ? lastAiMessage.content : JSON.stringify(lastAiMessage.content)).toLowerCase();\n\n\t\tconst failures: string[] = [];\n\n\t\tfor (const expected of mustContain) {\n\t\t\tif (!responseText.includes(expected.toLowerCase())) {\n\t\t\t\tfailures.push(`Missing expected text: \"${expected}\"`);\n\t\t\t}\n\t\t}\n\n\t\tfor (const forbidden of mustNotContain) {\n\t\t\tif (responseText.includes(forbidden.toLowerCase())) {\n\t\t\t\tfailures.push(`Contains forbidden text: \"${forbidden}\"`);\n\t\t\t}\n\t\t}\n\n\t\tconst passed = failures.length === 0;\n\n\t\treturn {\n\t\t\tkey: 'response_content',\n\t\t\tscore: passed,\n\t\t\tcomment: passed ? 'All content assertions passed' : failures.join('; '),\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that asserts the agent made zero tool calls.\n * Useful for scenarios like greetings where the agent should just respond with text.\n */\nexport function createNoToolCallsEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\t// Only run this evaluator if the reference explicitly expects no tool calls\n\t\tif (referenceOutputs?.maxToolCalls !== 0 && referenceOutputs?.expectNoToolCalls !== true) {\n\t\t\treturn { key: 'no_tool_calls', score: true, comment: 'No tool call restriction specified, skipping' };\n\t\t}\n\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\n\t\tconst toolCalls = messages\n\t\t\t.filter((m) => m instanceof AIMessage)\n\t\t\t.flatMap((m) => (m as AIMessage).tool_calls || []);\n\n\t\tconst passed = toolCalls.length === 0;\n\n\t\treturn {\n\t\t\tkey: 'no_tool_calls',\n\t\t\tscore: passed,\n\t\t\tcomment: passed\n\t\t\t\t? 'No tool calls made (as expected)'\n\t\t\t\t: `Agent made ${toolCalls.length} tool call(s): ${toolCalls.map((tc) => tc.name).join(', ')}`,\n\t\t};\n\t};\n}\n"],"mappings":";;;;;;AAqBA,IAAI,UAA6B;AAE1B,SAAS,eAAe,QAA0B;AACxD,YAAU;AACX;AAEO,SAAS,gBAA4B;AAC3C,MAAI,CAAC,SAAS;AACb,UAAM,IAAI,MAAM,wEAAwE;AAAA,EACzF;AACA,SAAO;AACR;;;AChCA,YAAY,QAAQ;;;ACCpB,SAAS,YAAY;AACrB,SAAS,WAAwB,eAAe,mBAAmB;AACnE,SAAS,SAAS;AA0BlB,IAAM,kBAAkB;AAMjB,SAAS,iBAAiB,aAAoC,aAAsB;AAC1F,SAAO,OAAO,WAAkE;AAC/E,UAAM,SAAS,eAAe,cAAc,EAAE,aAAa,OAAO,YAAY,IAAI,cAAc;AAChG,QAAI,CAAC,OAAO,OAAO;AAClB,YAAM,IAAI,MAAM,iFAAiF;AAAA,IAClG;AACA,UAAM,WAAW,IAAI,uBAAuB,OAAO,WAAW;AAC9D,UAAM,QAAQ,SAAS,QAAQ,OAAO,KAAK;AAG3C,UAAM,iBAAyC,CAAC;AAGhD,UAAM,iBAAiB,OAAO,MAAM,IAAI,CAAC,aAAa;AACrD,qBAAe,SAAS,IAAI,IAAI;AAEhC,aAAO;AAAA,QACN,OAAO,cAAuC;AAC7C,yBAAe,SAAS,IAAI;AAC5B,cAAI,OAAO,SAAS,aAAa,YAAY;AAC5C,mBAAO,SAAS,SAAS,WAAW,eAAe,SAAS,IAAI,CAAC;AAAA,UAClE;AACA,iBAAO,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,UACC,MAAM,SAAS;AAAA,UACf,aAAa,SAAS;AAAA,UACtB,QACC,SAAS,kBAAkB,EAAE,YAC1B,SAAS,SACT,EAAE;AAAA,YACF,OAAO;AAAA,cACN,OAAO,QAAQ,SAAS,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM;AACnD,oBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,GAAG,CAAC;AAClE,oBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,OAAO,GAAG,CAAC,CAAC;AAC1E,uBAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAAA,cACrB,CAAC;AAAA,YACF;AAAA,UACD;AAAA,QACJ;AAAA,MACD;AAAA,IACD,CAAC;AAED,UAAM,aAAa,eAAe,SAAS,IAAI,MAAM,UAAW,cAAc,IAAI;AAElF,UAAM,WAA0B,CAAC;AAEjC,QAAI,OAAO,cAAc;AACxB,eAAS,KAAK,IAAI,cAAc,OAAO,YAAY,CAAC;AAAA,IACrD;AAGA,aAAS,KAAK,GAAG,2BAA2B,OAAO,QAAQ,CAAC;AAG5D,QAAI,YAAY;AAChB,WAAO,YAAY,iBAAiB;AACnC;AAEA,YAAM,WAAW,MAAM,WAAW,OAAO,QAAQ;AACjD,eAAS,KAAK,QAAuB;AAErC,YAAM,YAAY;AAClB,UAAI,CAAC,UAAU,cAAc,UAAU,WAAW,WAAW,GAAG;AAC/D;AAAA,MACD;AAGA,iBAAW,MAAM,UAAU,YAAY;AACtC,cAAM,WAAW,eAAe,KAAK,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI;AAC9D,YAAI,UAAU;AACb,gBAAM,SAAS,MAAM,SAAS,OAAO,GAAG,IAAI;AAC5C,mBAAS;AAAA,YACR,IAAI,YAAY;AAAA,cACf,SAAS,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,cACpE,cAAc,GAAG;AAAA,cACjB,MAAM,GAAG;AAAA,YACV,CAAC;AAAA,UACF;AAAA,QACD,OAAO;AACN,mBAAS;AAAA,YACR,IAAI,YAAY;AAAA,cACf,SAAS,SAAS,GAAG,IAAI;AAAA,cACzB,cAAc,GAAG;AAAA,cACjB,MAAM,GAAG;AAAA,YACV,CAAC;AAAA,UACF;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAEA,WAAO,EAAE,SAAS;AAAA,EACnB;AACD;AAWO,SAAS,sBAAsB,eAA+B,QAAoC;AAExG,QAAM,WAA0B,2BAA2B,aAAa;AAGxE,MAAI,mBAAkG,CAAC;AAEvG,aAAW,SAAS,OAAO,SAAS;AACnC,QAAI,MAAM,SAAS,aAAa;AAC/B,YAAM,KAAK;AACX,uBAAiB,KAAK;AAAA,QACrB,IAAI,GAAG;AAAA,QACP,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,QAAQ,KAAK,MAAM,GAAG,KAAK,IAAI,CAAC;AAAA,QACzC,QAAQ,GAAG;AAAA,MACZ,CAAC;AAAA,IACF,WAAW,MAAM,SAAS,QAAQ;AAEjC,UAAI,iBAAiB,SAAS,GAAG;AAChC,iBAAS;AAAA,UACR,IAAI,UAAU;AAAA,YACb,SAAS;AAAA,YACT,YAAY,iBAAiB,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,MAAM,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE;AAAA,UACvF,CAAC;AAAA,QACF;AACA,mBAAW,MAAM,kBAAkB;AAClC,mBAAS,KAAK,IAAI,YAAY,EAAE,SAAS,GAAG,QAAQ,cAAc,GAAG,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,QAC1F;AACA,2BAAmB,CAAC;AAAA,MACrB;AACA,eAAS,KAAK,IAAI,UAAU,MAAM,MAAM,CAAC;AAAA,IAC1C;AAAA,EACD;AAGA,MAAI,iBAAiB,SAAS,GAAG;AAChC,aAAS;AAAA,MACR,IAAI,UAAU;AAAA,QACb,SAAS;AAAA,QACT,YAAY,iBAAiB,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,MAAM,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE;AAAA,MACvF,CAAC;AAAA,IACF;AACA,eAAW,MAAM,kBAAkB;AAClC,eAAS,KAAK,IAAI,YAAY,EAAE,SAAS,GAAG,QAAQ,cAAc,GAAG,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,IAC1F;AAAA,EACD;AAEA,SAAO;AACR;AAMO,SAAS,sBAAsB,MAAiD;AACtF,QAAM,aAAqC,CAAC;AAE5C,SAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM;AAChD,eAAW,IAAI,IAAI;AAEnB,WAAO;AAAA,MACN;AAAA,MACA,SAAS;AAAA,MACT,aAAa,IAAI;AAAA,MACjB,aACC,IAAI,kBAAkB,EAAE,YACrB,IAAI,SACJ,EAAE;AAAA,QACF,OAAO;AAAA,UACN,OAAO,QAAQ,IAAI,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM;AACpD,gBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,GAAG,CAAC;AAClE,mBAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAAA,UACrB,CAAC;AAAA,QACF;AAAA,MACD;AAAA,MACH,MAAM,OAAO,UAAmC;AAC/C,mBAAW,IAAI;AACf,YAAI,OAAO,IAAI,aAAa,YAAY;AACvC,iBAAQ,IAAI;AAAA,YACX;AAAA,YACA,WAAW,IAAI;AAAA,UAChB;AAAA,QACD;AACA,eAAO,OAAO,IAAI,aAAa,WAAW,IAAI,WAAW,KAAK,UAAU,IAAI,QAAQ;AAAA,MACrF;AAAA,IACD;AAAA,EACD,CAAC;AACF;AAMA,eAAsB,eACrB,cACA,cACA,eACuC;AACvC,QAAM,aAAa,OAAO,KAAK,aAAa,EAAE,SAAS,IAAI,sBAAsB,aAAa,IAAI,CAAC;AACnG,QAAM,QAAQ,MAAM,aAAa,UAAU;AAE3C,QAAM,SAAS,MAAM,MAAM,IAAI;AAAA,IAC9B,UAAU,QAAQ,KAAK,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AAAA,IACnE,UAAU;AAAA,EACX,CAAC;AAED,SAAO,EAAE,UAAU,sBAAsB,cAAc,MAAM,EAAE;AAChE;;;ADvOO,SAAS,MAAM,SAA+B;AACpD,SAAO,EAAE,MAAM,SAAS,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,QAAQ,CAAC,EAAE;AACpE;AAEO,SAAS,GAAG,SAAiB,WAAiC;AACpE,SAAO,EAAE,MAAM,MAAM,SAAS,GAAI,YAAY,EAAE,WAAW,UAAU,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,IAAI,CAAC,EAAG;AACxG;AAEO,SAAS,WAAW,MAAc,QAA6B;AACrE,SAAO,EAAE,MAAM,QAAQ,MAAM,OAAO;AACrC;AA8CO,SAAS,cACf,OACA,YAAiD,CAAC,GACxB;AAC1B,SAAO,OAAO;AAAA,IACb,MAAM,IAAI,CAAC,SAAS;AAAA,MACnB,KAAK;AAAA,MACL;AAAA,QACC,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,UAAU,UAAU,KAAK,IAAI,KAAK;AAAA,MACnC;AAAA,IACD,CAAC;AAAA,EACF;AACD;AAEA,SAAS,YAAY,MAA8C;AAClE,SAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO;AAAA,IACjD;AAAA,IACA,aAAa,IAAI;AAAA,IACjB,QAAQ,IAAI,UAAU,CAAC;AAAA,IACvB,UACC,OAAO,IAAI,aAAa,aACpB,IAAI,WACL,OAAO,IAAI,aAAa,WACvB,IAAI,WACJ,KAAK,UAAU,IAAI,QAAQ;AAAA,EACjC,EAAE;AACH;AAGA,SAAS,oBAAoB,OAAiD;AAC7E,SAAO,MAAM,IAAI,CAAC,OAAO;AAAA,IACxB,GAAG;AAAA,IACH,QAAQ,EAAE,kBAAkB,UAAU,WAAW,EAAE,SAAS,gBAAgB,EAAE;AAAA,IAC9E,UAAU,OAAO,EAAE,aAAa,aAAa,eAAe,EAAE;AAAA,EAC/D,EAAE;AACH;AAEA,SAAS,iBAAiB,UAA6B;AACtD,WAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC9C,UAAM,MAAM,SAAS,CAAC;AACtB,QAAI,IAAI,SAAS,SAAS;AACzB,YAAM,YAAY,IAAI,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM;AAC3D,aAAO,YAAY,UAAU,OAAO;AAAA,IACrC;AAAA,EACD;AACA,SAAO;AACR;AAQA,SAAS,mBAAmB,QAA+B;AAC1D,MAAI,OAAO,OAAO,WAAW,WAAY,QAAO,OAAO;AACvD,QAAM,aAAa,cAAc;AACjC,MAAI,CAAC,WAAW,SAAS,OAAO,OAAO,WAAW,UAAU;AAC3D,UAAM,IAAI,MAAM,iFAAiF;AAAA,EAClG;AACA,QAAM,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS,WAAW;AAC7E,SAAO,iBAAiB,WAAW,aAAa,KAAK;AACtD;AAEA,SAAS,oBAAoB,QAAiD;AAC7E,SAAO,OAAO,gBAAgB,cAAc,EAAE;AAC/C;AAEO,SAAS,YAAY,MAAc,QAA2B;AACpE,QAAM,aAAa,OAAO,SAAS,CAAC;AACpC,QAAM,eAAe,OAAO,SAAS,SAAY,oBAAoB,MAAM;AAE3E,EAAG,YAAS,MAAM,MAAM;AACvB,eAAW,MAAM,OAAO,OAAO;AAC9B,YAAM,WAAW,GAAG,QAAQ,iBAAiB,GAAG,QAAQ;AACxD,YAAM,eAAe,GAAG,SAAS;AACjC,YAAM,QAAQ,YAAY,YAAY;AACtC,YAAM,MAAM,EAAE,SAAS,iBAAiB,GAAG,QAAQ,EAAE;AAErD,YAAM,WAAW,GAAG,OAAO,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC;AAChD,YAAM,aAAa,SAAS,IAAI,CAAC,MAAM,EAAE,SAAS;AAClD,YAAM,mBAAmB,OAAO,OAAO,CAAC,GAAG,GAAG,SAAS,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;AAErF,MAAG;AAAA,QACF;AAAA,QACA;AAAA,UACC,QAAQ;AAAA,YACP,UAAU,GAAG;AAAA,YACb,OAAO,oBAAoB,KAAK;AAAA,UACjC;AAAA,UACA;AAAA,QACD;AAAA,QACA,OAAO,EAAE,kBAAkB,OAAO,MAAM;AACvC,cAAI;AAGJ,gBAAM,kBACL,GAAG,mBAAmB,OAAO,mBAAmB,cAAc,EAAE;AACjE,gBAAM,mBAAmB,kBACtB,MAAM,gBAAgB,GAAG,QAAQ,IACjC,GAAG;AAEN,cAAI,cAAc;AAEjB,qBAAS,MAAM,eAAe,cAAc,kBAAkB,YAAY;AAAA,UAC3E,OAAO;AAEN,kBAAM,SAAS,mBAAmB,MAAM;AACxC,kBAAM,eAAe,cAAc,EAAE;AACrC,kBAAM,eAAe,GAAG,gBAAgB,OAAO,gBAAgB;AAC/D,qBAAS,MAAM,OAAO;AAAA,cACrB,UAAU;AAAA,cACV;AAAA,cACA,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;AAAA,YACxC,CAAC;AAAA,UACF;AAEA,UAAG,cAAW,MAAM;AACpB,qBAAW,aAAa,YAAY;AACnC,kBAAM,UAAU,EAAE,SAAS,QAAQ,kBAAkB,UAAU,CAAC,EAAE,CAAC;AAAA,UACpE;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AACF;;;AEtMA,YAAYA,SAAQ;AACpB;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;;;ACLP,SAAS,aAAAC,kBAA8B;AAQhC,SAAS,wBAAwB,aAAmC,OAAe;AACzF,QAAM,WAAW,IAAI,uBAAuB,WAAW;AACvD,QAAM,QAAQ,SAAS,QAAQ,KAAK;AAEpC,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,UAAM,mBAAmB,kBAAkB;AAC3C,QAAI,CAAC,kBAAkB;AACtB,aAAO,EAAE,KAAK,kBAAkB,OAAO,MAAM,SAAS,2CAA2C;AAAA,IAClG;AAGA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,gBAAgB,CAAC,GAAG,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,aAAaC,UAAS;AAEhF,QAAI,CAAC,eAAe;AACnB,aAAO,EAAE,KAAK,kBAAkB,OAAO,OAAO,SAAS,oCAAoC;AAAA,IAC5F;AAEA,UAAM,eAAe,OAAO,cAAc,YAAY,WAAW,cAAc,UAAU,KAAK,UAAU,cAAc,OAAO;AAG7H,UAAM,YAAY,MAAM,MAAM,OAAO;AAAA,MACpC;AAAA,QACC,MAAM;AAAA,QACN,SAAS;AAAA,MACV;AAAA,MACA;AAAA,QACC,MAAM;AAAA,QACN,SAAS;AAAA,MACV;AAAA,IACD,CAAC;AAED,UAAM,oBAAoB,OAAO,UAAU,YAAY,WAAW,UAAU,UAAU,IAAI,KAAK,EAAE,YAAY;AAE7G,UAAM,UAAU,qBAAqB,iBAAiB,YAAY;AAElE,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,UACN,uCAAuC,gBAAgB,KACvD,aAAa,gBAAgB,mBAAmB,gBAAgB;AAAA,IACpE;AAAA,EACD;AACD;;;AC1DA,SAAS,aAAAC,kBAA8B;AAMhC,SAAS,iCAAiC;AAChD,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,UAAM,cAAwB,kBAAkB,oBAAoB,CAAC;AACrE,UAAM,iBAA2B,kBAAkB,0BAA0B,CAAC;AAE9E,QAAI,YAAY,WAAW,KAAK,eAAe,WAAW,GAAG;AAC5D,aAAO,EAAE,KAAK,oBAAoB,OAAO,MAAM,SAAS,4CAA4C;AAAA,IACrG;AAGA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,gBAAgB,CAAC,GAAG,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,aAAaA,UAAS;AAEhF,QAAI,CAAC,eAAe;AACnB,aAAO,EAAE,KAAK,oBAAoB,OAAO,OAAO,SAAS,oCAAoC;AAAA,IAC9F;AAEA,UAAM,gBAAgB,OAAO,cAAc,YAAY,WAAW,cAAc,UAAU,KAAK,UAAU,cAAc,OAAO,GAAG,YAAY;AAE7I,UAAM,WAAqB,CAAC;AAE5B,eAAW,YAAY,aAAa;AACnC,UAAI,CAAC,aAAa,SAAS,SAAS,YAAY,CAAC,GAAG;AACnD,iBAAS,KAAK,2BAA2B,QAAQ,GAAG;AAAA,MACrD;AAAA,IACD;AAEA,eAAW,aAAa,gBAAgB;AACvC,UAAI,aAAa,SAAS,UAAU,YAAY,CAAC,GAAG;AACnD,iBAAS,KAAK,6BAA6B,SAAS,GAAG;AAAA,MACxD;AAAA,IACD;AAEA,UAAM,SAAS,SAAS,WAAW;AAEnC,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SAAS,kCAAkC,SAAS,KAAK,IAAI;AAAA,IACvE;AAAA,EACD;AACD;;;ACrDA,SAAS,aAAAC,kBAA8B;AAMhC,SAAS,6BAA6B;AAC5C,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AAEL,QAAI,kBAAkB,iBAAiB,KAAK,kBAAkB,sBAAsB,MAAM;AACzF,aAAO,EAAE,KAAK,iBAAiB,OAAO,MAAM,SAAS,+CAA+C;AAAA,IACrG;AAEA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AAErD,UAAM,YAAY,SAChB,OAAO,CAAC,MAAM,aAAaA,UAAS,EACpC,QAAQ,CAAC,MAAO,EAAgB,cAAc,CAAC,CAAC;AAElD,UAAM,SAAS,UAAU,WAAW;AAEpC,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SACN,qCACA,cAAc,UAAU,MAAM,kBAAkB,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,IAC7F;AAAA,EACD;AACD;;;AHPA,SAAS,oBAAoB,WAAgB,KAA0B;AACtE,SAAO,OAAO,EAAE,SAAS,iBAAiB,MAAM;AAC/C,QAAI,CAAC,kBAAkB,qBAAqB;AAC3C,aAAO,EAAE,KAAK,OAAO,MAAM,SAAS,6CAA6C;AAAA,IAClF;AACA,WAAO,UAAU,EAAE,SAAS,kBAAkB,iBAAiB,oBAAoB,CAAC;AAAA,EACrF;AACD;AAEA,SAAS,gBAAgB,SAAiB,WAAgD;AACzF,QAAM,aAAwC,CAAC;AAC/C,MAAI,QAAQ;AAEZ,aAAW,KAAK,EAAE,MAAM,QAAQ,SAAS,QAAQ,CAAC;AAElD,aAAW,QAAQ,WAAW;AAC7B,UAAM,KAAK,KAAK,EAAE,KAAK;AACvB,eAAW,KAAK;AAAA,MACf,MAAM;AAAA,MACN,SAAS;AAAA,MACT,YAAY,CAAC,EAAE,UAAU,EAAE,MAAM,WAAW,KAAK,GAAG,IAAI,MAAM,WAAW,CAAC;AAAA,IAC3E,CAAC;AACD,eAAW,KAAK,EAAE,MAAM,QAAQ,SAAS,OAAO,cAAc,GAAG,CAAC;AAAA,EACnE;AAEA,aAAW,KAAK,EAAE,MAAM,aAAa,SAAS,MAAM,CAAC;AAErD,SAAO;AACR;AAQO,SAAS,YAAY,OAA8B;AACzD,SAAO,CAAC,SAAS;AAAA,IAChB,WAAc;AAAA,MACb;AAAA,QACC,+BAA+B,EAAE,qBAAqB,YAAY,mBAAmB,SAAS,CAAC;AAAA,QAC/F;AAAA,MACD;AAAA,IACD;AAAA,IACA,kBAAkB,EAAE,qBAAqB,gBAAgB,IAAI,SAAS,KAAK,EAAE;AAAA,EAC9E;AACD;AAOO,SAAS,WAAwB;AACvC,SAAO,MAAM;AACZ,UAAM,SAAS,cAAc;AAC7B,UAAM,QAAQ,OAAO;AACrB,WAAO;AAAA,MACN,WAAc;AAAA,QACb;AAAA,UACC,2BAA2B,EAAE,QAAQ,4BAA4B,MAAM,CAAC;AAAA,UACxE;AAAA,QACD;AAAA,MACD;AAAA,MACA,kBAAkB,CAAC;AAAA,IACpB;AAAA,EACD;AACD;AAGO,SAAS,UAAuB;AACtC,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,2BAA2B,CAAC;AAAA,IACxD,kBAAkB,EAAE,mBAAmB,KAAK;AAAA,EAC7C;AACD;AAOO,SAAS,mBAAmB,MAA2B;AAC7D,SAAO,MAAM;AACZ,UAAM,SAAS,cAAc;AAC7B,UAAM,QAAQ,OAAO;AACrB,WAAO;AAAA,MACN,WAAc,kBAAc,wBAAwB,OAAO,aAAa,KAAK,CAAC;AAAA,MAC9E,kBAAkB,EAAE,kBAAkB,KAAK;AAAA,IAC5C;AAAA,EACD;AACD;AAGO,SAAS,SAAS,SAAgC;AACxD,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,+BAA+B,CAAC;AAAA,IAC5D,kBAAkB,EAAE,kBAAkB,QAAQ;AAAA,EAC/C;AACD;AAGO,SAAS,YAAY,SAAgC;AAC3D,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,+BAA+B,CAAC;AAAA,IAC5D,kBAAkB,EAAE,wBAAwB,QAAQ;AAAA,EACrD;AACD;","names":["ls","AIMessage","AIMessage","AIMessage","AIMessage"]}
1
+ {"version":3,"sources":["../../src/eval/config.ts","../../src/eval/suite.ts","../../src/eval/target.ts","../../src/eval/expectations.ts","../../src/eval/evaluators/language.ts","../../src/eval/evaluators/response-content.ts","../../src/eval/evaluators/no-tool-calls.ts","../../src/eval/evaluators/any-tool-called.ts"],"sourcesContent":["import type { Agent, Message, ToolDefinition } from '../core/agent.interface';\nimport type { LangchainModelConfig } from '../runtime/langchain/model-resolver';\n\n/** Factory that creates a fresh Agent per test case. Receives extra suite-level tools as ToolDefinition[]. */\nexport type CreateTargetFn = (extraTools: ToolDefinition[]) => Agent | Promise<Agent>;\n\nexport interface EvalConfig {\n\t/** Required for model-based target and LLM evaluators (respondsInLanguage, llmJudge). */\n\tmodelConfig: LangchainModelConfig;\n\t/** Required for model-based target. Also used as fallback for evaluatorModel. */\n\tmodel?: string;\n\t/** Model for evaluators needing LLM calls (language detection, LLM-as-judge). */\n\tevaluatorModel: string;\n\t/** LangSmith experiment (dataset) name. All suites share this single experiment for easy comparison across runs. */\n\texperimentName: string;\n\t/** System prompt for model-based target. Ignored when createTarget is used. Can be overridden per-suite or per-case. */\n\tsystemPrompt?: string;\n\t/** Factory that creates a fresh Agent per test case. When set, this is the default target. */\n\tcreateTarget?: CreateTargetFn;\n\t/** Transforms test case messages before sending to target. Simulates production preprocessing (e.g., message enrichment). */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n}\n\nlet _config: EvalConfig | null = null;\n\nexport function configureEvals(config: EvalConfig): void {\n\t_config = config;\n}\n\nexport function getEvalConfig(): EvalConfig {\n\tif (!_config) {\n\t\tthrow new Error('Evals not configured. Call configureEvals() in your vitest setupFiles.');\n\t}\n\treturn _config;\n}\n\n","import * as ls from 'langsmith/vitest';\nimport { AIMessage, BaseMessage } from '@langchain/core/messages';\nimport { createEvalTarget, runAgentTarget, type MockToolDef } from './target';\nimport { type Expectation } from './expectations';\nimport { getEvalConfig, type CreateTargetFn } from './config';\nimport {\n\ttype Message,\n\ttype HumanMessage,\n\ttype AiMessage,\n\ttype ToolMessage,\n\ttype ToolSpec,\n} from '../core/agent.interface';\n\n// ── Message builders ─────────────────────────────────────────────────\n\nexport function human(content: string): HumanMessage {\n\treturn { role: 'human', content: [{ type: 'text', text: content }] };\n}\n\nexport function ai(content: string, toolCalls?: string[]): AiMessage {\n\treturn { role: 'ai', content, ...(toolCalls ? { toolCalls: toolCalls.map((name) => ({ name })) } : {}) };\n}\n\nexport function toolResult(name: string, output: string): ToolMessage {\n\treturn { role: 'tool', name, output };\n}\n\nexport interface ToolDef {\n\tdescription: string;\n\t/** A plain key→description record, or a ZodObject passed through from a ToolSpec. */\n\tschema?: Record<string, string> | import('zod').ZodObject<any>;\n\t/** Auto-stringified if not a string or function. */\n\tresponse: unknown | ((input: Record<string, unknown>, callCount: number) => string);\n}\n\nexport interface TestCase {\n\t/** Test name. Defaults to the last human message content if omitted. */\n\tname?: string;\n\tmessages: Message[];\n\tsystemPrompt?: string;\n\t/** Override suite-level tools for this case. */\n\ttools?: Record<string, ToolDef>;\n\t/** Transforms messages before sending to target. Overrides suite-level and global hooks. */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n\texpect: Expectation[];\n}\n\ntype TargetFn = (inputs: {\n\tsystemPrompt?: string;\n\tmessages: Message[];\n\ttools: MockToolDef[];\n}) => Promise<{ messages: BaseMessage[] }>;\n\nexport interface SuiteConfig {\n\t/** Custom target function, or model string override. Auto-created from global config if omitted. */\n\ttarget?: TargetFn | string;\n\t/** Factory that creates a fresh Agent per test case. Overrides global createTarget. */\n\tcreateTarget?: CreateTargetFn;\n\t/** System prompt for all cases in this suite. Overrides the global prompt; can be overridden per-case. */\n\tsystemPrompt?: string;\n\ttools?: Record<string, ToolDef>;\n\t/** Transforms messages before sending to target. Overrides global hook; can be overridden per-case. */\n\tprepareMessages?: (messages: Message[]) => Message[] | Promise<Message[]>;\n\tcases: TestCase[];\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\n/**\n * Converts a `ToolSpec[]` (from a real tool provider) into the\n * `Record<string, ToolDef>` that `defineSuite` expects.\n *\n * `responses` maps tool names to canned mock responses. Tools without an\n * entry in `responses` default to `''`.\n */\nexport function fromToolSpecs(\n\tspecs: ToolSpec[],\n\tresponses: Record<string, ToolDef['response']> = {},\n): Record<string, ToolDef> {\n\treturn Object.fromEntries(\n\t\tspecs.map((spec) => [\n\t\t\tspec.name,\n\t\t\t{\n\t\t\t\tdescription: spec.description,\n\t\t\t\tschema: spec.inputSchema,\n\t\t\t\tresponse: responses[spec.name] ?? '',\n\t\t\t} satisfies ToolDef,\n\t\t]),\n\t);\n}\n\nfunction toMockTools(defs: Record<string, ToolDef>): MockToolDef[] {\n\treturn Object.entries(defs).map(([name, def]) => ({\n\t\tname,\n\t\tdescription: def.description,\n\t\tschema: def.schema ?? {},\n\t\tresponse:\n\t\t\ttypeof def.response === 'function'\n\t\t\t\t? (def.response as MockToolDef['response'])\n\t\t\t\t: typeof def.response === 'string'\n\t\t\t\t\t? def.response\n\t\t\t\t\t: JSON.stringify(def.response),\n\t}));\n}\n\n/** Strip function responses and ZodObjects so the object is JSON-serialisable for langsmith hashing. */\nfunction toSerializableTools(tools: MockToolDef[]): Record<string, unknown>[] {\n\treturn tools.map((t) => ({\n\t\t...t,\n\t\tschema: t.schema instanceof Object && 'shape' in t.schema ? '<ZodObject>' : t.schema,\n\t\tresponse: typeof t.response === 'function' ? '<function>' : t.response,\n\t}));\n}\n\nfunction lastHumanContent(messages: Message[]): string {\n\tfor (let i = messages.length - 1; i >= 0; i--) {\n\t\tconst msg = messages[i];\n\t\tif (msg.role === 'human') {\n\t\t\tconst textBlock = msg.content.find((c) => c.type === 'text');\n\t\t\treturn textBlock ? textBlock.text : '';\n\t\t}\n\t}\n\treturn '';\n}\n\nfunction resolveModelTarget(config: SuiteConfig): TargetFn {\n\tif (typeof config.target === 'function') return config.target;\n\tconst evalConfig = getEvalConfig();\n\tif (!evalConfig.model && typeof config.target !== 'string') {\n\t\tthrow new Error('model is required for model-based target. Add it to your configureEvals() call.');\n\t}\n\tconst model = typeof config.target === 'string' ? config.target : evalConfig.model!;\n\treturn createEvalTarget(evalConfig.modelConfig, model);\n}\n\nfunction resolveCreateTarget(config: SuiteConfig): CreateTargetFn | undefined {\n\treturn config.createTarget ?? getEvalConfig().createTarget;\n}\n\n// ── Suite registry ───────────────────────────────────────────────────\n\ninterface RegisteredSuite {\n\tname: string;\n\tconfig: SuiteConfig;\n}\n\nconst _suites: RegisteredSuite[] = [];\n\n/**\n * Registers an eval suite. Does not create tests on its own — call\n * `runEvals()` after all suites are registered to emit a single\n * LangSmith experiment containing every test case.\n */\nexport function defineSuite(name: string, config: SuiteConfig): void {\n\t_suites.push({ name, config });\n}\n\n/**\n * Emits all registered suites under a single `ls.describe` block so\n * every test case lands in one LangSmith experiment / dataset.\n *\n * Call this once, after importing all suite files.\n *\n * Individual suites are grouped with native `describe` blocks for\n * readability; test names are prefixed with the suite name\n * (e.g. \"discovery > should use search tool\").\n */\nexport function runEvals(): void {\n\tconst evalConfig = getEvalConfig();\n\n\tls.describe(evalConfig.experimentName, () => {\n\t\tfor (const { name: suiteName, config } of _suites) {\n\t\t\tconst suiteTools = config.tools ?? {};\n\t\t\tconst createTarget = config.target ? undefined : resolveCreateTarget(config);\n\n\t\t\tconst categoryLabel = suiteName.charAt(0).toUpperCase() + suiteName.slice(1);\n\t\t\tconst model = typeof config.target === 'string' ? config.target : (evalConfig.model ?? 'agent');\n\n\t\t\tfor (const tc of config.cases) {\n\t\t\t\tconst testName = tc.name ?? lastHumanContent(tc.messages);\n\t\t\t\tconst caseToolDefs = tc.tools ?? suiteTools;\n\t\t\t\tconst tools = toMockTools(caseToolDefs);\n\t\t\t\tconst ctx = { message: lastHumanContent(tc.messages) };\n\n\t\t\t\tconst resolved = tc.expect.map((exp) => exp(ctx));\n\t\t\t\tconst evaluators = resolved.map((r) => r.evaluator);\n\t\t\t\tconst referenceOutputs = Object.assign({}, ...resolved.map((r) => r.referenceOutputs));\n\n\t\t\t\tconst fullTestName = `[${categoryLabel}] > ${testName}`;\n\n\t\t\t\tls.test(\n\t\t\t\t\tfullTestName,\n\t\t\t\t\t{\n\t\t\t\t\t\tinputs: {\n\t\t\t\t\t\t\tname: fullTestName,\n\t\t\t\t\t\t\tcategory: categoryLabel,\n\t\t\t\t\t\t\tmodel,\n\t\t\t\t\t\t\ttools: tools.map((t) => t.name).join(' | ') || 'none',\n\t\t\t\t\t\t\tmessages: tc.messages,\n\t\t\t\t\t\t},\n\t\t\t\t\t\treferenceOutputs,\n\t\t\t\t\t},\n\t\t\t\t\tasync ({ referenceOutputs: refOut }) => {\n\t\t\t\t\t\tlet output: { messages: BaseMessage[] };\n\n\t\t\t\t\t\t// Resolution order: case > suite > global > identity\n\t\t\t\t\t\tconst prepareMessages =\n\t\t\t\t\t\t\ttc.prepareMessages ?? config.prepareMessages ?? getEvalConfig().prepareMessages;\n\t\t\t\t\t\tconst preparedMessages = prepareMessages ? await prepareMessages(tc.messages) : tc.messages;\n\n\t\t\t\t\t\tif (createTarget) {\n\t\t\t\t\t\t\toutput = await runAgentTarget(createTarget, preparedMessages, caseToolDefs);\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tconst target = resolveModelTarget(config);\n\t\t\t\t\t\t\tconst globalPrompt = getEvalConfig().systemPrompt;\n\t\t\t\t\t\t\tconst systemPrompt = tc.systemPrompt ?? config.systemPrompt ?? globalPrompt;\n\t\t\t\t\t\t\toutput = await target({\n\t\t\t\t\t\t\t\tmessages: preparedMessages,\n\t\t\t\t\t\t\t\ttools,\n\t\t\t\t\t\t\t\t...(systemPrompt ? { systemPrompt } : {}),\n\t\t\t\t\t\t\t});\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tconst calledTools = output.messages\n\t\t\t\t\t\t\t.filter((m) => m instanceof AIMessage)\n\t\t\t\t\t\t\t.flatMap((m) => (m as AIMessage).tool_calls ?? [])\n\t\t\t\t\t\t\t.map((tc) => tc.name);\n\n\t\t\t\t\t\tls.logOutputs({\n\t\t\t\t\t\t\ttools_called: calledTools.length > 0 ? calledTools.join(' | ') : 'none',\n\t\t\t\t\t\t});\n\t\t\t\t\t\tfor (const evaluator of evaluators) {\n\t\t\t\t\t\t\tawait evaluator({ outputs: output, referenceOutputs: refOut ?? {} });\n\t\t\t\t\t\t}\n\t\t\t\t\t},\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\t});\n}\n","import { BaseChatModel } from '@langchain/core/language_models/chat_models';\nimport { tool } from '@langchain/core/tools';\nimport { AIMessage, BaseMessage, SystemMessage, ToolMessage } from '@langchain/core/messages';\nimport { z } from 'zod';\nimport { LangchainModelResolver, type LangchainModelConfig } from '../runtime/langchain/model-resolver';\nimport { type CreateTargetFn, getEvalConfig } from './config';\nimport type { AgentResult, Message as AgentMessage, ToolCallContentBlock, ToolDefinition } from '../core/agent.interface';\nimport { convertToLangchainMessages } from '../runtime/langchain/utils';\nimport type { ToolDef } from './suite';\n\nexport interface MockToolDef {\n\tname: string;\n\tdescription: string;\n\tschema: z.ZodObject<any> | Record<string, unknown>;\n\t/**\n\t * Canned response the mock tool returns.\n\t * Can be a static string, or a function that receives input and returns a response.\n\t * If a function is provided, it receives the full invocation count as a second arg\n\t * to support scenarios like \"first call fails, second call succeeds\".\n\t */\n\tresponse: string | ((input: Record<string, unknown>, callCount: number) => string);\n}\n\nexport interface EvalTargetInput {\n\tsystemPrompt?: string;\n\tmessages: AgentMessage[];\n\ttools: MockToolDef[];\n}\n\nconst MAX_AGENT_LOOPS = 10;\n\n/**\n * Creates a LangSmith-compatible target function that runs an agentic loop\n * with mock tools and returns the full message trajectory.\n */\nexport function createEvalTarget(modelConfig?: LangchainModelConfig, modelString?: string) {\n\treturn async (inputs: EvalTargetInput): Promise<{ messages: BaseMessage[] }> => {\n\t\tconst config = modelConfig && modelString ? { modelConfig, model: modelString } : getEvalConfig();\n\t\tif (!config.model) {\n\t\t\tthrow new Error('model is required for model-based target. Add it to your configureEvals() call.');\n\t\t}\n\t\tconst resolver = new LangchainModelResolver(config.modelConfig);\n\t\tconst model = resolver.resolve(config.model) as BaseChatModel;\n\n\t\t// Track invocation counts per tool for stateful mock responses\n\t\tconst toolCallCounts: Record<string, number> = {};\n\n\t\t// Create langchain tools from mock definitions\n\t\tconst langchainTools = inputs.tools.map((mockTool) => {\n\t\t\ttoolCallCounts[mockTool.name] = 0;\n\n\t\t\treturn tool(\n\t\t\t\tasync (toolInput: Record<string, unknown>) => {\n\t\t\t\t\ttoolCallCounts[mockTool.name]++;\n\t\t\t\t\tif (typeof mockTool.response === 'function') {\n\t\t\t\t\t\treturn mockTool.response(toolInput, toolCallCounts[mockTool.name]);\n\t\t\t\t\t}\n\t\t\t\t\treturn mockTool.response;\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tname: mockTool.name,\n\t\t\t\t\tdescription: mockTool.description,\n\t\t\t\t\tschema:\n\t\t\t\t\t\tmockTool.schema instanceof z.ZodObject\n\t\t\t\t\t\t\t? mockTool.schema\n\t\t\t\t\t\t\t: z.object(\n\t\t\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\t\t\tObject.entries(mockTool.schema).map(([key, val]) => {\n\t\t\t\t\t\t\t\t\t\t\tif (typeof val === 'string') return [key, z.string().describe(val)];\n\t\t\t\t\t\t\t\t\t\t\tif (typeof val === 'number') return [key, z.number().describe(String(val))];\n\t\t\t\t\t\t\t\t\t\t\treturn [key, z.any()];\n\t\t\t\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t\t\t\t),\n\t\t\t\t\t\t\t\t),\n\t\t\t\t},\n\t\t\t);\n\t\t});\n\n\t\tconst boundModel = langchainTools.length > 0 ? model.bindTools!(langchainTools) : model;\n\n\t\tconst messages: BaseMessage[] = [];\n\n\t\tif (inputs.systemPrompt) {\n\t\t\tmessages.push(new SystemMessage(inputs.systemPrompt));\n\t\t}\n\n\t\t// Convert and push all messages (history + final human)\n\t\tmessages.push(...convertToLangchainMessages(inputs.messages));\n\n\t\t// Agentic loop: keep calling model until it stops making tool calls\n\t\tlet loopCount = 0;\n\t\twhile (loopCount < MAX_AGENT_LOOPS) {\n\t\t\tloopCount++;\n\n\t\t\tconst response = await boundModel.invoke(messages);\n\t\t\tmessages.push(response as BaseMessage);\n\n\t\t\tconst aiMessage = response as AIMessage;\n\t\t\tif (!aiMessage.tool_calls || aiMessage.tool_calls.length === 0) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\t// Execute tool calls and add results\n\t\t\tfor (const tc of aiMessage.tool_calls) {\n\t\t\t\tconst mockTool = langchainTools.find((t) => t.name === tc.name);\n\t\t\t\tif (mockTool) {\n\t\t\t\t\tconst result = await mockTool.invoke(tc.args);\n\t\t\t\t\tmessages.push(\n\t\t\t\t\t\tnew ToolMessage({\n\t\t\t\t\t\t\tcontent: typeof result === 'string' ? result : JSON.stringify(result),\n\t\t\t\t\t\t\ttool_call_id: tc.id!,\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t} else {\n\t\t\t\t\tmessages.push(\n\t\t\t\t\t\tnew ToolMessage({\n\t\t\t\t\t\t\tcontent: `Tool \"${tc.name}\" not found`,\n\t\t\t\t\t\t\ttool_call_id: tc.id!,\n\t\t\t\t\t\t\tname: tc.name,\n\t\t\t\t\t\t}),\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn { messages };\n\t};\n}\n\n// ── Agent-based target ──────────────────────────────────────────────\n\n/**\n * Converts an `AgentResult` (from `Agent.run()`) into LangChain `BaseMessage[]`\n * so existing evaluators (trajectory match, no-tool-calls, response-content, language) work unchanged.\n *\n * Consecutive `tool_call` content blocks are grouped into a single `AIMessage` with `tool_calls`,\n * followed by one `ToolMessage` per call.\n */\nexport function agentResultToMessages(inputMessages: AgentMessage[], result: AgentResult): BaseMessage[] {\n\t// Include input messages for trajectory context\n\tconst messages: BaseMessage[] = convertToLangchainMessages(inputMessages);\n\n\t// Group content blocks into BaseMessages\n\tlet pendingToolCalls: { id: string; name: string; args: Record<string, unknown>; output: string }[] = [];\n\n\tfor (const block of result.content) {\n\t\tif (block.type === 'tool_call') {\n\t\t\tconst tc = block as ToolCallContentBlock;\n\t\t\tpendingToolCalls.push({\n\t\t\t\tid: tc.toolCallId,\n\t\t\t\tname: tc.name,\n\t\t\t\targs: tc.input ? JSON.parse(tc.input) : {},\n\t\t\t\toutput: tc.output,\n\t\t\t});\n\t\t} else if (block.type === 'text') {\n\t\t\t// Flush any pending tool calls before the text block\n\t\t\tif (pendingToolCalls.length > 0) {\n\t\t\t\tmessages.push(\n\t\t\t\t\tnew AIMessage({\n\t\t\t\t\t\tcontent: '',\n\t\t\t\t\t\ttool_calls: pendingToolCalls.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })),\n\t\t\t\t\t}),\n\t\t\t\t);\n\t\t\t\tfor (const tc of pendingToolCalls) {\n\t\t\t\t\tmessages.push(new ToolMessage({ content: tc.output, tool_call_id: tc.id, name: tc.name }));\n\t\t\t\t}\n\t\t\t\tpendingToolCalls = [];\n\t\t\t}\n\t\t\tmessages.push(new AIMessage(block.output));\n\t\t}\n\t}\n\n\t// Flush remaining tool calls (agent ended mid-tool-use, unlikely but safe)\n\tif (pendingToolCalls.length > 0) {\n\t\tmessages.push(\n\t\t\tnew AIMessage({\n\t\t\t\tcontent: '',\n\t\t\t\ttool_calls: pendingToolCalls.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })),\n\t\t\t}),\n\t\t);\n\t\tfor (const tc of pendingToolCalls) {\n\t\t\tmessages.push(new ToolMessage({ content: tc.output, tool_call_id: tc.id, name: tc.name }));\n\t\t}\n\t}\n\n\treturn messages;\n}\n\n/**\n * Converts eval `Record<string, ToolDef>` into `ToolDefinition[]` with mock `exec` functions,\n * suitable for passing to an `AgentFactory.createAgent()` call.\n */\nexport function toolDefsToDefinitions(defs: Record<string, ToolDef>): ToolDefinition[] {\n\tconst callCounts: Record<string, number> = {};\n\n\treturn Object.entries(defs).map(([name, def]) => {\n\t\tcallCounts[name] = 0;\n\n\t\treturn {\n\t\t\tname,\n\t\t\ttoolKit: 'eval-mock',\n\t\t\tdescription: def.description,\n\t\t\tinputSchema:\n\t\t\t\tdef.schema instanceof z.ZodObject\n\t\t\t\t\t? def.schema\n\t\t\t\t\t: z.object(\n\t\t\t\t\t\t\tObject.fromEntries(\n\t\t\t\t\t\t\t\tObject.entries(def.schema ?? {}).map(([key, val]) => {\n\t\t\t\t\t\t\t\t\tif (typeof val === 'string') return [key, z.string().describe(val)];\n\t\t\t\t\t\t\t\t\treturn [key, z.any()];\n\t\t\t\t\t\t\t\t}),\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t),\n\t\t\texec: async (input: Record<string, unknown>) => {\n\t\t\t\tcallCounts[name]++;\n\t\t\t\tif (typeof def.response === 'function') {\n\t\t\t\t\treturn (def.response as (input: Record<string, unknown>, callCount: number) => string)(\n\t\t\t\t\t\tinput,\n\t\t\t\t\t\tcallCounts[name],\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t\treturn typeof def.response === 'string' ? def.response : JSON.stringify(def.response);\n\t\t\t},\n\t\t} satisfies ToolDefinition;\n\t});\n}\n\n/**\n * Runs a real `Agent` as the eval target. Creates a fresh agent per invocation via the factory,\n * sends human messages, and converts the `AgentResult` to `{ messages: BaseMessage[] }`.\n */\nexport async function runAgentTarget(\n\tcreateTarget: CreateTargetFn,\n\tevalMessages: AgentMessage[],\n\textraToolDefs: Record<string, ToolDef>,\n): Promise<{ messages: BaseMessage[] }> {\n\tconst extraTools = Object.keys(extraToolDefs).length > 0 ? toolDefsToDefinitions(extraToolDefs) : [];\n\tconst agent = await createTarget(extraTools);\n\n\tconst result = await agent.run({\n\t\tthreadId: `eval_${Date.now()}_${Math.random().toString(36).slice(2)}`,\n\t\tmessages: evalMessages,\n\t});\n\n\treturn { messages: agentResultToMessages(evalMessages, result) };\n}\n","import * as ls from 'langsmith/vitest';\nimport { createTrajectoryMatchEvaluator, createTrajectoryLLMAsJudge, TRAJECTORY_ACCURACY_PROMPT } from 'agentevals';\nimport { createLanguageEvaluator } from './evaluators/language';\nimport { createResponseContentEvaluator } from './evaluators/response-content';\nimport { createNoToolCallsEvaluator } from './evaluators/no-tool-calls';\nimport { createAnyToolCalledEvaluator } from './evaluators/any-tool-called';\nimport { getEvalConfig } from './config';\n\n// ── Types ────────────────────────────────────────────────────────────\n\ntype EvaluatorFn = (args: { outputs: Record<string, any>; referenceOutputs: Record<string, any> }) => Promise<any>;\n\ninterface ResolvedExpectation {\n\tevaluator: EvaluatorFn;\n\treferenceOutputs: Record<string, unknown>;\n}\n\n/** A factory that receives test context and returns an evaluator + its referenceOutputs. */\nexport type Expectation = (ctx: { message: string }) => ResolvedExpectation;\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nfunction withTrajectoryGuard(evaluator: any, key: string): EvaluatorFn {\n\treturn async ({ outputs, referenceOutputs }) => {\n\t\tif (!referenceOutputs?.referenceTrajectory) {\n\t\t\treturn { key, score: true, comment: 'No referenceTrajectory specified, skipping' };\n\t\t}\n\t\treturn evaluator({ outputs, referenceOutputs: referenceOutputs.referenceTrajectory });\n\t};\n}\n\nfunction buildTrajectory(message: string, toolNames: string[]): Record<string, unknown>[] {\n\tconst trajectory: Record<string, unknown>[] = [];\n\tlet tcIdx = 0;\n\n\ttrajectory.push({ role: 'user', content: message });\n\n\tfor (const name of toolNames) {\n\t\tconst id = `tc${++tcIdx}`;\n\t\ttrajectory.push({\n\t\t\trole: 'assistant',\n\t\t\tcontent: '',\n\t\t\ttool_calls: [{ function: { name, arguments: '{}' }, id, type: 'function' }],\n\t\t});\n\t\ttrajectory.push({ role: 'tool', content: '...', tool_call_id: id });\n\t}\n\n\ttrajectory.push({ role: 'assistant', content: '...' });\n\n\treturn trajectory;\n}\n\n// ── Expectation functions ────────────────────────────────────────────\n\n/**\n * Expect the agent to call tools in order (superset trajectory match).\n * Empty `[]` means the agent should answer directly without calling any tools.\n */\nexport function toolsCalled(tools: string[]): Expectation {\n\treturn (ctx) => ({\n\t\tevaluator: ls.wrapEvaluator(\n\t\t\twithTrajectoryGuard(\n\t\t\t\tcreateTrajectoryMatchEvaluator({ trajectoryMatchMode: 'superset', toolArgsMatchMode: 'ignore' }) as any,\n\t\t\t\t'trajectory_match',\n\t\t\t),\n\t\t),\n\t\treferenceOutputs: { referenceTrajectory: buildTrajectory(ctx.message, tools) },\n\t});\n}\n\n/**\n * Run an LLM-as-judge evaluator on the trajectory.\n * Requires `toolsCalled` in the same expect array.\n * Uses the globally configured evaluator model.\n */\nexport function llmJudge(): Expectation {\n\treturn () => {\n\t\tconst config = getEvalConfig();\n\t\tconst model = config.evaluatorModel;\n\t\treturn {\n\t\t\tevaluator: ls.wrapEvaluator(\n\t\t\t\twithTrajectoryGuard(\n\t\t\t\t\tcreateTrajectoryLLMAsJudge({ prompt: TRAJECTORY_ACCURACY_PROMPT, model }) as any,\n\t\t\t\t\t'trajectory_llm_judge',\n\t\t\t\t),\n\t\t\t),\n\t\t\treferenceOutputs: {},\n\t\t};\n\t};\n}\n\n/**\n * Assert the agent made zero tool calls.\n * Optionally allow specific tools via `except` — calls to those tools\n * are permitted (but not required), while any other tool call fails.\n */\nexport function noTools(options?: { except: string[] }): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createNoToolCallsEvaluator()),\n\t\treferenceOutputs: {\n\t\t\texpectNoToolCalls: true,\n\t\t\t...(options?.except?.length ? { exceptTools: options.except } : {}),\n\t\t},\n\t});\n}\n\n/**\n * Assert the response is in the given language (ISO 639-1 code).\n * Uses the globally configured evaluator model for language detection.\n * @param code - ISO 639-1 language code (e.g. 'en', 'tr', 'de').\n */\nexport function respondsInLanguage(code: string): Expectation {\n\treturn () => {\n\t\tconst config = getEvalConfig();\n\t\tconst model = config.evaluatorModel;\n\t\treturn {\n\t\t\tevaluator: ls.wrapEvaluator(createLanguageEvaluator(config.modelConfig, model)),\n\t\t\treferenceOutputs: { expectedLanguage: code },\n\t\t};\n\t};\n}\n\n/**\n * Assert that at least one tool call was made.\n * When `tools` is provided, at least one of those specific tools must\n * appear in the trajectory. When omitted, any tool call satisfies it.\n */\nexport function anyToolCalled(tools?: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createAnyToolCalledEvaluator()),\n\t\treferenceOutputs: {\n\t\t\texpectAnyToolCall: true,\n\t\t\t...(tools?.length ? { anyToolsExpected: tools } : {}),\n\t\t},\n\t});\n}\n\n/** Assert the response contains all given strings. */\nexport function contains(strings: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createResponseContentEvaluator()),\n\t\treferenceOutputs: { responseContains: strings },\n\t});\n}\n\n/** Assert the response does not contain any of the given strings. */\nexport function notContains(strings: string[]): Expectation {\n\treturn () => ({\n\t\tevaluator: ls.wrapEvaluator(createResponseContentEvaluator()),\n\t\treferenceOutputs: { responseMustNotContain: strings },\n\t});\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\nimport { BaseChatModel } from '@langchain/core/language_models/chat_models';\nimport { LangchainModelResolver, type LangchainModelConfig } from '../../runtime/langchain/model-resolver';\n\n/**\n * Creates a custom evaluator that checks whether the agent's final response\n * is in the expected language. Uses a cheap LLM call for language detection.\n */\nexport function createLanguageEvaluator(modelConfig: LangchainModelConfig, model: string) {\n\tconst resolver = new LangchainModelResolver(modelConfig);\n\tconst judge = resolver.resolve(model) as BaseChatModel;\n\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tconst expectedLanguage = referenceOutputs?.expectedLanguage;\n\t\tif (!expectedLanguage) {\n\t\t\treturn { key: 'language_match', score: true, comment: 'No expected language specified, skipping' };\n\t\t}\n\n\t\t// Extract the last AI message text from the trajectory\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage);\n\n\t\tif (!lastAiMessage) {\n\t\t\treturn { key: 'language_match', score: false, comment: 'No AI message found in trajectory' };\n\t\t}\n\n\t\tconst responseText = typeof lastAiMessage.content === 'string' ? lastAiMessage.content : JSON.stringify(lastAiMessage.content);\n\n\t\t// Use LLM to detect the language\n\t\tconst detection = await judge.invoke([\n\t\t\t{\n\t\t\t\trole: 'system',\n\t\t\t\tcontent: 'You are a language detection tool. Respond with ONLY the ISO 639-1 language code (e.g., \"en\", \"tr\", \"de\", \"fr\") of the text provided. Nothing else.',\n\t\t\t},\n\t\t\t{\n\t\t\t\trole: 'user',\n\t\t\t\tcontent: responseText,\n\t\t\t},\n\t\t]);\n\n\t\tconst detectedLanguage = (typeof detection.content === 'string' ? detection.content : '').trim().toLowerCase();\n\n\t\tconst matches = detectedLanguage === expectedLanguage.toLowerCase();\n\n\t\treturn {\n\t\t\tkey: 'language_match',\n\t\t\tscore: matches,\n\t\t\tcomment: matches\n\t\t\t\t? `Response language matches expected: ${expectedLanguage}`\n\t\t\t\t: `Expected \"${expectedLanguage}\" but detected \"${detectedLanguage}\"`,\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that checks whether the agent's final response\n * contains expected strings and doesn't contain forbidden strings.\n */\nexport function createResponseContentEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tconst mustContain: string[] = referenceOutputs?.responseContains || [];\n\t\tconst mustNotContain: string[] = referenceOutputs?.responseMustNotContain || [];\n\n\t\tif (mustContain.length === 0 && mustNotContain.length === 0) {\n\t\t\treturn { key: 'response_content', score: true, comment: 'No content assertions specified, skipping' };\n\t\t}\n\n\t\t// Extract the last AI message text from the trajectory\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst lastAiMessage = [...messages].reverse().find((m) => m instanceof AIMessage);\n\n\t\tif (!lastAiMessage) {\n\t\t\treturn { key: 'response_content', score: false, comment: 'No AI message found in trajectory' };\n\t\t}\n\n\t\tconst responseText = (typeof lastAiMessage.content === 'string' ? lastAiMessage.content : JSON.stringify(lastAiMessage.content)).toLowerCase();\n\n\t\tconst failures: string[] = [];\n\n\t\tfor (const expected of mustContain) {\n\t\t\tif (!responseText.includes(expected.toLowerCase())) {\n\t\t\t\tfailures.push(`Missing expected text: \"${expected}\"`);\n\t\t\t}\n\t\t}\n\n\t\tfor (const forbidden of mustNotContain) {\n\t\t\tif (responseText.includes(forbidden.toLowerCase())) {\n\t\t\t\tfailures.push(`Contains forbidden text: \"${forbidden}\"`);\n\t\t\t}\n\t\t}\n\n\t\tconst passed = failures.length === 0;\n\n\t\treturn {\n\t\t\tkey: 'response_content',\n\t\t\tscore: passed,\n\t\t\tcomment: passed ? 'All content assertions passed' : failures.join('; '),\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that asserts the agent made zero tool calls.\n * When `referenceOutputs.exceptTools` is set, calls to those tools are\n * allowed (but not required) — only calls to non-excepted tools cause failure.\n */\nexport function createNoToolCallsEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\t// Only run this evaluator if the reference explicitly expects no tool calls\n\t\tif (referenceOutputs?.maxToolCalls !== 0 && referenceOutputs?.expectNoToolCalls !== true) {\n\t\t\treturn { key: 'no_tool_calls', score: true, comment: 'No tool call restriction specified, skipping' };\n\t\t}\n\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\t\tconst exceptTools: string[] = referenceOutputs?.exceptTools ?? [];\n\n\t\tconst toolCalls = messages\n\t\t\t.filter((m) => m instanceof AIMessage)\n\t\t\t.flatMap((m) => (m as AIMessage).tool_calls || []);\n\n\t\tconst disallowedCalls = exceptTools.length > 0 ? toolCalls.filter((tc) => !exceptTools.includes(tc.name)) : toolCalls;\n\n\t\tconst passed = disallowedCalls.length === 0;\n\n\t\tif (exceptTools.length > 0) {\n\t\t\treturn {\n\t\t\t\tkey: 'no_tool_calls',\n\t\t\t\tscore: passed,\n\t\t\t\tcomment: passed\n\t\t\t\t\t? `No disallowed tool calls made (allowed: ${exceptTools.join(', ')})`\n\t\t\t\t\t: `Agent made ${disallowedCalls.length} disallowed tool call(s): ${disallowedCalls.map((tc) => tc.name).join(', ')}`,\n\t\t\t};\n\t\t}\n\n\t\treturn {\n\t\t\tkey: 'no_tool_calls',\n\t\t\tscore: passed,\n\t\t\tcomment: passed\n\t\t\t\t? 'No tool calls made (as expected)'\n\t\t\t\t: `Agent made ${toolCalls.length} tool call(s): ${toolCalls.map((tc) => tc.name).join(', ')}`,\n\t\t};\n\t};\n}\n","import { AIMessage, BaseMessage } from '@langchain/core/messages';\n\n/**\n * Creates a custom evaluator that asserts at least one tool was called.\n * When `referenceOutputs.anyToolsExpected` contains tool names, at least\n * one of those specific tools must appear. When the list is empty, any\n * tool call satisfies the expectation.\n */\nexport function createAnyToolCalledEvaluator() {\n\treturn async ({\n\t\toutputs,\n\t\treferenceOutputs,\n\t}: {\n\t\toutputs: Record<string, any>;\n\t\treferenceOutputs?: Record<string, any>;\n\t}) => {\n\t\tif (referenceOutputs?.expectAnyToolCall !== true) {\n\t\t\treturn { key: 'any_tool_called', score: true, comment: 'No any-tool-call expectation specified, skipping' };\n\t\t}\n\n\t\tconst expectedTools: string[] = referenceOutputs?.anyToolsExpected ?? [];\n\t\tconst messages: BaseMessage[] = outputs.messages || [];\n\n\t\tconst calledToolNames = messages\n\t\t\t.filter((m) => m instanceof AIMessage)\n\t\t\t.flatMap((m) => (m as AIMessage).tool_calls || [])\n\t\t\t.map((tc) => tc.name);\n\n\t\t// No specific tools requested — any tool call satisfies the expectation\n\t\tif (expectedTools.length === 0) {\n\t\t\tconst passed = calledToolNames.length > 0;\n\t\t\treturn {\n\t\t\t\tkey: 'any_tool_called',\n\t\t\t\tscore: passed,\n\t\t\t\tcomment: passed\n\t\t\t\t\t? `Agent called tool(s): ${calledToolNames.join(', ')}`\n\t\t\t\t\t: 'Agent made no tool calls (expected at least one)',\n\t\t\t};\n\t\t}\n\n\t\tconst matchedTools = expectedTools.filter((name) => calledToolNames.includes(name));\n\t\tconst passed = matchedTools.length > 0;\n\n\t\treturn {\n\t\t\tkey: 'any_tool_called',\n\t\t\tscore: passed,\n\t\t\tcomment: passed\n\t\t\t\t? `Called expected tool(s): ${matchedTools.join(', ')}`\n\t\t\t\t: `None of the expected tools were called (expected one of: ${expectedTools.join(', ')}; actual: ${calledToolNames.length > 0 ? calledToolNames.join(', ') : 'none'})`,\n\t\t};\n\t};\n}\n"],"mappings":";;;;;;AAuBA,IAAI,UAA6B;AAE1B,SAAS,eAAe,QAA0B;AACxD,YAAU;AACX;AAEO,SAAS,gBAA4B;AAC3C,MAAI,CAAC,SAAS;AACb,UAAM,IAAI,MAAM,wEAAwE;AAAA,EACzF;AACA,SAAO;AACR;;;AClCA,YAAY,QAAQ;AACpB,SAAS,aAAAA,kBAA8B;;;ACAvC,SAAS,YAAY;AACrB,SAAS,WAAwB,eAAe,mBAAmB;AACnE,SAAS,SAAS;AA0BlB,IAAM,kBAAkB;AAMjB,SAAS,iBAAiB,aAAoC,aAAsB;AAC1F,SAAO,OAAO,WAAkE;AAC/E,UAAM,SAAS,eAAe,cAAc,EAAE,aAAa,OAAO,YAAY,IAAI,cAAc;AAChG,QAAI,CAAC,OAAO,OAAO;AAClB,YAAM,IAAI,MAAM,iFAAiF;AAAA,IAClG;AACA,UAAM,WAAW,IAAI,uBAAuB,OAAO,WAAW;AAC9D,UAAM,QAAQ,SAAS,QAAQ,OAAO,KAAK;AAG3C,UAAM,iBAAyC,CAAC;AAGhD,UAAM,iBAAiB,OAAO,MAAM,IAAI,CAAC,aAAa;AACrD,qBAAe,SAAS,IAAI,IAAI;AAEhC,aAAO;AAAA,QACN,OAAO,cAAuC;AAC7C,yBAAe,SAAS,IAAI;AAC5B,cAAI,OAAO,SAAS,aAAa,YAAY;AAC5C,mBAAO,SAAS,SAAS,WAAW,eAAe,SAAS,IAAI,CAAC;AAAA,UAClE;AACA,iBAAO,SAAS;AAAA,QACjB;AAAA,QACA;AAAA,UACC,MAAM,SAAS;AAAA,UACf,aAAa,SAAS;AAAA,UACtB,QACC,SAAS,kBAAkB,EAAE,YAC1B,SAAS,SACT,EAAE;AAAA,YACF,OAAO;AAAA,cACN,OAAO,QAAQ,SAAS,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM;AACnD,oBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,GAAG,CAAC;AAClE,oBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,OAAO,GAAG,CAAC,CAAC;AAC1E,uBAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAAA,cACrB,CAAC;AAAA,YACF;AAAA,UACD;AAAA,QACJ;AAAA,MACD;AAAA,IACD,CAAC;AAED,UAAM,aAAa,eAAe,SAAS,IAAI,MAAM,UAAW,cAAc,IAAI;AAElF,UAAM,WAA0B,CAAC;AAEjC,QAAI,OAAO,cAAc;AACxB,eAAS,KAAK,IAAI,cAAc,OAAO,YAAY,CAAC;AAAA,IACrD;AAGA,aAAS,KAAK,GAAG,2BAA2B,OAAO,QAAQ,CAAC;AAG5D,QAAI,YAAY;AAChB,WAAO,YAAY,iBAAiB;AACnC;AAEA,YAAM,WAAW,MAAM,WAAW,OAAO,QAAQ;AACjD,eAAS,KAAK,QAAuB;AAErC,YAAM,YAAY;AAClB,UAAI,CAAC,UAAU,cAAc,UAAU,WAAW,WAAW,GAAG;AAC/D;AAAA,MACD;AAGA,iBAAW,MAAM,UAAU,YAAY;AACtC,cAAM,WAAW,eAAe,KAAK,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI;AAC9D,YAAI,UAAU;AACb,gBAAM,SAAS,MAAM,SAAS,OAAO,GAAG,IAAI;AAC5C,mBAAS;AAAA,YACR,IAAI,YAAY;AAAA,cACf,SAAS,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,cACpE,cAAc,GAAG;AAAA,cACjB,MAAM,GAAG;AAAA,YACV,CAAC;AAAA,UACF;AAAA,QACD,OAAO;AACN,mBAAS;AAAA,YACR,IAAI,YAAY;AAAA,cACf,SAAS,SAAS,GAAG,IAAI;AAAA,cACzB,cAAc,GAAG;AAAA,cACjB,MAAM,GAAG;AAAA,YACV,CAAC;AAAA,UACF;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAEA,WAAO,EAAE,SAAS;AAAA,EACnB;AACD;AAWO,SAAS,sBAAsB,eAA+B,QAAoC;AAExG,QAAM,WAA0B,2BAA2B,aAAa;AAGxE,MAAI,mBAAkG,CAAC;AAEvG,aAAW,SAAS,OAAO,SAAS;AACnC,QAAI,MAAM,SAAS,aAAa;AAC/B,YAAM,KAAK;AACX,uBAAiB,KAAK;AAAA,QACrB,IAAI,GAAG;AAAA,QACP,MAAM,GAAG;AAAA,QACT,MAAM,GAAG,QAAQ,KAAK,MAAM,GAAG,KAAK,IAAI,CAAC;AAAA,QACzC,QAAQ,GAAG;AAAA,MACZ,CAAC;AAAA,IACF,WAAW,MAAM,SAAS,QAAQ;AAEjC,UAAI,iBAAiB,SAAS,GAAG;AAChC,iBAAS;AAAA,UACR,IAAI,UAAU;AAAA,YACb,SAAS;AAAA,YACT,YAAY,iBAAiB,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,MAAM,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE;AAAA,UACvF,CAAC;AAAA,QACF;AACA,mBAAW,MAAM,kBAAkB;AAClC,mBAAS,KAAK,IAAI,YAAY,EAAE,SAAS,GAAG,QAAQ,cAAc,GAAG,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,QAC1F;AACA,2BAAmB,CAAC;AAAA,MACrB;AACA,eAAS,KAAK,IAAI,UAAU,MAAM,MAAM,CAAC;AAAA,IAC1C;AAAA,EACD;AAGA,MAAI,iBAAiB,SAAS,GAAG;AAChC,aAAS;AAAA,MACR,IAAI,UAAU;AAAA,QACb,SAAS;AAAA,QACT,YAAY,iBAAiB,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,MAAM,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE;AAAA,MACvF,CAAC;AAAA,IACF;AACA,eAAW,MAAM,kBAAkB;AAClC,eAAS,KAAK,IAAI,YAAY,EAAE,SAAS,GAAG,QAAQ,cAAc,GAAG,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC;AAAA,IAC1F;AAAA,EACD;AAEA,SAAO;AACR;AAMO,SAAS,sBAAsB,MAAiD;AACtF,QAAM,aAAqC,CAAC;AAE5C,SAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM;AAChD,eAAW,IAAI,IAAI;AAEnB,WAAO;AAAA,MACN;AAAA,MACA,SAAS;AAAA,MACT,aAAa,IAAI;AAAA,MACjB,aACC,IAAI,kBAAkB,EAAE,YACrB,IAAI,SACJ,EAAE;AAAA,QACF,OAAO;AAAA,UACN,OAAO,QAAQ,IAAI,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM;AACpD,gBAAI,OAAO,QAAQ,SAAU,QAAO,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,GAAG,CAAC;AAClE,mBAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAAA,UACrB,CAAC;AAAA,QACF;AAAA,MACD;AAAA,MACH,MAAM,OAAO,UAAmC;AAC/C,mBAAW,IAAI;AACf,YAAI,OAAO,IAAI,aAAa,YAAY;AACvC,iBAAQ,IAAI;AAAA,YACX;AAAA,YACA,WAAW,IAAI;AAAA,UAChB;AAAA,QACD;AACA,eAAO,OAAO,IAAI,aAAa,WAAW,IAAI,WAAW,KAAK,UAAU,IAAI,QAAQ;AAAA,MACrF;AAAA,IACD;AAAA,EACD,CAAC;AACF;AAMA,eAAsB,eACrB,cACA,cACA,eACuC;AACvC,QAAM,aAAa,OAAO,KAAK,aAAa,EAAE,SAAS,IAAI,sBAAsB,aAAa,IAAI,CAAC;AACnG,QAAM,QAAQ,MAAM,aAAa,UAAU;AAE3C,QAAM,SAAS,MAAM,MAAM,IAAI;AAAA,IAC9B,UAAU,QAAQ,KAAK,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;AAAA,IACnE,UAAU;AAAA,EACX,CAAC;AAED,SAAO,EAAE,UAAU,sBAAsB,cAAc,MAAM,EAAE;AAChE;;;ADvOO,SAAS,MAAM,SAA+B;AACpD,SAAO,EAAE,MAAM,SAAS,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,QAAQ,CAAC,EAAE;AACpE;AAEO,SAAS,GAAG,SAAiB,WAAiC;AACpE,SAAO,EAAE,MAAM,MAAM,SAAS,GAAI,YAAY,EAAE,WAAW,UAAU,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,IAAI,CAAC,EAAG;AACxG;AAEO,SAAS,WAAW,MAAc,QAA6B;AACrE,SAAO,EAAE,MAAM,QAAQ,MAAM,OAAO;AACrC;AAkDO,SAAS,cACf,OACA,YAAiD,CAAC,GACxB;AAC1B,SAAO,OAAO;AAAA,IACb,MAAM,IAAI,CAAC,SAAS;AAAA,MACnB,KAAK;AAAA,MACL;AAAA,QACC,aAAa,KAAK;AAAA,QAClB,QAAQ,KAAK;AAAA,QACb,UAAU,UAAU,KAAK,IAAI,KAAK;AAAA,MACnC;AAAA,IACD,CAAC;AAAA,EACF;AACD;AAEA,SAAS,YAAY,MAA8C;AAClE,SAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO;AAAA,IACjD;AAAA,IACA,aAAa,IAAI;AAAA,IACjB,QAAQ,IAAI,UAAU,CAAC;AAAA,IACvB,UACC,OAAO,IAAI,aAAa,aACpB,IAAI,WACL,OAAO,IAAI,aAAa,WACvB,IAAI,WACJ,KAAK,UAAU,IAAI,QAAQ;AAAA,EACjC,EAAE;AACH;AAWA,SAAS,iBAAiB,UAA6B;AACtD,WAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC9C,UAAM,MAAM,SAAS,CAAC;AACtB,QAAI,IAAI,SAAS,SAAS;AACzB,YAAM,YAAY,IAAI,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM;AAC3D,aAAO,YAAY,UAAU,OAAO;AAAA,IACrC;AAAA,EACD;AACA,SAAO;AACR;AAEA,SAAS,mBAAmB,QAA+B;AAC1D,MAAI,OAAO,OAAO,WAAW,WAAY,QAAO,OAAO;AACvD,QAAM,aAAa,cAAc;AACjC,MAAI,CAAC,WAAW,SAAS,OAAO,OAAO,WAAW,UAAU;AAC3D,UAAM,IAAI,MAAM,iFAAiF;AAAA,EAClG;AACA,QAAM,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAS,WAAW;AAC7E,SAAO,iBAAiB,WAAW,aAAa,KAAK;AACtD;AAEA,SAAS,oBAAoB,QAAiD;AAC7E,SAAO,OAAO,gBAAgB,cAAc,EAAE;AAC/C;AASA,IAAM,UAA6B,CAAC;AAO7B,SAAS,YAAY,MAAc,QAA2B;AACpE,UAAQ,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B;AAYO,SAAS,WAAiB;AAChC,QAAM,aAAa,cAAc;AAEjC,EAAG,YAAS,WAAW,gBAAgB,MAAM;AAC5C,eAAW,EAAE,MAAM,WAAW,OAAO,KAAK,SAAS;AAClD,YAAM,aAAa,OAAO,SAAS,CAAC;AACpC,YAAM,eAAe,OAAO,SAAS,SAAY,oBAAoB,MAAM;AAE3E,YAAM,gBAAgB,UAAU,OAAO,CAAC,EAAE,YAAY,IAAI,UAAU,MAAM,CAAC;AAC3E,YAAM,QAAQ,OAAO,OAAO,WAAW,WAAW,OAAO,SAAU,WAAW,SAAS;AAEvF,iBAAW,MAAM,OAAO,OAAO;AAC9B,cAAM,WAAW,GAAG,QAAQ,iBAAiB,GAAG,QAAQ;AACxD,cAAM,eAAe,GAAG,SAAS;AACjC,cAAM,QAAQ,YAAY,YAAY;AACtC,cAAM,MAAM,EAAE,SAAS,iBAAiB,GAAG,QAAQ,EAAE;AAErD,cAAM,WAAW,GAAG,OAAO,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC;AAChD,cAAM,aAAa,SAAS,IAAI,CAAC,MAAM,EAAE,SAAS;AAClD,cAAM,mBAAmB,OAAO,OAAO,CAAC,GAAG,GAAG,SAAS,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;AAErF,cAAM,eAAe,IAAI,aAAa,OAAO,QAAQ;AAErD,QAAG;AAAA,UACF;AAAA,UACA;AAAA,YACC,QAAQ;AAAA,cACP,MAAM;AAAA,cACN,UAAU;AAAA,cACV;AAAA,cACA,OAAO,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,KAAK,KAAK;AAAA,cAC/C,UAAU,GAAG;AAAA,YACd;AAAA,YACA;AAAA,UACD;AAAA,UACA,OAAO,EAAE,kBAAkB,OAAO,MAAM;AACvC,gBAAI;AAGJ,kBAAM,kBACL,GAAG,mBAAmB,OAAO,mBAAmB,cAAc,EAAE;AACjE,kBAAM,mBAAmB,kBAAkB,MAAM,gBAAgB,GAAG,QAAQ,IAAI,GAAG;AAEnF,gBAAI,cAAc;AACjB,uBAAS,MAAM,eAAe,cAAc,kBAAkB,YAAY;AAAA,YAC3E,OAAO;AACN,oBAAM,SAAS,mBAAmB,MAAM;AACxC,oBAAM,eAAe,cAAc,EAAE;AACrC,oBAAM,eAAe,GAAG,gBAAgB,OAAO,gBAAgB;AAC/D,uBAAS,MAAM,OAAO;AAAA,gBACrB,UAAU;AAAA,gBACV;AAAA,gBACA,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;AAAA,cACxC,CAAC;AAAA,YACF;AAEA,kBAAM,cAAc,OAAO,SACzB,OAAO,CAAC,MAAM,aAAaC,UAAS,EACpC,QAAQ,CAAC,MAAO,EAAgB,cAAc,CAAC,CAAC,EAChD,IAAI,CAACC,QAAOA,IAAG,IAAI;AAErB,YAAG,cAAW;AAAA,cACb,cAAc,YAAY,SAAS,IAAI,YAAY,KAAK,KAAK,IAAI;AAAA,YAClE,CAAC;AACD,uBAAW,aAAa,YAAY;AACnC,oBAAM,UAAU,EAAE,SAAS,QAAQ,kBAAkB,UAAU,CAAC,EAAE,CAAC;AAAA,YACpE;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAAA,EACD,CAAC;AACF;;;AE/OA,YAAYC,SAAQ;AACpB,SAAS,gCAAgC,4BAA4B,kCAAkC;;;ACDvG,SAAS,aAAAC,kBAA8B;AAQhC,SAAS,wBAAwB,aAAmC,OAAe;AACzF,QAAM,WAAW,IAAI,uBAAuB,WAAW;AACvD,QAAM,QAAQ,SAAS,QAAQ,KAAK;AAEpC,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,UAAM,mBAAmB,kBAAkB;AAC3C,QAAI,CAAC,kBAAkB;AACtB,aAAO,EAAE,KAAK,kBAAkB,OAAO,MAAM,SAAS,2CAA2C;AAAA,IAClG;AAGA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,gBAAgB,CAAC,GAAG,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,aAAaC,UAAS;AAEhF,QAAI,CAAC,eAAe;AACnB,aAAO,EAAE,KAAK,kBAAkB,OAAO,OAAO,SAAS,oCAAoC;AAAA,IAC5F;AAEA,UAAM,eAAe,OAAO,cAAc,YAAY,WAAW,cAAc,UAAU,KAAK,UAAU,cAAc,OAAO;AAG7H,UAAM,YAAY,MAAM,MAAM,OAAO;AAAA,MACpC;AAAA,QACC,MAAM;AAAA,QACN,SAAS;AAAA,MACV;AAAA,MACA;AAAA,QACC,MAAM;AAAA,QACN,SAAS;AAAA,MACV;AAAA,IACD,CAAC;AAED,UAAM,oBAAoB,OAAO,UAAU,YAAY,WAAW,UAAU,UAAU,IAAI,KAAK,EAAE,YAAY;AAE7G,UAAM,UAAU,qBAAqB,iBAAiB,YAAY;AAElE,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,UACN,uCAAuC,gBAAgB,KACvD,aAAa,gBAAgB,mBAAmB,gBAAgB;AAAA,IACpE;AAAA,EACD;AACD;;;AC1DA,SAAS,aAAAC,kBAA8B;AAMhC,SAAS,iCAAiC;AAChD,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,UAAM,cAAwB,kBAAkB,oBAAoB,CAAC;AACrE,UAAM,iBAA2B,kBAAkB,0BAA0B,CAAC;AAE9E,QAAI,YAAY,WAAW,KAAK,eAAe,WAAW,GAAG;AAC5D,aAAO,EAAE,KAAK,oBAAoB,OAAO,MAAM,SAAS,4CAA4C;AAAA,IACrG;AAGA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,gBAAgB,CAAC,GAAG,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,aAAaA,UAAS;AAEhF,QAAI,CAAC,eAAe;AACnB,aAAO,EAAE,KAAK,oBAAoB,OAAO,OAAO,SAAS,oCAAoC;AAAA,IAC9F;AAEA,UAAM,gBAAgB,OAAO,cAAc,YAAY,WAAW,cAAc,UAAU,KAAK,UAAU,cAAc,OAAO,GAAG,YAAY;AAE7I,UAAM,WAAqB,CAAC;AAE5B,eAAW,YAAY,aAAa;AACnC,UAAI,CAAC,aAAa,SAAS,SAAS,YAAY,CAAC,GAAG;AACnD,iBAAS,KAAK,2BAA2B,QAAQ,GAAG;AAAA,MACrD;AAAA,IACD;AAEA,eAAW,aAAa,gBAAgB;AACvC,UAAI,aAAa,SAAS,UAAU,YAAY,CAAC,GAAG;AACnD,iBAAS,KAAK,6BAA6B,SAAS,GAAG;AAAA,MACxD;AAAA,IACD;AAEA,UAAM,SAAS,SAAS,WAAW;AAEnC,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SAAS,kCAAkC,SAAS,KAAK,IAAI;AAAA,IACvE;AAAA,EACD;AACD;;;ACrDA,SAAS,aAAAC,kBAA8B;AAOhC,SAAS,6BAA6B;AAC5C,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AAEL,QAAI,kBAAkB,iBAAiB,KAAK,kBAAkB,sBAAsB,MAAM;AACzF,aAAO,EAAE,KAAK,iBAAiB,OAAO,MAAM,SAAS,+CAA+C;AAAA,IACrG;AAEA,UAAM,WAA0B,QAAQ,YAAY,CAAC;AACrD,UAAM,cAAwB,kBAAkB,eAAe,CAAC;AAEhE,UAAM,YAAY,SAChB,OAAO,CAAC,MAAM,aAAaA,UAAS,EACpC,QAAQ,CAAC,MAAO,EAAgB,cAAc,CAAC,CAAC;AAElD,UAAM,kBAAkB,YAAY,SAAS,IAAI,UAAU,OAAO,CAAC,OAAO,CAAC,YAAY,SAAS,GAAG,IAAI,CAAC,IAAI;AAE5G,UAAM,SAAS,gBAAgB,WAAW;AAE1C,QAAI,YAAY,SAAS,GAAG;AAC3B,aAAO;AAAA,QACN,KAAK;AAAA,QACL,OAAO;AAAA,QACP,SAAS,SACN,2CAA2C,YAAY,KAAK,IAAI,CAAC,MACjE,cAAc,gBAAgB,MAAM,6BAA6B,gBAAgB,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,MACpH;AAAA,IACD;AAEA,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SACN,qCACA,cAAc,UAAU,MAAM,kBAAkB,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,IAC7F;AAAA,EACD;AACD;;;ACjDA,SAAS,aAAAC,kBAA8B;AAQhC,SAAS,+BAA+B;AAC9C,SAAO,OAAO;AAAA,IACb;AAAA,IACA;AAAA,EACD,MAGM;AACL,QAAI,kBAAkB,sBAAsB,MAAM;AACjD,aAAO,EAAE,KAAK,mBAAmB,OAAO,MAAM,SAAS,mDAAmD;AAAA,IAC3G;AAEA,UAAM,gBAA0B,kBAAkB,oBAAoB,CAAC;AACvE,UAAM,WAA0B,QAAQ,YAAY,CAAC;AAErD,UAAM,kBAAkB,SACtB,OAAO,CAAC,MAAM,aAAaA,UAAS,EACpC,QAAQ,CAAC,MAAO,EAAgB,cAAc,CAAC,CAAC,EAChD,IAAI,CAAC,OAAO,GAAG,IAAI;AAGrB,QAAI,cAAc,WAAW,GAAG;AAC/B,YAAMC,UAAS,gBAAgB,SAAS;AACxC,aAAO;AAAA,QACN,KAAK;AAAA,QACL,OAAOA;AAAA,QACP,SAASA,UACN,yBAAyB,gBAAgB,KAAK,IAAI,CAAC,KACnD;AAAA,MACJ;AAAA,IACD;AAEA,UAAM,eAAe,cAAc,OAAO,CAAC,SAAS,gBAAgB,SAAS,IAAI,CAAC;AAClF,UAAM,SAAS,aAAa,SAAS;AAErC,WAAO;AAAA,MACN,KAAK;AAAA,MACL,OAAO;AAAA,MACP,SAAS,SACN,4BAA4B,aAAa,KAAK,IAAI,CAAC,KACnD,4DAA4D,cAAc,KAAK,IAAI,CAAC,aAAa,gBAAgB,SAAS,IAAI,gBAAgB,KAAK,IAAI,IAAI,MAAM;AAAA,IACrK;AAAA,EACD;AACD;;;AJ7BA,SAAS,oBAAoB,WAAgB,KAA0B;AACtE,SAAO,OAAO,EAAE,SAAS,iBAAiB,MAAM;AAC/C,QAAI,CAAC,kBAAkB,qBAAqB;AAC3C,aAAO,EAAE,KAAK,OAAO,MAAM,SAAS,6CAA6C;AAAA,IAClF;AACA,WAAO,UAAU,EAAE,SAAS,kBAAkB,iBAAiB,oBAAoB,CAAC;AAAA,EACrF;AACD;AAEA,SAAS,gBAAgB,SAAiB,WAAgD;AACzF,QAAM,aAAwC,CAAC;AAC/C,MAAI,QAAQ;AAEZ,aAAW,KAAK,EAAE,MAAM,QAAQ,SAAS,QAAQ,CAAC;AAElD,aAAW,QAAQ,WAAW;AAC7B,UAAM,KAAK,KAAK,EAAE,KAAK;AACvB,eAAW,KAAK;AAAA,MACf,MAAM;AAAA,MACN,SAAS;AAAA,MACT,YAAY,CAAC,EAAE,UAAU,EAAE,MAAM,WAAW,KAAK,GAAG,IAAI,MAAM,WAAW,CAAC;AAAA,IAC3E,CAAC;AACD,eAAW,KAAK,EAAE,MAAM,QAAQ,SAAS,OAAO,cAAc,GAAG,CAAC;AAAA,EACnE;AAEA,aAAW,KAAK,EAAE,MAAM,aAAa,SAAS,MAAM,CAAC;AAErD,SAAO;AACR;AAQO,SAAS,YAAY,OAA8B;AACzD,SAAO,CAAC,SAAS;AAAA,IAChB,WAAc;AAAA,MACb;AAAA,QACC,+BAA+B,EAAE,qBAAqB,YAAY,mBAAmB,SAAS,CAAC;AAAA,QAC/F;AAAA,MACD;AAAA,IACD;AAAA,IACA,kBAAkB,EAAE,qBAAqB,gBAAgB,IAAI,SAAS,KAAK,EAAE;AAAA,EAC9E;AACD;AAOO,SAAS,WAAwB;AACvC,SAAO,MAAM;AACZ,UAAM,SAAS,cAAc;AAC7B,UAAM,QAAQ,OAAO;AACrB,WAAO;AAAA,MACN,WAAc;AAAA,QACb;AAAA,UACC,2BAA2B,EAAE,QAAQ,4BAA4B,MAAM,CAAC;AAAA,UACxE;AAAA,QACD;AAAA,MACD;AAAA,MACA,kBAAkB,CAAC;AAAA,IACpB;AAAA,EACD;AACD;AAOO,SAAS,QAAQ,SAA6C;AACpE,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,2BAA2B,CAAC;AAAA,IACxD,kBAAkB;AAAA,MACjB,mBAAmB;AAAA,MACnB,GAAI,SAAS,QAAQ,SAAS,EAAE,aAAa,QAAQ,OAAO,IAAI,CAAC;AAAA,IAClE;AAAA,EACD;AACD;AAOO,SAAS,mBAAmB,MAA2B;AAC7D,SAAO,MAAM;AACZ,UAAM,SAAS,cAAc;AAC7B,UAAM,QAAQ,OAAO;AACrB,WAAO;AAAA,MACN,WAAc,kBAAc,wBAAwB,OAAO,aAAa,KAAK,CAAC;AAAA,MAC9E,kBAAkB,EAAE,kBAAkB,KAAK;AAAA,IAC5C;AAAA,EACD;AACD;AAOO,SAAS,cAAc,OAA+B;AAC5D,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,6BAA6B,CAAC;AAAA,IAC1D,kBAAkB;AAAA,MACjB,mBAAmB;AAAA,MACnB,GAAI,OAAO,SAAS,EAAE,kBAAkB,MAAM,IAAI,CAAC;AAAA,IACpD;AAAA,EACD;AACD;AAGO,SAAS,SAAS,SAAgC;AACxD,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,+BAA+B,CAAC;AAAA,IAC5D,kBAAkB,EAAE,kBAAkB,QAAQ;AAAA,EAC/C;AACD;AAGO,SAAS,YAAY,SAAgC;AAC3D,SAAO,OAAO;AAAA,IACb,WAAc,kBAAc,+BAA+B,CAAC;AAAA,IAC5D,kBAAkB,EAAE,wBAAwB,QAAQ;AAAA,EACrD;AACD;","names":["AIMessage","AIMessage","tc","ls","AIMessage","AIMessage","AIMessage","AIMessage","AIMessage","passed"]}
package/dist/index.d.mts CHANGED
@@ -1,5 +1,5 @@
1
- import { T as ToolDefinition, A as Agent, L as LangchainModelConfig, a as ToolSpec } from './model-resolver-lIpXv0Pc.mjs';
2
- export { b as AgentResult, c as AgentRunInput, d as AiMessage, C as ContentBlock, H as HumanMessage, M as Message, S as StreamEvent, e as ToolCall, f as ToolMessage, U as UsageMeta } from './model-resolver-lIpXv0Pc.mjs';
1
+ import { T as ToolDefinition, A as Agent, L as LangchainModelConfig, a as ToolKitSpec, b as ToolSpec } from './model-resolver-DjKRXKtu.mjs';
2
+ export { c as AgentResult, d as AgentRunInput, e as AiMessage, C as ContentBlock, H as HumanMessage, M as Message, S as StreamEvent, f as ToolCall, g as ToolMessage, U as UsageMeta } from './model-resolver-DjKRXKtu.mjs';
3
3
  import 'zod';
4
4
 
5
5
  type AgentHandoff = {
@@ -67,9 +67,11 @@ declare class LangchainAgentFactory implements AgentFactory {
67
67
  */
68
68
  interface ToolProvider<TContext> {
69
69
  /**
70
- * Return serializable tool specs that can be stored in a shared cache.
70
+ * Return a serializable toolkit spec that can be stored in a shared cache.
71
+ * The returned ToolKitSpec is the single source of truth for the toolkit's
72
+ * metadata (name, icon, instruction) and its tool declarations.
71
73
  */
72
- listToolKitSpecs: (toolKit: string, context: TContext) => Promise<ToolSpec[]> | ToolSpec[];
74
+ listToolKitSpec: (toolKit: string, context: TContext) => Promise<ToolKitSpec> | ToolKitSpec;
73
75
  /**
74
76
  * Bind specs into executable tools for a specific request context.
75
77
  * This must not make network calls to "list tools"; only build ToolDefinition objects whose `exec`
@@ -79,21 +81,21 @@ interface ToolProvider<TContext> {
79
81
  }
80
82
 
81
83
  interface ToolCache {
82
- get(toolKit: string): Promise<ToolSpec[] | null> | ToolSpec[] | null;
83
- set(toolKit: string, specs: ToolSpec[]): Promise<void>;
84
+ get(toolKit: string): Promise<ToolKitSpec | null> | ToolKitSpec | null;
85
+ set(toolKit: string, spec: ToolKitSpec): Promise<void>;
84
86
  del(toolKit: string): Promise<void>;
85
87
  }
86
88
  declare class RedisToolCache implements ToolCache {
87
89
  private redis;
88
90
  constructor(connectionString: string);
89
- get(toolKit: string): Promise<ToolSpec[] | null>;
90
- set(toolKit: string, specs: ToolSpec[]): Promise<void>;
91
+ get(toolKit: string): Promise<ToolKitSpec | null>;
92
+ set(toolKit: string, spec: ToolKitSpec): Promise<void>;
91
93
  del(toolKit: string): Promise<void>;
92
94
  }
93
95
  declare class InMemoryToolCache implements ToolCache {
94
96
  private cache;
95
- get(toolKit: string): ToolSpec[] | null;
96
- set(toolKit: string, specs: ToolSpec[]): Promise<void>;
97
+ get(toolKit: string): ToolKitSpec | null;
98
+ set(toolKit: string, spec: ToolKitSpec): Promise<void>;
97
99
  del(toolKit: string): Promise<void>;
98
100
  }
99
101
 
@@ -110,8 +112,9 @@ declare class ToolRegistry<TContext> {
110
112
  registerProvider(toolKit: string, provider: ToolProvider<TContext>): void;
111
113
  registerTools(toolKit: string, tools: ToolDefinition[]): void;
112
114
  getTools(toolKits: string[], context: TContext): Promise<ToolDefinition[]>;
115
+ getToolKitSpecs(toolKits: string[], context: TContext): Promise<ToolKitSpec[]>;
113
116
  invalidate(toolKit: string): void;
114
- private getToolKitSpecs;
117
+ private getToolKitSpec;
115
118
  }
116
119
 
117
- export { Agent, type AgentFactory, type AgentHandoff, type CreateAgentOptions, InMemoryToolCache, LangchainAgentFactory, LangchainModelConfig, RedisToolCache, type ToolCache, ToolDefinition, type ToolProvider, ToolRegistry, ToolSpec };
120
+ export { Agent, type AgentFactory, type AgentHandoff, type CreateAgentOptions, InMemoryToolCache, LangchainAgentFactory, LangchainModelConfig, RedisToolCache, type ToolCache, ToolDefinition, ToolKitSpec, type ToolProvider, ToolRegistry, ToolSpec };
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
- import { T as ToolDefinition, A as Agent, L as LangchainModelConfig, a as ToolSpec } from './model-resolver-lIpXv0Pc.js';
2
- export { b as AgentResult, c as AgentRunInput, d as AiMessage, C as ContentBlock, H as HumanMessage, M as Message, S as StreamEvent, e as ToolCall, f as ToolMessage, U as UsageMeta } from './model-resolver-lIpXv0Pc.js';
1
+ import { T as ToolDefinition, A as Agent, L as LangchainModelConfig, a as ToolKitSpec, b as ToolSpec } from './model-resolver-DjKRXKtu.js';
2
+ export { c as AgentResult, d as AgentRunInput, e as AiMessage, C as ContentBlock, H as HumanMessage, M as Message, S as StreamEvent, f as ToolCall, g as ToolMessage, U as UsageMeta } from './model-resolver-DjKRXKtu.js';
3
3
  import 'zod';
4
4
 
5
5
  type AgentHandoff = {
@@ -67,9 +67,11 @@ declare class LangchainAgentFactory implements AgentFactory {
67
67
  */
68
68
  interface ToolProvider<TContext> {
69
69
  /**
70
- * Return serializable tool specs that can be stored in a shared cache.
70
+ * Return a serializable toolkit spec that can be stored in a shared cache.
71
+ * The returned ToolKitSpec is the single source of truth for the toolkit's
72
+ * metadata (name, icon, instruction) and its tool declarations.
71
73
  */
72
- listToolKitSpecs: (toolKit: string, context: TContext) => Promise<ToolSpec[]> | ToolSpec[];
74
+ listToolKitSpec: (toolKit: string, context: TContext) => Promise<ToolKitSpec> | ToolKitSpec;
73
75
  /**
74
76
  * Bind specs into executable tools for a specific request context.
75
77
  * This must not make network calls to "list tools"; only build ToolDefinition objects whose `exec`
@@ -79,21 +81,21 @@ interface ToolProvider<TContext> {
79
81
  }
80
82
 
81
83
  interface ToolCache {
82
- get(toolKit: string): Promise<ToolSpec[] | null> | ToolSpec[] | null;
83
- set(toolKit: string, specs: ToolSpec[]): Promise<void>;
84
+ get(toolKit: string): Promise<ToolKitSpec | null> | ToolKitSpec | null;
85
+ set(toolKit: string, spec: ToolKitSpec): Promise<void>;
84
86
  del(toolKit: string): Promise<void>;
85
87
  }
86
88
  declare class RedisToolCache implements ToolCache {
87
89
  private redis;
88
90
  constructor(connectionString: string);
89
- get(toolKit: string): Promise<ToolSpec[] | null>;
90
- set(toolKit: string, specs: ToolSpec[]): Promise<void>;
91
+ get(toolKit: string): Promise<ToolKitSpec | null>;
92
+ set(toolKit: string, spec: ToolKitSpec): Promise<void>;
91
93
  del(toolKit: string): Promise<void>;
92
94
  }
93
95
  declare class InMemoryToolCache implements ToolCache {
94
96
  private cache;
95
- get(toolKit: string): ToolSpec[] | null;
96
- set(toolKit: string, specs: ToolSpec[]): Promise<void>;
97
+ get(toolKit: string): ToolKitSpec | null;
98
+ set(toolKit: string, spec: ToolKitSpec): Promise<void>;
97
99
  del(toolKit: string): Promise<void>;
98
100
  }
99
101
 
@@ -110,8 +112,9 @@ declare class ToolRegistry<TContext> {
110
112
  registerProvider(toolKit: string, provider: ToolProvider<TContext>): void;
111
113
  registerTools(toolKit: string, tools: ToolDefinition[]): void;
112
114
  getTools(toolKits: string[], context: TContext): Promise<ToolDefinition[]>;
115
+ getToolKitSpecs(toolKits: string[], context: TContext): Promise<ToolKitSpec[]>;
113
116
  invalidate(toolKit: string): void;
114
- private getToolKitSpecs;
117
+ private getToolKitSpec;
115
118
  }
116
119
 
117
- export { Agent, type AgentFactory, type AgentHandoff, type CreateAgentOptions, InMemoryToolCache, LangchainAgentFactory, LangchainModelConfig, RedisToolCache, type ToolCache, ToolDefinition, type ToolProvider, ToolRegistry, ToolSpec };
120
+ export { Agent, type AgentFactory, type AgentHandoff, type CreateAgentOptions, InMemoryToolCache, LangchainAgentFactory, LangchainModelConfig, RedisToolCache, type ToolCache, ToolDefinition, ToolKitSpec, type ToolProvider, ToolRegistry, ToolSpec };
package/dist/index.js CHANGED
@@ -799,18 +799,37 @@ var ToolRegistry = class {
799
799
  results.push(...entry.tools);
800
800
  } else {
801
801
  const provider = entry.provider;
802
- const specs = await this.getToolKitSpecs(kit, provider, context);
803
- const tools = await provider.bindTools(specs, context);
802
+ const spec = await this.getToolKitSpec(kit, provider, context);
803
+ const tools = await provider.bindTools(spec.tools, context);
804
+ for (const tool2 of tools) {
805
+ tool2.toolKit = spec.name;
806
+ tool2.toolKitIconUrl = spec.iconUrl;
807
+ }
804
808
  results.push(...tools);
805
809
  }
806
810
  }
807
811
  return results;
808
812
  }
813
+ async getToolKitSpecs(toolKits, context) {
814
+ const results = [];
815
+ for (const kit of toolKits) {
816
+ const entry = this.entries.get(kit);
817
+ if (!entry) {
818
+ throw new Error(`No tools or provider registered for tool kit: ${kit}`);
819
+ }
820
+ if (entry.type === "static") {
821
+ throw new Error(`Tool kit "${kit}" was registered with static tools and has no ToolKitSpec`);
822
+ }
823
+ const spec = await this.getToolKitSpec(kit, entry.provider, context);
824
+ results.push(spec);
825
+ }
826
+ return results;
827
+ }
809
828
  invalidate(toolKit) {
810
829
  const entry = this.entries.get(toolKit);
811
830
  if (entry && entry.type === "provider") this.toolSpecCache?.del(toolKit);
812
831
  }
813
- async getToolKitSpecs(toolKit, provider, context) {
832
+ async getToolKitSpec(toolKit, provider, context) {
814
833
  const cache = this.toolSpecCache;
815
834
  if (cache) {
816
835
  try {
@@ -820,15 +839,15 @@ var ToolRegistry = class {
820
839
  console.error("Something went wrong when hitting the cache", e);
821
840
  }
822
841
  }
823
- const specs = await provider.listToolKitSpecs(toolKit, context) ?? [];
842
+ const spec = await provider.listToolKitSpec(toolKit, context);
824
843
  if (cache) {
825
844
  try {
826
- await cache.set(toolKit, specs);
845
+ await cache.set(toolKit, spec);
827
846
  } catch (e) {
828
847
  console.error("Something went wrong when setting the cache", e);
829
848
  }
830
849
  }
831
- return specs;
850
+ return spec;
832
851
  }
833
852
  };
834
853
 
@@ -845,21 +864,27 @@ var RedisToolCache = class {
845
864
  if (!data) return null;
846
865
  try {
847
866
  const parsed = JSON.parse(data);
848
- if (!Array.isArray(parsed)) return null;
849
- return parsed.map((spec) => ({
850
- ...spec,
851
- inputSchema: import_zod.z.fromJSONSchema(spec.inputSchema)
852
- }));
867
+ if (!parsed || !Array.isArray(parsed.tools)) return null;
868
+ return {
869
+ ...parsed,
870
+ tools: parsed.tools.map((spec) => ({
871
+ ...spec,
872
+ inputSchema: import_zod.z.fromJSONSchema(spec.inputSchema)
873
+ }))
874
+ };
853
875
  } catch (e) {
854
876
  console.error("Failed to parse cached tools", e);
855
877
  return null;
856
878
  }
857
879
  }
858
- async set(toolKit, specs) {
859
- const serialized = specs.map((spec) => ({
880
+ async set(toolKit, spec) {
881
+ const serialized = {
860
882
  ...spec,
861
- inputSchema: spec.inputSchema.toJSONSchema()
862
- }));
883
+ tools: spec.tools.map((tool2) => ({
884
+ ...tool2,
885
+ inputSchema: tool2.inputSchema.toJSONSchema()
886
+ }))
887
+ };
863
888
  await this.redis.set(`tool-cache:${toolKit}`, JSON.stringify(serialized));
864
889
  }
865
890
  async del(toolKit) {
@@ -871,8 +896,8 @@ var InMemoryToolCache = class {
871
896
  get(toolKit) {
872
897
  return this.cache.get(`tool-cache:${toolKit}`) ?? null;
873
898
  }
874
- async set(toolKit, specs) {
875
- this.cache.set(`tool-cache:${toolKit}`, specs);
899
+ async set(toolKit, spec) {
900
+ this.cache.set(`tool-cache:${toolKit}`, spec);
876
901
  }
877
902
  async del(toolKit) {
878
903
  this.cache.delete(`tool-cache:${toolKit}`);