@cap-js/agents 0.9.2 → 0.9.3

package/README.md

@@ -1,3 +1,5 @@
+[![REUSE status](https://api.reuse.software/badge/github.com/cap-js/agents)](https://api.reuse.software/info/github.com/cap-js/agents)
+
 # SAP Cloud Application Programming Model, agent development plugin for Node.js
 
 ## About this project

package/cds-plugin.js

@@ -4,7 +4,7 @@ const LOG = cds.log("agents")
 import { patchLangChain } from "./lib/telemetry/tracing.js"
 import cds_compile_to_a2a from "./lib/compile.js"
 import registerDefaultAgentHandlers from "./srv/handlers/index.js"
-import { slugified } from "./lib/utils/markdown.js"
+import { slugified } from "./lib/utils/utils.js"
 
 cds.compile.to.a2a = cds_compile_to_a2a
 
@@ -96,7 +96,7 @@ cds.on("served", async () => {
   }
 
   if (cds.env.agents?.mlflow) {
-    const { setupMlflowExporter } = await import("./lib/telemetry/mlflow.js")
+    const { setupMlflowExporter } = await import("./lib/telemetry/mlflow/index.js")
     setupMlflowExporter()
   }
 })

package/lib/agents/middleware/content-filter.js

@@ -2,7 +2,7 @@ import cds from "@sap/cds"
 import { createMiddleware } from "langchain"
 import { AIMessage } from "@langchain/core/messages"
 import { OrchestrationClient } from "@sap-ai-sdk/langchain"
-import { timeout, retry } from "@sap-cloud-sdk/resilience"
+import { timeout, retry } from "../../utils/resilience.js"
 import { toSdkFilterFormat } from "../../../lib/models/aicore.js"
 import { audit } from "../../utils/utils.js"
 import { INSTRUMENTED } from "../../telemetry/tracing.js"
@@ -136,9 +136,10 @@ export async function contentFilterMiddleware(model) {
             }
           } else {
             // Fail closed: unrecognized errors (timeout, 5xx, network) block the message
-            LOG.warn("Content filter unavailable — blocking input", {
+            LOG.error("Content filter unavailable — blocking input", {
               error: err.message,
               status,
+              data,
             })
             if (hasHumanMessage) {
               return {

package/lib/agents/middleware/index.js

@@ -7,7 +7,17 @@ export default async function buildMiddleware(srv, options = {}) {
   const { statusUpdateMiddleware } = await import("./status-update.js")
   const { humanInTheLoopMiddleware } = await import("./hitl.js")
   const { toolSelectionMiddleware } = await import("./tool-selection.js")
+  const { toolWrapMiddleware } = await import("./tool-wrap.js")
+
+  const hasDynamicMcp = tools?.some((t) => t._mcpDynamic) ?? false
+  let dynamicMcpMiddlewares = []
+  if (hasDynamicMcp) {
+    const { remoteMcpMiddleware } = await import("./remote-mcp.js")
+    dynamicMcpMiddlewares = [remoteMcpMiddleware()]
+  }
+
   return [
+    ...dynamicMcpMiddlewares,
     ...(await quotaEnforcerMiddleware()),
     await contentFilterMiddleware(model),
     await agentActionsMiddleware(),
@@ -15,5 +25,6 @@ export default async function buildMiddleware(srv, options = {}) {
     await statusUpdateMiddleware(),
     ...(await humanInTheLoopMiddleware(srv, tools)),
     toolSelectionMiddleware(),
+    toolWrapMiddleware(),
   ].filter(Boolean)
 }

package/lib/agents/middleware/remote-mcp.js

@@ -0,0 +1,62 @@
+import cds from "@sap/cds"
+import { createMiddleware } from "langchain"
+import { ToolMessage } from "@langchain/core/messages"
+import { MultiServerMCPClient } from "@langchain/mcp-adapters"
+import { toolName } from "../../utils/utils.js"
+
+const LOG = cds.log("agents:mcp")
+
+/**
+ * Middleware that resolves remote MCP tools per-request using the current user's auth headers.
+ * Tools are cached on cds.context.__mcpDynamicTools keyed by mcpUrl for the lifetime of the
+ * request so multi-turn ReAct loops don't issue a new tools/list call on every model invocation.
+ */
+export function remoteMcpMiddleware() {
+  return createMiddleware({
+    name: "RemoteMcpMiddleware",
+
+    wrapModelCall: async (request, handler) => {
+      const cache = (cds.context.__mcpDynamicTools ??= {})
+      const placeholders = (request.tools ?? []).filter((t) => t._mcpDynamic)
+      await Promise.all(
+        placeholders.map(async ({ mcpUrl, serviceName, resolveHeaders }) => {
+          if (cache[mcpUrl]) return
+          const headers = await resolveHeaders()
+          const client = new MultiServerMCPClient({
+            mcpServers: { default: { transport: "http", url: mcpUrl, headers } },
+          })
+          const raw = await client.getTools()
+          for (const t of raw) t.name = toolName(`${serviceName}_${t.name}`)
+          cache[mcpUrl] = raw
+          LOG.debug(
+            `Got ${raw.length} MCP tools from ${mcpUrl}: ${raw.map((t) => t.name).join(", ")}`,
+          )
+        }),
+      )
+      const resolved = placeholders.flatMap(({ mcpUrl }) => cache[mcpUrl] ?? [])
+      const staticTools = (request.tools ?? []).filter((t) => !t._mcpDynamic)
+      return handler({ ...request, tools: [...staticTools, ...resolved] })
+    },
+
+    wrapToolCall: async (request, handler) => {
+      const allCached = Object.values(cds.context.__mcpDynamicTools ?? {}).flat()
+      const tool = allCached.find((t) => t.name === request.toolCall.name)
+      if (!tool) return handler(request)
+      try {
+        const output = await tool.invoke(request.toolCall.args)
+        return new ToolMessage({
+          name: request.toolCall.name,
+          content: typeof output === "string" ? output : JSON.stringify(output),
+          tool_call_id: request.toolCall.id,
+        })
+      } catch (err) {
+        LOG.warn(`MCP tool "${request.toolCall.name}" error: ${err.message}`)
+        return new ToolMessage({
+          name: request.toolCall.name,
+          content: `Error: ${err.message}`,
+          tool_call_id: request.toolCall.id,
+        })
+      }
+    },
+  })
+}

package/lib/agents/middleware/tool-wrap.js

@@ -0,0 +1,52 @@
+import { createMiddleware } from "langchain"
+import { ToolMessage } from "@langchain/core/messages"
+import { isGraphInterrupt } from "@langchain/langgraph"
+import cds from "@sap/cds"
+
+const LOG = cds.log("agents")
+
+/**
+ * Converts tool errors into error ToolMessages so the LLM can retry.
+ * Handles two paths: thrown errors (err.details appended when present)
+ * and tools returning artifact.isError=true (@cap-js/mcp action pattern).
+ */
+export function toolWrapMiddleware() {
+  return createMiddleware({
+    name: "ToolWrapMiddleware",
+    wrapToolCall: async (request, handler) => {
+      const { name, id, args } = request.toolCall
+      try {
+        LOG.debug("[tool]", name, args)
+        const result = await handler(request)
+        if (ToolMessage.isInstance(result) && result.artifact?.isError === true) {
+          result.status = "error"
+        }
+        if (
+          ToolMessage.isInstance(result) &&
+          result.status !== "error" &&
+          typeof result.content === "string" &&
+          result.content.startsWith("Error:")
+        ) {
+          result.status = "error"
+        }
+        if (result?.status === "error") LOG.debug("[tool] error", name, result.content)
+        else LOG.debug("[tool] completed", name)
+        return result
+      } catch (err) {
+        if (isGraphInterrupt(err)) throw err
+        LOG.debug("[tool] error", name, err)
+        let content = `Error: ${err.message}`
+        if (Array.isArray(err.details) && err.details.length > 0) {
+          const lines = err.details.map((d) => `- ${d.message}`).join("\n")
+          content += `\n${lines}`
+        }
+        return new ToolMessage({
+          content,
+          tool_call_id: id ?? "",
+          name,
+          status: "error",
+        })
+      }
+    },
+  })
+}

package/lib/compile.js

@@ -1,7 +1,11 @@
 import cds from "@sap/cds"
-import { getDescription, getFilteredEntities, getFilteredActions } from "./utils/utils.js"
+import {
+  getDescription,
+  getFilteredEntities,
+  getFilteredActions,
+  slugified,
+} from "./utils/utils.js"
 import { buildAgentCard } from "./protocol/agent-card.js"
-import { slugified } from "./utils/markdown.js"
 
 const A2A_BASE_PATH = "/a2a"
 

package/lib/eval/Judge.js

@@ -0,0 +1,239 @@
+import cds from "@sap/cds"
+import { recordEvaluation } from "./eval-run.js"
+
+const LOG = cds.log("agents-judge")
+const CRITERIA_SEPARATOR = "\n\n"
+
+// ─── Base Judge ───────────────────────────────────────────────────────────────
+
+export class Judge {
+  constructor(opts) {
+    _assertSingleConstructorArg(arguments)
+    const { criteria, assessmentName, continuous, invertedScala, type } = _judgeOptions(
+      opts,
+      "ANSWER_RELEVANCE_PROMPT",
+      arguments.length,
+    )
+    if (!criteria || typeof criteria !== "string") {
+      throw new TypeError("Judge: 'criteria' is required and must be a string")
+    }
+    if (type !== undefined && type !== "trajectory") {
+      throw new TypeError("Judge: 'type' must be 'trajectory' when provided")
+    }
+    this._criteria = criteria
+    this._type = type
+    const isInvertedScala = (criteria) => {
+      if (_promptKeyFromCriteria(criteria) === "TOXICITY_PROMPT") {
+        return true
+      }
+      return false
+    }
+    this._invertedScala = invertedScala ?? isInvertedScala(criteria)
+
+    this._assessmentName =
+      assessmentName ?? _assessmentNameFromCriteria(this._criteria, this._defaultAssessmentName())
+    this._continuous = continuous ?? true
+    this._judgeImpl = null
+    this._sessionJudgeImpl = null
+  }
+
+  _defaultAssessmentName() {
+    return this._type === "trajectory" ? "trajectory" : "relevance"
+  }
+
+  async _ensureJudge(type) {
+    const judgeType = arguments.length === 0 ? this._type : type
+    if (judgeType === this._type && this._judgeImpl) return this._judgeImpl
+    if (judgeType !== this._type && this._sessionJudgeImpl) return this._sessionJudgeImpl
+    const judgeImpl = await _loadJudgeImpl(
+      this._criteria,
+      this._assessmentName,
+      this._continuous,
+      judgeType,
+    )
+    if (judgeType === this._type) this._judgeImpl = judgeImpl
+    else this._sessionJudgeImpl = judgeImpl
+    return judgeImpl
+  }
+
+  /** Sibling with appended prompt/criteria. */
+  criteria(criteria) {
+    if (!criteria || typeof criteria !== "string") {
+      throw new TypeError("Judge.criteria: argument 'criteria' is required and must be a string")
+    }
+    const sibling = new this.constructor({
+      criteria: `${this._criteria}${CRITERIA_SEPARATOR}${criteria}`,
+      assessmentName: this._assessmentName,
+      continuous: this._continuous,
+      type: this._type,
+    })
+    return sibling
+  }
+
+  /** @returns {Promise<{score, comment, pass}>} */
+  async evaluate(result) {
+    if (!result) throw new Error("evaluate: result is required")
+    if (Array.isArray(result)) return this._evaluateSession(result)
+    const judgeImpl = await this._ensureJudge()
+    const judgement = await judgeImpl(this._buildInput(result))
+    const { score, pass, comment } = this._judgementResult(judgement)
+    LOG.debug(`[${this._assessmentName}] score=${score} pass=${pass} — ${comment}`)
+    await recordEvaluation(result, {
+      pass,
+      score,
+      comment,
+      assessmentName: this._assessmentName,
+    })
+    return { score, comment, pass }
+  }
+
+  async _evaluateSession(results) {
+    if (results.length === 0) {
+      throw new Error(
+        "Judge.evaluate: session assessment requires a non-empty array of chat() results",
+      )
+    }
+    const judgeImpl = await this._ensureJudge(undefined)
+    const judgement = await judgeImpl({ outputs: results.flatMap((r) => r.messages ?? []) })
+    const { score, pass, comment } = this._judgementResult(judgement)
+    LOG.info(`[${this._assessmentName}] pass=${pass} score=${score} — ${comment}`)
+    const first = results[0]
+    await recordEvaluation(first, {
+      score: pass,
+      comment,
+      sessionId: first.contextId,
+      assessmentName: this._assessmentName,
+      conversationLevel: true,
+    })
+    return { score, comment, pass }
+  }
+
+  _judgementResult(judgement) {
+    const raw = judgement?.score
+    const score = typeof raw === "boolean" ? raw : (raw ?? 0)
+    const pass = this._invertedScala
+      ? typeof score === "boolean"
+        ? !score
+        : score <= 0.5
+      : typeof score === "boolean"
+        ? score
+        : score >= 0.5
+    const comment = judgement?.comment ?? ""
+    return { score, comment, pass }
+  }
+
+  _buildInput(result) {
+    if (this._type === "trajectory") {
+      return {
+        inputs: result.query ?? "",
+        outputs: result.messages ?? [],
+      }
+    }
+    return {
+      inputs: result.query ?? "",
+      outputs: result.text,
+    }
+  }
+}
+
+// ─── matchToolCall ───────────────────────────────────────────────────────────
+
+/** Deterministic tool call assertion. Contributes to success_rate rollup. */
+export function matchToolCall(result, toolName, matcher) {
+  const match = (result?.toolCalls ?? []).find((c) => {
+    if (c.tool !== toolName) return false
+    if (matcher === undefined) return true
+    if (typeof matcher === "function") return !!matcher(c.args)
+    return _partialMatch(c.args, matcher)
+  })
+  const pass = !!match
+  recordEvaluation(result, { pass })
+  return pass
+}
+
+function _partialMatch(actual, expected) {
+  if (!expected || typeof expected !== "object") return actual === expected
+  for (const [k, v] of Object.entries(expected)) {
+    if (actual?.[k] !== v) return false
+  }
+  return true
+}
+
+function _judgeOptions(opts, defaultCriteria = "ANSWER_RELEVANCE_PROMPT", argCount = 1) {
+  if (argCount === 0 || opts === undefined) return { criteria: defaultCriteria }
+  if (typeof opts === "string") return { criteria: opts }
+  if (opts && typeof opts === "object")
+    return { ...opts, criteria: opts.criteria ?? defaultCriteria }
+  throw new TypeError("Judge: constructor argument must be a string or an object")
+}
+
+function _assertSingleConstructorArg(args) {
+  if (args.length > 1) throw new TypeError("Judge: constructor accepts a single argument")
+}
+
+function _assessmentNameFromCriteria(criteria, fallback) {
+  const key = _promptKeyFromCriteria(criteria)
+  return key ? key.replace(/_PROMPT$/, "").toLowerCase() : fallback
+}
+
+function _promptKeyFromCriteria(criteria) {
+  if (!criteria || typeof criteria !== "string") return null
+  const key = criteria.split(CRITERIA_SEPARATOR, 1)[0].trim()
+  return key.endsWith("_PROMPT") ? key : null
+}
+
+const INPUT_OUTPUTS_PLACEHOLDER = `<input>
+{inputs}
+</input>
+
+<output>
+{outputs}
+</output>`
+
+function _resolvePrompt(openevals, criteria) {
+  if (Object.prototype.hasOwnProperty.call(openevals, criteria)) return openevals[criteria]
+  const key = _promptKeyFromCriteria(criteria)
+  if (!key || !Object.prototype.hasOwnProperty.call(openevals, key))
+    return `${criteria}${CRITERIA_SEPARATOR}${INPUT_OUTPUTS_PLACEHOLDER}`
+  const rest = criteria.slice(criteria.indexOf(key) + key.length).trimStart()
+  return rest ? `${openevals[key]}${CRITERIA_SEPARATOR}${rest}` : openevals[key]
+}
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+async function _loadOpenevals() {
+  const savedExpect = globalThis.expect
+  let openevals
+  try {
+    openevals = await import("openevals")
+  } catch (err) {
+    throw new Error(
+      "openevals is required for Judge.evaluate(). Install it as a devDependency:\n  npm install --save-dev openevals\n" +
+        `Original error: ${err.message}`,
+      { cause: err },
+    )
+  }
+  if (savedExpect && globalThis.expect !== savedExpect) globalThis.expect = savedExpect
+  return openevals
+}
+
+async function _buildLlm() {
+  const name = "llm"
+  const { kind, impl, ...options } = cds.env.requires[name] ?? {}
+  const providerImpl = impl ?? cds.env.requires.kinds?.[kind]?.impl
+  if (!providerImpl) throw new Error("No service implementation found for " + name)
+  const { default: LLMProvider } = await import(providerImpl)
+  const llm = new LLMProvider(name, options)
+  llm[Symbol.for("@cap-js/agents:instrumented")] = true
+  return llm
+}
+
+async function _loadJudgeImpl(criteria, assessmentName, continuous, type) {
+  const openevals = await _loadOpenevals()
+  const llm = await _buildLlm()
+  const prompt = _resolvePrompt(openevals, criteria)
+  if (type === "trajectory") {
+    return openevals.createTrajectoryLLMAsJudge({ judge: llm, prompt, assessmentName })
+  }
+  return openevals.createLLMAsJudge({ judge: llm, prompt, continuous, assessmentName })
+}

package/lib/eval/eval-describe.js

@@ -0,0 +1,69 @@
+import { evalRun } from "../eval/eval-run.js"
+
+const PATCHED = Symbol.for("@cap-js/agents:eval-describe-patched")
+
+export function installEvalDescribe({
+  target = globalThis,
+  evalRun: registerEvalRun = evalRun,
+} = {}) {
+  const original = target.describe
+  if (typeof original !== "function") return false
+  if (original[PATCHED]) return false
+
+  let depth = 0
+
+  const wrapSuite = (suite) => {
+    if (typeof suite !== "function") return suite
+
+    const wrapped = function evalDescribe(name, factory, ...args) {
+      if (typeof factory !== "function") return suite.call(this, name, factory, ...args)
+
+      return suite.call(
+        this,
+        name,
+        function evalSuite(...suiteArgs) {
+          depth += 1
+          const topLevel = depth === 1
+          try {
+            if (topLevel && typeof name === "string") registerEvalRun({ name })
+            return factory.apply(this, suiteArgs)
+          } finally {
+            depth -= 1
+          }
+        },
+        ...args,
+      )
+    }
+
+    copySuiteProperties(suite, wrapped, wrapSuite)
+    return wrapped
+  }
+
+  target.describe = wrapSuite(original)
+  target.describe[PATCHED] = true
+  target.describe._original = original
+  return true
+}
+
+function copySuiteProperties(source, target, wrapSuite) {
+  for (const key of Reflect.ownKeys(source)) {
+    if (["length", "name", "prototype"].includes(key)) continue
+    const descriptor = Object.getOwnPropertyDescriptor(source, key)
+    if (!descriptor) continue
+    if (typeof descriptor.value === "function") {
+      if (["each", "skipIf", "runIf"].includes(key)) {
+        descriptor.value = function describeFactory(...args) {
+          return wrapSuite(source[key].apply(this, args))
+        }
+      } else {
+        descriptor.value = wrapSuite(descriptor.value)
+      }
+    }
+    try {
+      Object.defineProperty(target, key, descriptor)
+    } catch {
+      // Vitest may expose non-configurable helper properties. The base describe
+      // still works, so ignore properties that cannot be mirrored.
+    }
+  }
+}

package/lib/eval/eval-run.js

@@ -0,0 +1,147 @@
+/* global beforeAll, afterEach, afterAll */
+import cds from "@sap/cds"
+import {
+  postMlflowAssessment,
+  createEvalRun,
+  closeEvalRun,
+  logMlflowMetrics,
+} from "../telemetry/mlflow/evaluation.js"
+import { flushMlflowTraces } from "../telemetry/mlflow/tracing.js"
+
+export function getActiveRunState() {
+  return cds._activeEvalRun ?? null
+}
+
+export function evalRun(opts = {}) {
+  if (typeof beforeAll !== "function" || typeof afterAll !== "function") return
+  if (!cds.env.agents?.mlflow) return
+
+  let state = null
+
+  function _makeState(runId, mlflowRunId) {
+    return {
+      runId,
+      mlflowRunId,
+      validationsByTask: new Map(),
+    }
+  }
+
+  beforeAll(async () => {
+    const runId = cds.utils.uuid()
+    const mlflowRunId = await createEvalRun(opts).catch(() => null)
+    state = _makeState(runId, mlflowRunId)
+    cds._activeEvalRun = state
+  })
+
+  if (typeof afterEach === "function") {
+    afterEach(async () => {
+      if (state) await _flushValidations(state)
+    })
+  }
+
+  afterAll(async () => {
+    if (state) await _flushValidations(state)
+    await flushMlflowTraces()
+    await closeEvalRun(state?.mlflowRunId).catch(() => {})
+    if (cds._activeEvalRun === state) cds._activeEvalRun = null
+    state = null
+  })
+}
+
+async function _flushValidations(state) {
+  if (!state?.validationsByTask.size) return
+
+  const tasks = []
+  for (const [, entry] of state.validationsByTask) {
+    const { passes, traceId } = entry
+    if (!passes.length) continue
+
+    const success_rate = passes.every(Boolean) ? 1 : 0
+    const output_correctness = passes.filter(Boolean).length / passes.length
+    const codeOpts = { sourceType: "CODE" }
+
+    if (state.mlflowRunId) {
+      tasks.push(
+        logMlflowMetrics(state.mlflowRunId, { success_rate, output_correctness }).catch(() => {}),
+      )
+    }
+
+    if (traceId) {
+      tasks.push(
+        flushMlflowTraces().then(() =>
+          Promise.all([
+            postMlflowAssessment(traceId, success_rate, "", "success_rate", null, codeOpts).catch(
+              () => {},
+            ),
+            postMlflowAssessment(
+              traceId,
+              output_correctness,
+              "",
+              "output_correctness",
+              null,
+              codeOpts,
+            ).catch(() => {}),
+          ]),
+        ),
+      )
+    }
+  }
+
+  await Promise.all(tasks)
+  state.validationsByTask.clear()
+}
+
+export function recordEvaluation(result, assessment = {}) {
+  if (!cds.env.agents?.mlflow) return
+  const { pass, score, comment, ...config } = assessment
+  // Conversation level evaluations shall not be included in the per task roll-up
+  if (pass !== undefined && !config.conversationLevel) _addValidation(result, pass)
+  if (score === undefined) return
+  return _postAssessmentScore(result, score, comment, config)
+}
+
+function _addValidation(result, pass) {
+  const state = result?._evalState ?? cds._activeEvalRun
+  if (!state || !result?.taskId) return
+  const key = result.taskId
+  if (!state.validationsByTask.has(key)) {
+    state.validationsByTask.set(key, { passes: [], traceId: result.traceId })
+  }
+  state.validationsByTask.get(key).passes.push(pass)
+}
+
+async function _postAssessmentScore(result, score, comment, config) {
+  const traceId = config.traceId ?? result?.traceId
+  if (!traceId) return
+
+  const opts = { sourceType: config.sourceType }
+  await flushMlflowTraces()
+
+  // conversationLevel: session assessment — post with session metadata only
+  if (config.conversationLevel) {
+    await postMlflowAssessment(
+      traceId,
+      score,
+      comment ?? "",
+      config.assessmentName,
+      config.model ?? null,
+      { ...opts, metadata: { "mlflow.trace.session": config.sessionId ?? "" } },
+    )
+  } else {
+    // Single-turn: post to this trace only, no session metadata
+    await postMlflowAssessment(
+      traceId,
+      score,
+      comment ?? "",
+      config.assessmentName,
+      config.model ?? null,
+      opts,
+    )
+  }
+}
+
+export async function logMlflowMetricsForResult(result, state = null) {
+  state = state ?? cds._activeEvalRun
+  if (!state?.mlflowRunId) return
+  await logMlflowMetrics(state.mlflowRunId, result.metrics).catch(() => {})
+}

package/lib/eval/index.js

@@ -0,0 +1,6 @@
+import { installEvalDescribe } from "./eval-describe.js"
+
+installEvalDescribe()
+
+export { Judge, matchToolCall } from "./Judge.js"
+export { evalRun } from "./eval-run.js"