@cap-js/agents 0.9.1 → 0.9.3

package/lib/eval/Judge.js

@@ -0,0 +1,239 @@
+import cds from "@sap/cds"
+import { recordEvaluation } from "./eval-run.js"
+
+const LOG = cds.log("agents-judge")
+const CRITERIA_SEPARATOR = "\n\n"
+
+// ─── Base Judge ───────────────────────────────────────────────────────────────
+
+export class Judge {
+  constructor(opts) {
+    _assertSingleConstructorArg(arguments)
+    const { criteria, assessmentName, continuous, invertedScala, type } = _judgeOptions(
+      opts,
+      "ANSWER_RELEVANCE_PROMPT",
+      arguments.length,
+    )
+    if (!criteria || typeof criteria !== "string") {
+      throw new TypeError("Judge: 'criteria' is required and must be a string")
+    }
+    if (type !== undefined && type !== "trajectory") {
+      throw new TypeError("Judge: 'type' must be 'trajectory' when provided")
+    }
+    this._criteria = criteria
+    this._type = type
+    const isInvertedScala = (criteria) => {
+      if (_promptKeyFromCriteria(criteria) === "TOXICITY_PROMPT") {
+        return true
+      }
+      return false
+    }
+    this._invertedScala = invertedScala ?? isInvertedScala(criteria)
+
+    this._assessmentName =
+      assessmentName ?? _assessmentNameFromCriteria(this._criteria, this._defaultAssessmentName())
+    this._continuous = continuous ?? true
+    this._judgeImpl = null
+    this._sessionJudgeImpl = null
+  }
+
+  _defaultAssessmentName() {
+    return this._type === "trajectory" ? "trajectory" : "relevance"
+  }
+
+  async _ensureJudge(type) {
+    const judgeType = arguments.length === 0 ? this._type : type
+    if (judgeType === this._type && this._judgeImpl) return this._judgeImpl
+    if (judgeType !== this._type && this._sessionJudgeImpl) return this._sessionJudgeImpl
+    const judgeImpl = await _loadJudgeImpl(
+      this._criteria,
+      this._assessmentName,
+      this._continuous,
+      judgeType,
+    )
+    if (judgeType === this._type) this._judgeImpl = judgeImpl
+    else this._sessionJudgeImpl = judgeImpl
+    return judgeImpl
+  }
+
+  /** Sibling with appended prompt/criteria. */
+  criteria(criteria) {
+    if (!criteria || typeof criteria !== "string") {
+      throw new TypeError("Judge.criteria: argument 'criteria' is required and must be a string")
+    }
+    const sibling = new this.constructor({
+      criteria: `${this._criteria}${CRITERIA_SEPARATOR}${criteria}`,
+      assessmentName: this._assessmentName,
+      continuous: this._continuous,
+      type: this._type,
+    })
+    return sibling
+  }
+
+  /** @returns {Promise<{score, comment, pass}>} */
+  async evaluate(result) {
+    if (!result) throw new Error("evaluate: result is required")
+    if (Array.isArray(result)) return this._evaluateSession(result)
+    const judgeImpl = await this._ensureJudge()
+    const judgement = await judgeImpl(this._buildInput(result))
+    const { score, pass, comment } = this._judgementResult(judgement)
+    LOG.debug(`[${this._assessmentName}] score=${score} pass=${pass} — ${comment}`)
+    await recordEvaluation(result, {
+      pass,
+      score,
+      comment,
+      assessmentName: this._assessmentName,
+    })
+    return { score, comment, pass }
+  }
+
+  async _evaluateSession(results) {
+    if (results.length === 0) {
+      throw new Error(
+        "Judge.evaluate: session assessment requires a non-empty array of chat() results",
+      )
+    }
+    const judgeImpl = await this._ensureJudge(undefined)
+    const judgement = await judgeImpl({ outputs: results.flatMap((r) => r.messages ?? []) })
+    const { score, pass, comment } = this._judgementResult(judgement)
+    LOG.info(`[${this._assessmentName}] pass=${pass} score=${score} — ${comment}`)
+    const first = results[0]
+    await recordEvaluation(first, {
+      score: pass,
+      comment,
+      sessionId: first.contextId,
+      assessmentName: this._assessmentName,
+      conversationLevel: true,
+    })
+    return { score, comment, pass }
+  }
+
+  _judgementResult(judgement) {
+    const raw = judgement?.score
+    const score = typeof raw === "boolean" ? raw : (raw ?? 0)
+    const pass = this._invertedScala
+      ? typeof score === "boolean"
+        ? !score
+        : score <= 0.5
+      : typeof score === "boolean"
+        ? score
+        : score >= 0.5
+    const comment = judgement?.comment ?? ""
+    return { score, comment, pass }
+  }
+
+  _buildInput(result) {
+    if (this._type === "trajectory") {
+      return {
+        inputs: result.query ?? "",
+        outputs: result.messages ?? [],
+      }
+    }
+    return {
+      inputs: result.query ?? "",
+      outputs: result.text,
+    }
+  }
+}
+
+// ─── matchToolCall ───────────────────────────────────────────────────────────
+
+/** Deterministic tool call assertion. Contributes to success_rate rollup. */
+export function matchToolCall(result, toolName, matcher) {
+  const match = (result?.toolCalls ?? []).find((c) => {
+    if (c.tool !== toolName) return false
+    if (matcher === undefined) return true
+    if (typeof matcher === "function") return !!matcher(c.args)
+    return _partialMatch(c.args, matcher)
+  })
+  const pass = !!match
+  recordEvaluation(result, { pass })
+  return pass
+}
+
+function _partialMatch(actual, expected) {
+  if (!expected || typeof expected !== "object") return actual === expected
+  for (const [k, v] of Object.entries(expected)) {
+    if (actual?.[k] !== v) return false
+  }
+  return true
+}
+
+function _judgeOptions(opts, defaultCriteria = "ANSWER_RELEVANCE_PROMPT", argCount = 1) {
+  if (argCount === 0 || opts === undefined) return { criteria: defaultCriteria }
+  if (typeof opts === "string") return { criteria: opts }
+  if (opts && typeof opts === "object")
+    return { ...opts, criteria: opts.criteria ?? defaultCriteria }
+  throw new TypeError("Judge: constructor argument must be a string or an object")
+}
+
+function _assertSingleConstructorArg(args) {
+  if (args.length > 1) throw new TypeError("Judge: constructor accepts a single argument")
+}
+
+function _assessmentNameFromCriteria(criteria, fallback) {
+  const key = _promptKeyFromCriteria(criteria)
+  return key ? key.replace(/_PROMPT$/, "").toLowerCase() : fallback
+}
+
+function _promptKeyFromCriteria(criteria) {
+  if (!criteria || typeof criteria !== "string") return null
+  const key = criteria.split(CRITERIA_SEPARATOR, 1)[0].trim()
+  return key.endsWith("_PROMPT") ? key : null
+}
+
+const INPUT_OUTPUTS_PLACEHOLDER = `<input>
+{inputs}
+</input>
+
+<output>
+{outputs}
+</output>`
+
+function _resolvePrompt(openevals, criteria) {
+  if (Object.prototype.hasOwnProperty.call(openevals, criteria)) return openevals[criteria]
+  const key = _promptKeyFromCriteria(criteria)
+  if (!key || !Object.prototype.hasOwnProperty.call(openevals, key))
+    return `${criteria}${CRITERIA_SEPARATOR}${INPUT_OUTPUTS_PLACEHOLDER}`
+  const rest = criteria.slice(criteria.indexOf(key) + key.length).trimStart()
+  return rest ? `${openevals[key]}${CRITERIA_SEPARATOR}${rest}` : openevals[key]
+}
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+async function _loadOpenevals() {
+  const savedExpect = globalThis.expect
+  let openevals
+  try {
+    openevals = await import("openevals")
+  } catch (err) {
+    throw new Error(
+      "openevals is required for Judge.evaluate(). Install it as a devDependency:\n  npm install --save-dev openevals\n" +
+        `Original error: ${err.message}`,
+      { cause: err },
+    )
+  }
+  if (savedExpect && globalThis.expect !== savedExpect) globalThis.expect = savedExpect
+  return openevals
+}
+
+async function _buildLlm() {
+  const name = "llm"
+  const { kind, impl, ...options } = cds.env.requires[name] ?? {}
+  const providerImpl = impl ?? cds.env.requires.kinds?.[kind]?.impl
+  if (!providerImpl) throw new Error("No service implementation found for " + name)
+  const { default: LLMProvider } = await import(providerImpl)
+  const llm = new LLMProvider(name, options)
+  llm[Symbol.for("@cap-js/agents:instrumented")] = true
+  return llm
+}
+
+async function _loadJudgeImpl(criteria, assessmentName, continuous, type) {
+  const openevals = await _loadOpenevals()
+  const llm = await _buildLlm()
+  const prompt = _resolvePrompt(openevals, criteria)
+  if (type === "trajectory") {
+    return openevals.createTrajectoryLLMAsJudge({ judge: llm, prompt, assessmentName })
+  }
+  return openevals.createLLMAsJudge({ judge: llm, prompt, continuous, assessmentName })
+}

package/lib/eval/eval-describe.js

@@ -0,0 +1,69 @@
+import { evalRun } from "../eval/eval-run.js"
+
+const PATCHED = Symbol.for("@cap-js/agents:eval-describe-patched")
+
+export function installEvalDescribe({
+  target = globalThis,
+  evalRun: registerEvalRun = evalRun,
+} = {}) {
+  const original = target.describe
+  if (typeof original !== "function") return false
+  if (original[PATCHED]) return false
+
+  let depth = 0
+
+  const wrapSuite = (suite) => {
+    if (typeof suite !== "function") return suite
+
+    const wrapped = function evalDescribe(name, factory, ...args) {
+      if (typeof factory !== "function") return suite.call(this, name, factory, ...args)
+
+      return suite.call(
+        this,
+        name,
+        function evalSuite(...suiteArgs) {
+          depth += 1
+          const topLevel = depth === 1
+          try {
+            if (topLevel && typeof name === "string") registerEvalRun({ name })
+            return factory.apply(this, suiteArgs)
+          } finally {
+            depth -= 1
+          }
+        },
+        ...args,
+      )
+    }
+
+    copySuiteProperties(suite, wrapped, wrapSuite)
+    return wrapped
+  }
+
+  target.describe = wrapSuite(original)
+  target.describe[PATCHED] = true
+  target.describe._original = original
+  return true
+}
+
+function copySuiteProperties(source, target, wrapSuite) {
+  for (const key of Reflect.ownKeys(source)) {
+    if (["length", "name", "prototype"].includes(key)) continue
+    const descriptor = Object.getOwnPropertyDescriptor(source, key)
+    if (!descriptor) continue
+    if (typeof descriptor.value === "function") {
+      if (["each", "skipIf", "runIf"].includes(key)) {
+        descriptor.value = function describeFactory(...args) {
+          return wrapSuite(source[key].apply(this, args))
+        }
+      } else {
+        descriptor.value = wrapSuite(descriptor.value)
+      }
+    }
+    try {
+      Object.defineProperty(target, key, descriptor)
+    } catch {
+      // Vitest may expose non-configurable helper properties. The base describe
+      // still works, so ignore properties that cannot be mirrored.
+    }
+  }
+}

package/lib/eval/eval-run.js

@@ -0,0 +1,147 @@
+/* global beforeAll, afterEach, afterAll */
+import cds from "@sap/cds"
+import {
+  postMlflowAssessment,
+  createEvalRun,
+  closeEvalRun,
+  logMlflowMetrics,
+} from "../telemetry/mlflow/evaluation.js"
+import { flushMlflowTraces } from "../telemetry/mlflow/tracing.js"
+
+export function getActiveRunState() {
+  return cds._activeEvalRun ?? null
+}
+
+export function evalRun(opts = {}) {
+  if (typeof beforeAll !== "function" || typeof afterAll !== "function") return
+  if (!cds.env.agents?.mlflow) return
+
+  let state = null
+
+  function _makeState(runId, mlflowRunId) {
+    return {
+      runId,
+      mlflowRunId,
+      validationsByTask: new Map(),
+    }
+  }
+
+  beforeAll(async () => {
+    const runId = cds.utils.uuid()
+    const mlflowRunId = await createEvalRun(opts).catch(() => null)
+    state = _makeState(runId, mlflowRunId)
+    cds._activeEvalRun = state
+  })
+
+  if (typeof afterEach === "function") {
+    afterEach(async () => {
+      if (state) await _flushValidations(state)
+    })
+  }
+
+  afterAll(async () => {
+    if (state) await _flushValidations(state)
+    await flushMlflowTraces()
+    await closeEvalRun(state?.mlflowRunId).catch(() => {})
+    if (cds._activeEvalRun === state) cds._activeEvalRun = null
+    state = null
+  })
+}
+
+async function _flushValidations(state) {
+  if (!state?.validationsByTask.size) return
+
+  const tasks = []
+  for (const [, entry] of state.validationsByTask) {
+    const { passes, traceId } = entry
+    if (!passes.length) continue
+
+    const success_rate = passes.every(Boolean) ? 1 : 0
+    const output_correctness = passes.filter(Boolean).length / passes.length
+    const codeOpts = { sourceType: "CODE" }
+
+    if (state.mlflowRunId) {
+      tasks.push(
+        logMlflowMetrics(state.mlflowRunId, { success_rate, output_correctness }).catch(() => {}),
+      )
+    }
+
+    if (traceId) {
+      tasks.push(
+        flushMlflowTraces().then(() =>
+          Promise.all([
+            postMlflowAssessment(traceId, success_rate, "", "success_rate", null, codeOpts).catch(
+              () => {},
+            ),
+            postMlflowAssessment(
+              traceId,
+              output_correctness,
+              "",
+              "output_correctness",
+              null,
+              codeOpts,
+            ).catch(() => {}),
+          ]),
+        ),
+      )
+    }
+  }
+
+  await Promise.all(tasks)
+  state.validationsByTask.clear()
+}
+
+export function recordEvaluation(result, assessment = {}) {
+  if (!cds.env.agents?.mlflow) return
+  const { pass, score, comment, ...config } = assessment
+  // Conversation level evaluations shall not be included in the per task roll-up
+  if (pass !== undefined && !config.conversationLevel) _addValidation(result, pass)
+  if (score === undefined) return
+  return _postAssessmentScore(result, score, comment, config)
+}
+
+function _addValidation(result, pass) {
+  const state = result?._evalState ?? cds._activeEvalRun
+  if (!state || !result?.taskId) return
+  const key = result.taskId
+  if (!state.validationsByTask.has(key)) {
+    state.validationsByTask.set(key, { passes: [], traceId: result.traceId })
+  }
+  state.validationsByTask.get(key).passes.push(pass)
+}
+
+async function _postAssessmentScore(result, score, comment, config) {
+  const traceId = config.traceId ?? result?.traceId
+  if (!traceId) return
+
+  const opts = { sourceType: config.sourceType }
+  await flushMlflowTraces()
+
+  // conversationLevel: session assessment — post with session metadata only
+  if (config.conversationLevel) {
+    await postMlflowAssessment(
+      traceId,
+      score,
+      comment ?? "",
+      config.assessmentName,
+      config.model ?? null,
+      { ...opts, metadata: { "mlflow.trace.session": config.sessionId ?? "" } },
+    )
+  } else {
+    // Single-turn: post to this trace only, no session metadata
+    await postMlflowAssessment(
+      traceId,
+      score,
+      comment ?? "",
+      config.assessmentName,
+      config.model ?? null,
+      opts,
+    )
+  }
+}
+
+export async function logMlflowMetricsForResult(result, state = null) {
+  state = state ?? cds._activeEvalRun
+  if (!state?.mlflowRunId) return
+  await logMlflowMetrics(state.mlflowRunId, result.metrics).catch(() => {})
+}

package/lib/eval/index.js

@@ -0,0 +1,6 @@
+import { installEvalDescribe } from "./eval-describe.js"
+
+installEvalDescribe()
+
+export { Judge, matchToolCall } from "./Judge.js"
+export { evalRun } from "./eval-run.js"

package/lib/eval/metrics.js

@@ -0,0 +1,53 @@
+function _hrtimeToMs(hr) {
+  return hr[0] * 1000 + hr[1] / 1e6
+}
+
+/**
+ * @param {object[]} spans     Finished OTel spans from span-collector.js
+ * @returns {{ input_tokens, output_tokens, total_tokens, tool_call_count, latency_ms, cost_usd }}
+ */
+export function metricsFromSpans(spans) {
+  let input_tokens = 0
+  let output_tokens = 0
+  let tool_call_count = 0
+  let cost_usd = 0
+  let latency_ms = null
+
+  for (const span of spans) {
+    const attrs = span.attributes ?? {}
+    const op = attrs["gen_ai.operation.name"]
+
+    if (op === "chat") {
+      input_tokens += Number(attrs["gen_ai.usage.input_tokens"] ?? 0)
+      output_tokens += Number(attrs["gen_ai.usage.output_tokens"] ?? 0)
+
+      // mlflow.llm.cost is set by apps themselves
+      const rawCost = attrs["mlflow.llm.cost"]
+      if (rawCost) {
+        try {
+          const c = typeof rawCost === "string" ? JSON.parse(rawCost) : rawCost
+          cost_usd += c.total_cost ?? 0
+        } catch {
+          /* malformed — skip */
+        }
+      }
+    }
+
+    if (op === "execute_tool") {
+      tool_call_count++
+    }
+
+    if (op === "invoke_agent" && span.endTime && span.startTime) {
+      latency_ms = _hrtimeToMs(span.endTime) - _hrtimeToMs(span.startTime)
+    }
+  }
+
+  return {
+    input_tokens,
+    output_tokens,
+    total_tokens: input_tokens + output_tokens,
+    tool_call_count,
+    latency_ms,
+    cost_usd: cost_usd > 0 ? cost_usd : null,
+  }
+}

package/lib/eval/span-collector.js

@@ -0,0 +1,50 @@
+const _sessions = new Map()
+let _registered = false
+let _sessionCounter = 0
+
+const _processor = {
+  onStart() {},
+  onEnd(span) {
+    if (_sessions.size === 0) return
+    for (const session of _sessions.values()) {
+      session.spans.push(span)
+    }
+  },
+  async forceFlush() {},
+  async shutdown() {},
+}
+
+async function _ensureRegistered() {
+  if (_registered) return
+  _registered = true
+  try {
+    const { trace } = await import("@opentelemetry/api")
+    const provider = trace.getTracerProvider()
+    const delegate = provider.getDelegate?.() || provider
+    if (delegate.constructor?.name === "NoopTracerProvider") return
+    if (typeof delegate.addSpanProcessor === "function") {
+      delegate.addSpanProcessor(_processor)
+    } else if (Array.isArray(delegate._activeSpanProcessor?._spanProcessors)) {
+      delegate._activeSpanProcessor._spanProcessors.push(_processor)
+    }
+  } catch {
+    /* OTel not present */
+  }
+}
+
+function _openSession() {
+  const id = ++_sessionCounter
+  const session = { spans: [] }
+  _sessions.set(id, session)
+  return {
+    collect() {
+      _sessions.delete(id)
+      return session.spans
+    },
+  }
+}
+
+export async function startCollection() {
+  await _ensureRegistered()
+  return _openSession()
+}

package/lib/index.js

@@ -122,7 +122,7 @@ export default function A2AProtocolAdapter(srv, options = {}) {
     }
   })
 
-  if (!cds.env.profiles?.includes("production")) {
+  if (cds.env?.server?.index) {
     linkProviders.push((entity, endpoint) => {
       if (entity || endpoint?.kind !== "agent") return undefined
       return {
@@ -161,7 +161,8 @@ export default function A2AProtocolAdapter(srv, options = {}) {
   }
 
   router.get("/.well-known/agent-card.json", (req, res) => {
-    const url = proxyUrl || `${req.protocol}://${req.get("host")}${req.baseUrl}`
+    const proto = req.headers["x-forwarded-proto"]?.split(",")[0].trim() || req.protocol
+    const url = proxyUrl || `${proto}://${req.get("host")}${req.baseUrl}`
     // Regenerate agent card when feature toggles are active (annotations may differ)
     let card
     if (cds.context?.features && Object.keys(cds.context.features).length > 0) {
@@ -182,7 +183,7 @@ export default function A2AProtocolAdapter(srv, options = {}) {
     res.json(card)
   })
 
-  if (!cds.env.profiles?.includes("production")) {
+  if (cds.env?.server?.index) {
     router.use("/preview", previewAuthChallenge(srv), preview(agentCard.name || srv.name))
   }
 

package/lib/models/aicore.js

@@ -1,11 +1,13 @@
 import cds from "@sap/cds"
 import { OrchestrationClient } from "@sap-ai-sdk/langchain"
-import { circuitBreaker, timeout } from "@sap-cloud-sdk/resilience"
+import { circuitBreaker, timeout } from "../utils/resilience.js"
 
 import { SystemMessage, ToolMessage, HumanMessage, AIMessage } from "@langchain/core/messages"
 import { ms4 } from "../utils/utils.js"
+import { syncSystemPrompt } from "../telemetry/mlflow/prompts.js"
 
 const LOG = cds.log("agents")
+const DEFAULT_STREAM_DELIMITERS = [".", "!", "?"]
 
 class _InstrumentedOrchestrationClient extends OrchestrationClient {
   constructor(name, options) {
@@ -18,6 +20,7 @@ class _InstrumentedOrchestrationClient extends OrchestrationClient {
     // and extra ReAct iterations — see PR #188 review).
     const params =
       options.params || (deepAgent ? { max_tokens: 4096, temperature: 0 } : cds.env.agents?.params)
+    const auditParams = params && typeof params === "object" ? { ...params } : params
 
     LOG.debug("Initializing LLM", { model, deepAgent: !!deepAgent })
 
@@ -42,15 +45,15 @@ class _InstrumentedOrchestrationClient extends OrchestrationClient {
       {
         promptTemplating: { model: { name: model, params } },
         ...(filtering && { filtering }),
+      },
+      {
         // `streaming` controls the SDK's auto-stream-and-concatenate in _generate()
         // (i.e. direct model.invoke() calls). It does NOT gate LangGraph token
         // streaming — that is driven by graph.stream(streamMode:["messages"]) via a
         // streaming callback handler + our overridden _streamResponseChunks, and
         // works for every agent regardless of this flag (deep or managed alike).
         // Default on; `streaming: false` opts out.
-        ...(streaming !== false ? { streaming: true } : {}),
-      },
-      {
+        streaming: streaming !== false,
         onFailedAttempt: (err) => {
           // Abort retries when circuit breaker is open (otherwise pRetry delays ~30-60s)
           if (err.code === "EOPENBREAKER" || err.message === "Breaker is open") {
@@ -62,7 +65,7 @@ class _InstrumentedOrchestrationClient extends OrchestrationClient {
       destination,
     )
     this.name = name
-    this.options = { ...options, params, contentFilter, flatten }
+    this.options = { ...options, params: auditParams, contentFilter, flatten }
   }
 
   async _generate(messages, opts, runManager) {
@@ -88,7 +91,7 @@ class _InstrumentedOrchestrationClient extends OrchestrationClient {
     opts = _withMiddleware(this, opts)
     const prepared = _prepareMessages(messages, { flatten, model, opts })
     const inputMessages = prepared.inputMessages
-    opts = prepared.opts
+    opts = withDefaultStreamDelimiters(prepared.opts)
 
     let turnHasToolCall = false
     for await (const chunk of super._streamResponseChunks(inputMessages, opts, runManager)) {
@@ -144,9 +147,23 @@ function _prepareMessages(messages, { flatten, model, opts }) {
       opts = { ...opts, tools }
     }
   }
+  syncSystemPrompt(inputMessages)
   return { inputMessages, opts }
 }
 
+export function withDefaultStreamDelimiters(opts = {}) {
+  return {
+    ...opts,
+    streamOptions: {
+      ...opts.streamOptions,
+      global: {
+        ...opts.streamOptions?.global,
+        delimiters: opts.streamOptions?.global?.delimiters ?? DEFAULT_STREAM_DELIMITERS,
+      },
+    },
+  }
+}
+
 // Claude currently only supports caching of type ephemeral. TTL can differ between 5min or 1h but
 // we use the 5min default
 const CACHE_CONTROL_EPHEMERAL = { type: "ephemeral" }