@cap-js/agents 0.9.2 → 0.9.4

package/lib/eval/eval-run.js

@@ -0,0 +1,147 @@
+/* global beforeAll, afterEach, afterAll */
+import cds from "@sap/cds"
+import {
+  postMlflowAssessment,
+  createEvalRun,
+  closeEvalRun,
+  logMlflowMetrics,
+} from "../telemetry/mlflow/evaluation.js"
+import { flushMlflowTraces } from "../telemetry/mlflow/tracing.js"
+
+export function getActiveRunState() {
+  return cds._activeEvalRun ?? null
+}
+
+export function evalRun(opts = {}) {
+  if (typeof beforeAll !== "function" || typeof afterAll !== "function") return
+  if (!cds.env.agents?.mlflow) return
+
+  let state = null
+
+  function _makeState(runId, mlflowRunId) {
+    return {
+      runId,
+      mlflowRunId,
+      validationsByTask: new Map(),
+    }
+  }
+
+  beforeAll(async () => {
+    const runId = cds.utils.uuid()
+    const mlflowRunId = await createEvalRun(opts).catch(() => null)
+    state = _makeState(runId, mlflowRunId)
+    cds._activeEvalRun = state
+  })
+
+  if (typeof afterEach === "function") {
+    afterEach(async () => {
+      if (state) await _flushValidations(state)
+    })
+  }
+
+  afterAll(async () => {
+    if (state) await _flushValidations(state)
+    await flushMlflowTraces()
+    await closeEvalRun(state?.mlflowRunId).catch(() => {})
+    if (cds._activeEvalRun === state) cds._activeEvalRun = null
+    state = null
+  })
+}
+
+async function _flushValidations(state) {
+  if (!state?.validationsByTask.size) return
+
+  const tasks = []
+  for (const [, entry] of state.validationsByTask) {
+    const { passes, traceId } = entry
+    if (!passes.length) continue
+
+    const success_rate = passes.every(Boolean) ? 1 : 0
+    const output_correctness = passes.filter(Boolean).length / passes.length
+    const codeOpts = { sourceType: "CODE" }
+
+    if (state.mlflowRunId) {
+      tasks.push(
+        logMlflowMetrics(state.mlflowRunId, { success_rate, output_correctness }).catch(() => {}),
+      )
+    }
+
+    if (traceId) {
+      tasks.push(
+        flushMlflowTraces().then(() =>
+          Promise.all([
+            postMlflowAssessment(traceId, success_rate, "", "success_rate", null, codeOpts).catch(
+              () => {},
+            ),
+            postMlflowAssessment(
+              traceId,
+              output_correctness,
+              "",
+              "output_correctness",
+              null,
+              codeOpts,
+            ).catch(() => {}),
+          ]),
+        ),
+      )
+    }
+  }
+
+  await Promise.all(tasks)
+  state.validationsByTask.clear()
+}
+
+export function recordEvaluation(result, assessment = {}) {
+  if (!cds.env.agents?.mlflow) return
+  const { pass, score, comment, ...config } = assessment
+  // Conversation level evaluations shall not be included in the per task roll-up
+  if (pass !== undefined && !config.conversationLevel) _addValidation(result, pass)
+  if (score === undefined) return
+  return _postAssessmentScore(result, score, comment, config)
+}
+
+function _addValidation(result, pass) {
+  const state = result?._evalState ?? cds._activeEvalRun
+  if (!state || !result?.taskId) return
+  const key = result.taskId
+  if (!state.validationsByTask.has(key)) {
+    state.validationsByTask.set(key, { passes: [], traceId: result.traceId })
+  }
+  state.validationsByTask.get(key).passes.push(pass)
+}
+
+async function _postAssessmentScore(result, score, comment, config) {
+  const traceId = config.traceId ?? result?.traceId
+  if (!traceId) return
+
+  const opts = { sourceType: config.sourceType }
+  await flushMlflowTraces()
+
+  // conversationLevel: session assessment — post with session metadata only
+  if (config.conversationLevel) {
+    await postMlflowAssessment(
+      traceId,
+      score,
+      comment ?? "",
+      config.assessmentName,
+      config.model ?? null,
+      { ...opts, metadata: { "mlflow.trace.session": config.sessionId ?? "" } },
+    )
+  } else {
+    // Single-turn: post to this trace only, no session metadata
+    await postMlflowAssessment(
+      traceId,
+      score,
+      comment ?? "",
+      config.assessmentName,
+      config.model ?? null,
+      opts,
+    )
+  }
+}
+
+export async function logMlflowMetricsForResult(result, state = null) {
+  state = state ?? cds._activeEvalRun
+  if (!state?.mlflowRunId) return
+  await logMlflowMetrics(state.mlflowRunId, result.metrics).catch(() => {})
+}

package/lib/eval/index.js

@@ -0,0 +1,6 @@
+import { installEvalDescribe } from "./eval-describe.js"
+
+installEvalDescribe()
+
+export { Judge, matchToolCall } from "./Judge.js"
+export { evalRun } from "./eval-run.js"

package/lib/eval/metrics.js

@@ -0,0 +1,53 @@
+function _hrtimeToMs(hr) {
+  return hr[0] * 1000 + hr[1] / 1e6
+}
+
+/**
+ * @param {object[]} spans     Finished OTel spans from span-collector.js
+ * @returns {{ input_tokens, output_tokens, total_tokens, tool_call_count, latency_ms, cost_usd }}
+ */
+export function metricsFromSpans(spans) {
+  let input_tokens = 0
+  let output_tokens = 0
+  let tool_call_count = 0
+  let cost_usd = 0
+  let latency_ms = null
+
+  for (const span of spans) {
+    const attrs = span.attributes ?? {}
+    const op = attrs["gen_ai.operation.name"]
+
+    if (op === "chat") {
+      input_tokens += Number(attrs["gen_ai.usage.input_tokens"] ?? 0)
+      output_tokens += Number(attrs["gen_ai.usage.output_tokens"] ?? 0)
+
+      // mlflow.llm.cost is set by apps themselves
+      const rawCost = attrs["mlflow.llm.cost"]
+      if (rawCost) {
+        try {
+          const c = typeof rawCost === "string" ? JSON.parse(rawCost) : rawCost
+          cost_usd += c.total_cost ?? 0
+        } catch {
+          /* malformed — skip */
+        }
+      }
+    }
+
+    if (op === "execute_tool") {
+      tool_call_count++
+    }
+
+    if (op === "invoke_agent" && span.endTime && span.startTime) {
+      latency_ms = _hrtimeToMs(span.endTime) - _hrtimeToMs(span.startTime)
+    }
+  }
+
+  return {
+    input_tokens,
+    output_tokens,
+    total_tokens: input_tokens + output_tokens,
+    tool_call_count,
+    latency_ms,
+    cost_usd: cost_usd > 0 ? cost_usd : null,
+  }
+}

package/lib/eval/span-collector.js

@@ -0,0 +1,50 @@
+const _sessions = new Map()
+let _registered = false
+let _sessionCounter = 0
+
+const _processor = {
+  onStart() {},
+  onEnd(span) {
+    if (_sessions.size === 0) return
+    for (const session of _sessions.values()) {
+      session.spans.push(span)
+    }
+  },
+  async forceFlush() {},
+  async shutdown() {},
+}
+
+async function _ensureRegistered() {
+  if (_registered) return
+  _registered = true
+  try {
+    const { trace } = await import("@opentelemetry/api")
+    const provider = trace.getTracerProvider()
+    const delegate = provider.getDelegate?.() || provider
+    if (delegate.constructor?.name === "NoopTracerProvider") return
+    if (typeof delegate.addSpanProcessor === "function") {
+      delegate.addSpanProcessor(_processor)
+    } else if (Array.isArray(delegate._activeSpanProcessor?._spanProcessors)) {
+      delegate._activeSpanProcessor._spanProcessors.push(_processor)
+    }
+  } catch {
+    /* OTel not present */
+  }
+}
+
+function _openSession() {
+  const id = ++_sessionCounter
+  const session = { spans: [] }
+  _sessions.set(id, session)
+  return {
+    collect() {
+      _sessions.delete(id)
+      return session.spans
+    },
+  }
+}
+
+export async function startCollection() {
+  await _ensureRegistered()
+  return _openSession()
+}

package/lib/index.js

@@ -161,7 +161,8 @@ export default function A2AProtocolAdapter(srv, options = {}) {
   }
 
   router.get("/.well-known/agent-card.json", (req, res) => {
-    const url = proxyUrl || `${req.protocol}://${req.get("host")}${req.baseUrl}`
+    const proto = req.headers["x-forwarded-proto"]?.split(",")[0].trim() || req.protocol
+    const url = proxyUrl || `${proto}://${req.get("host")}${req.baseUrl}`
     // Regenerate agent card when feature toggles are active (annotations may differ)
     let card
     if (cds.context?.features && Object.keys(cds.context.features).length > 0) {

package/lib/models/aicore.js

@@ -1,11 +1,13 @@
 import cds from "@sap/cds"
 import { OrchestrationClient } from "@sap-ai-sdk/langchain"
-import { circuitBreaker, timeout } from "@sap-cloud-sdk/resilience"
+import { circuitBreaker, timeout } from "../utils/resilience.js"
 
 import { SystemMessage, ToolMessage, HumanMessage, AIMessage } from "@langchain/core/messages"
 import { ms4 } from "../utils/utils.js"
+import { syncSystemPrompt } from "../telemetry/mlflow/prompts.js"
 
 const LOG = cds.log("agents")
+const DEFAULT_STREAM_DELIMITERS = [".", "!", "?"]
 
 class _InstrumentedOrchestrationClient extends OrchestrationClient {
   constructor(name, options) {
@@ -18,6 +20,7 @@ class _InstrumentedOrchestrationClient extends OrchestrationClient {
     // and extra ReAct iterations — see PR #188 review).
     const params =
       options.params || (deepAgent ? { max_tokens: 4096, temperature: 0 } : cds.env.agents?.params)
+    const auditParams = params && typeof params === "object" ? { ...params } : params
 
     LOG.debug("Initializing LLM", { model, deepAgent: !!deepAgent })
 
@@ -42,15 +45,15 @@ class _InstrumentedOrchestrationClient extends OrchestrationClient {
       {
         promptTemplating: { model: { name: model, params } },
         ...(filtering && { filtering }),
+      },
+      {
         // `streaming` controls the SDK's auto-stream-and-concatenate in _generate()
         // (i.e. direct model.invoke() calls). It does NOT gate LangGraph token
         // streaming — that is driven by graph.stream(streamMode:["messages"]) via a
         // streaming callback handler + our overridden _streamResponseChunks, and
         // works for every agent regardless of this flag (deep or managed alike).
         // Default on; `streaming: false` opts out.
-        ...(streaming !== false ? { streaming: true } : {}),
-      },
-      {
+        streaming: streaming !== false,
         onFailedAttempt: (err) => {
           // Abort retries when circuit breaker is open (otherwise pRetry delays ~30-60s)
           if (err.code === "EOPENBREAKER" || err.message === "Breaker is open") {
@@ -62,7 +65,7 @@ class _InstrumentedOrchestrationClient extends OrchestrationClient {
       destination,
     )
     this.name = name
-    this.options = { ...options, params, contentFilter, flatten }
+    this.options = { ...options, params: auditParams, contentFilter, flatten }
   }
 
   async _generate(messages, opts, runManager) {
@@ -88,7 +91,7 @@ class _InstrumentedOrchestrationClient extends OrchestrationClient {
     opts = _withMiddleware(this, opts)
     const prepared = _prepareMessages(messages, { flatten, model, opts })
     const inputMessages = prepared.inputMessages
-    opts = prepared.opts
+    opts = withDefaultStreamDelimiters(prepared.opts)
 
     let turnHasToolCall = false
     for await (const chunk of super._streamResponseChunks(inputMessages, opts, runManager)) {
@@ -144,9 +147,23 @@ function _prepareMessages(messages, { flatten, model, opts }) {
       opts = { ...opts, tools }
     }
   }
+  syncSystemPrompt(inputMessages)
   return { inputMessages, opts }
 }
 
+export function withDefaultStreamDelimiters(opts = {}) {
+  return {
+    ...opts,
+    streamOptions: {
+      ...opts.streamOptions,
+      global: {
+        ...opts.streamOptions?.global,
+        delimiters: opts.streamOptions?.global?.delimiters ?? DEFAULT_STREAM_DELIMITERS,
+      },
+    },
+  }
+}
+
 // Claude currently only supports caching of type ephemeral. TTL can differ between 5min or 1h but
 // we use the 5min default
 const CACHE_CONTROL_EPHEMERAL = { type: "ephemeral" }

package/lib/models/anthropic.js

@@ -5,6 +5,7 @@ import os from 'node:os'
 import cds from '@sap/cds'
 
 const HOME = os.homedir() || process.env.HOME || process.env.USERPROFILE
+const local = file => file.replace(HOME,'~')
 const LOG = cds.log('agents')
 
 /**
@@ -15,27 +16,36 @@ const LOG = cds.log('agents')
  */
 export default class ChatAnthropicService extends ChatAnthropic {
   constructor (name, options) {
-    // REVISIT: may be better handled via options.credentials?
-    let config = { ...options, ...fromEnv() }
+    let config = { ...options, ...options?.credentials, ...fromEnv() }
     if (!config.anthropicApiUrl) config = {
       ...fromClaude() || fromOpencode(),
       ...config
     }
-    LOG.debug (`Using effective config for ChatAnthropic:`, config)
+    if (LOG._debug) {
+      const { kind, model, anthropicApiUrl, apiKey } = config
+      LOG.info (`Using effective config:`, {
+        kind,
+        model,
+        credentials: {
+          anthropicApiUrl,
+          apiKey: apiKey ? '***' : undefined
+        }
+      })
+    }
     super (config)
     this.name = name
     this.options = config
   }
 }
 
-function fromEnv (env = process.env) {
+function fromEnv (env = process.env, silent) {
   let any, config = {}
   if ((any = env.ANTHROPIC_BASE_URL)) config.anthropicApiUrl = any
   if ((any = env.ANTHROPIC_AUTH_TOKEN)) config.apiKey = any
   if ((any = env.ANTHROPIC_API_KEY)) config.apiKey = any
   if ((any = env.ANTHROPIC_MODEL)) config.model = any
   if (!Object.keys(config).length) return null
-  LOG.debug (`Loaded Anthropic settings from env:`, config)
+  if (!silent) LOG.debug (`Loaded Anthropic settings from env:`, config)
   return config
 }
 
@@ -43,12 +53,17 @@ function fromClaude() {
   if ('cached' in fromClaude) return fromClaude.cached
   const settings_json = path.join (HOME,'.claude/settings.json')
   try {
-    let conf = JSON.parse (fs.readFileSync (settings_json,'utf8'))
+    let settings = JSON.parse (fs.readFileSync (settings_json,'utf8'))
     // https://www.schemastore.org/claude-code-settings.json
-    fromClaude.cached = conf = fromEnv (conf?.env)
-    LOG.debug(`Loaded Claude settings from`, settings_json, ':', conf)
+
+    let conf = fromClaude.cached = fromEnv (settings?.env, 'silent')
+    if (!conf.model && settings.env) {
+      let family = (settings.model||'sonnet').toUpperCase()
+      conf.model = settings.env[`ANTHROPIC_DEFAULT_${family}_MODEL`] || settings?.model
+    }
+    LOG.debug(`Loaded Claude settings from`, local(settings_json), ':', conf)
   } catch {
-    LOG.debug(`Failed loading Claude settings from`, settings_json)
+    LOG.debug(`Failed loading Claude settings from`, local(settings_json))
     fromClaude.cached = null
   }
   return fromClaude.cached
@@ -59,7 +74,7 @@ function fromOpencode() {
   const opencode_json = path.join (HOME,'.config/opencode/opencode.json')
   try {
     let conf = JSON.parse (fs.readFileSync (opencode_json,'utf8'))
-    LOG.debug(`Loaded OpenCode settings from`, opencode_json)
+    LOG.debug(`Loaded OpenCode settings from`, local(opencode_json))
     // https://opencode.ai/config.json
     let o = conf?.provider?.anthropic?.options
     if (!o) return fromOpencode.cached = null
@@ -68,9 +83,9 @@ function fromOpencode() {
     if ((any = o.anthropicApiKey ?? o.apiKey)) config.apiKey = any
     if ((any = conf?.model)) config.model = any.replace('anthropic/','')
     fromOpencode.cached = Object.keys(config).length ? config : null
-    LOG.debug(`Loaded OpenCode settings from`, opencode_json, ':', conf)
+    LOG.debug(`Loaded OpenCode settings from`, local(opencode_json), ':', conf)
   } catch {
-    LOG.debug(`Failed loading OpenCode settings from`, opencode_json)
+    LOG.debug(`Failed loading OpenCode settings from`, local(opencode_json))
     fromOpencode.cached = null
   }
   return fromOpencode.cached