@cap-js/agents 0.9.2 → 0.9.3

package/lib/telemetry/chat-tracing.js

@@ -1,12 +1,23 @@
 import cds from "@sap/cds"
 import * as metrics from "./metrics.js"
-import { mlflowAttrs, setSpanAttrs } from "./mlflow.js"
+import { mlflowAttrs, setSpanAttrs } from "./mlflow/index.js"
 import { audit } from "../utils/utils.js"
 
 const LOG = cds.log("agents")
 
 const PATCHED = Symbol.for("@cap-js/agents:patched")
 const SPAN_KIND_CLIENT = 3
+const MODEL_PARAM_KEYS = [
+  "max_tokens",
+  "temperature",
+  "frequency_penalty",
+  "presence_penalty",
+  "top_p",
+  "n",
+  "stop",
+  "reasoning_effort",
+  "tool_choice",
+]
 
 // ─── Public API ──────────────────────────────────────────────────────────────
 
@@ -48,7 +59,7 @@ export function _patchChatModelProto(proto) {
     if (!tracer) return original.call(this, input, opts)
 
     const model = this.options?.model || this.model || this.constructor.name
-    const provider = _detectProvider(this)
+    const provider = _resolveProvider()
     const node = opts?.runName || "agent"
     const messages = Array.isArray(input) ? input : undefined
     const cacheControl = _detectCacheControl(messages)
@@ -135,10 +146,11 @@ export function _patchChatModelProto(proto) {
 
 // ─── Internal helpers ────────────────────────────────────────────────────────
 
-/** Detect LLM provider from model instance properties. */
-function _detectProvider(instance) {
-  if (instance.orchestrationConfig) return "sap-ai-core"
-  return "langchain"
+/** Resolve LLM provider from cds.env.requires. */
+function _resolveProvider() {
+  const kind = cds.env.requires?.llm?.kind || "langchain"
+  if (kind === "ai-core" || kind === "aicore") return "sap-ai-core"
+  return kind.replace(/^llm-/, "")
 }
 
 function _metricAttrs(model, node) {
@@ -184,15 +196,19 @@ function _handleSuccess(
 
   const taskId = opts?.configurable?._taskId || cds.context?.["agent.task.id"]
   if (taskId) {
+    const modelParams = pickModelParams(params)
     audit("AgentDecision", {
       data: {
         taskId,
         contextId:
           opts?.configurable?.thread_id?.split(":")[1] || cds.context?.["agent.context.id"],
         service: opts?.configurable?._service || cds.context?.["agent.service"],
-        model,
+        provider,
+        model: response?.model,
+        ...(modelParams ? { modelParams } : {}),
+        finishReason: response?.finishReason,
         iteration: opts?.configurable?._iteration ?? cds.context?.["agent.iteration"],
-        toolCalls: response?.toolCalls,
+        toolCalls: response?.toolCalls.map((t) => t.name),
         tokenUsage,
         duration,
       },
@@ -200,6 +216,21 @@ function _handleSuccess(
   }
 }
 
+function pickModelParams(params) {
+  if (!params || typeof params !== "object" || Array.isArray(params)) return undefined
+  const result = {}
+  for (const key of MODEL_PARAM_KEYS) {
+    const value = params[key]
+    if (value === undefined || typeof value === "function" || typeof value === "symbol") continue
+    try {
+      result[key] = structuredClone(value)
+    } catch {
+      result[key] = String(value)
+    }
+  }
+  return Object.keys(result).length ? result : undefined
+}
+
 /** Handle LLM error: content-filter warnings, metrics, span error attrs. */
 function _handleError(span, { err, mAttrs, model, node, messages }) {
   const status = err.rootCause?.status

package/lib/telemetry/mlflow/credentials.js

@@ -0,0 +1,79 @@
+import cds from "@sap/cds"
+
+/** @returns {{ host: string, uc: {catalog,schema,tablePrefix}|null, warehouseId: string|undefined, getAuthHeaders: () => Promise<Record<string,string>> } | null} */
+export function resolveMlflowCredentials() {
+  const creds = cds.env.requires?.mlflow?.credentials || {}
+  const host = (creds.MLFLOW_HOST || "").replace(/\/$/, "")
+  if (!host) return null
+
+  const ucCatalog = creds.UC_CATALOG
+  const ucSchema = creds.UC_SCHEMA
+  const ucTablePrefix = creds.UC_TABLE_PREFIX
+  const uc =
+    ucCatalog && ucSchema && ucTablePrefix
+      ? { catalog: ucCatalog, schema: ucSchema, tablePrefix: ucTablePrefix }
+      : null
+
+  const warehouseId = creds.DATABRICKS_SQL_WAREHOUSE_ID || undefined
+
+  const getAuthHeaders = _buildAuthHeaderFactory(creds)
+
+  return { host, uc, warehouseId, getAuthHeaders }
+}
+
+// Resolution order: @Core.SchemaVersion annotation → credentials → env.
+// MLflow experiment IDs must be numeric strings (int64).
+export function resolveExperimentId() {
+  const srvName = cds.context?.["agent.service"]
+  if (srvName) {
+    const def = cds.context?.model?.definitions?.[srvName] || cds.services?.[srvName]?.definition
+    const annotated = def?.["@Core.SchemaVersion"]
+    if (annotated) return String(annotated)
+  }
+  const creds = cds.env.requires?.mlflow?.credentials || {}
+  return creds.MLFLOW_EXPERIMENT_ID || process.env.MLFLOW_EXPERIMENT_ID || null
+}
+
+/**
+ * Build an async factory that returns { Authorization: "Bearer ..." } headers.
+ * OAuth tokens are cached and refreshed 60 s before expiry.
+ * Returns an empty object factory when no auth is configured (local MLflow).
+ */
+function _buildAuthHeaderFactory(creds) {
+  if (creds.clientid && creds.clientsecret) {
+    const tokenUrl = `${(creds.url || creds.MLFLOW_HOST || "").replace(/\/$/, "")}/oidc/v1/token`
+    let cached = null
+    let expiresAt = 0
+    return async function fetchOAuthHeaders() {
+      if (cached && Date.now() < expiresAt) return cached
+      const res = await fetch(tokenUrl, {
+        method: "POST",
+        headers: {
+          Authorization:
+            "Basic " + Buffer.from(`${creds.clientid}:${creds.clientsecret}`).toString("base64"),
+          "Content-Type": "application/x-www-form-urlencoded",
+        },
+        body: "grant_type=client_credentials&scope=all-apis",
+        signal: AbortSignal.timeout(10_000),
+      })
+      if (!res.ok) {
+        const body = await res.text().catch(() => "")
+        throw new Error(
+          `MLflow OAuth token fetch failed: HTTP ${res.status} — ${body.slice(0, 200)}`,
+        )
+      }
+      const { access_token, expires_in } = await res.json()
+      cached = { Authorization: `Bearer ${access_token}` }
+      expiresAt = Date.now() + (expires_in - 60) * 1000
+      return cached
+    }
+  }
+
+  if (creds.MLFLOW_TOKEN) {
+    const headers = { Authorization: `Bearer ${creds.MLFLOW_TOKEN}` }
+    return async () => headers
+  }
+
+  // Local / unauthenticated MLflow
+  return async () => ({})
+}

package/lib/telemetry/mlflow/evaluation.js

@@ -0,0 +1,38 @@
+import cds from "@sap/cds"
+import { getMlflowExporter } from "./exporter/index.js"
+
+export async function createEvalRun({ name } = {}) {
+  const exporter = getMlflowExporter()
+  if (!exporter) return null
+  const creds = cds.env.requires?.mlflow?.credentials || {}
+  const experimentId = creds.MLFLOW_EXPERIMENT_ID || process.env.MLFLOW_EXPERIMENT_ID || "0"
+  return exporter.createRun(experimentId, name)
+}
+
+export async function closeEvalRun(runId) {
+  if (!runId) return
+  getMlflowExporter()?.closeRun(runId)
+}
+
+// Log a flat metrics object; null/undefined values are skipped.
+export async function logMlflowMetrics(runId, metrics) {
+  if (!runId) return
+  const exporter = getMlflowExporter()
+  if (!exporter) return
+  await Promise.allSettled(
+    Object.entries(metrics)
+      .filter(([, v]) => v != null)
+      .map(([key, value]) => exporter.logMetric(runId, key, value)),
+  )
+}
+
+export async function postMlflowAssessment(
+  traceId,
+  score,
+  rationale,
+  assessmentName,
+  sourceId,
+  opts,
+) {
+  getMlflowExporter()?.postAssessment(traceId, score, rationale, assessmentName, sourceId, opts)
+}

package/lib/telemetry/mlflow/exporter/DatabricksExporter.js

@@ -0,0 +1,75 @@
+import { MlflowExporter } from "./MlflowExporter.js"
+
+export class DatabricksExporter extends MlflowExporter {
+  async postAssessment(
+    traceId,
+    score,
+    rationale,
+    assessmentName,
+    sourceId,
+    { metadata, sourceType } = {},
+  ) {
+    const { uc, warehouseId } = this._creds
+    const ucPrefix = `${uc.catalog}.${uc.schema}.${uc.tablePrefix}`
+    let url = `/api/4.0/mlflow/traces/${encodeURIComponent(ucPrefix)}/tr-${traceId}/assessments`
+    if (warehouseId) url += `?sql_warehouse_id=${encodeURIComponent(warehouseId)}`
+    await this._fetch(url, {
+      trace_id: traceId,
+      assessment_name: assessmentName,
+      source: { source_type: sourceType ?? "LLM_JUDGE", source_id: sourceId ?? null },
+      feedback: { value: score },
+      rationale: String(rationale),
+      ...(metadata && { metadata }),
+      trace_location: {
+        type: "UC_TABLE_PREFIX",
+        uc_table_prefix: {
+          catalog_name: uc.catalog,
+          schema_name: uc.schema,
+          table_prefix: uc.tablePrefix,
+        },
+      },
+    })
+  }
+
+  // Returns { tags: [{key,value}], latestVersion: {version, tags} | null }.
+  async ensurePrompt(name, description) {
+    let res = await this._fetch(
+      `/mlflow/unity-catalog/prompts/${encodeURIComponent(name)}`,
+      undefined,
+      "GET",
+    )
+    if (!res) {
+      res = await this._fetch("/mlflow/unity-catalog/prompts", { name, prompt: { description } })
+    }
+    const latestVersion = await this._getLatestUcVersion(name)
+    return { tags: res?.tags ?? [], latestVersion }
+  }
+
+  async createPromptVersion(name, description, tags = [], template = "") {
+    const res = await this._fetch(
+      `/mlflow/unity-catalog/prompts/${encodeURIComponent(name)}/versions`,
+      {
+        prompt_version: { template, description, tags },
+      },
+    )
+    return res?.version ? String(res.version) : null
+  }
+
+  async setRegisteredModelTag(name, key, value) {
+    await this._fetch(`/mlflow/unity-catalog/prompts/${encodeURIComponent(name)}/tags`, {
+      key,
+      value,
+    })
+  }
+
+  // Returns { version, tags } of the latest UC prompt version, or null.
+  async _getLatestUcVersion(name) {
+    const res = await this._fetch(
+      `/mlflow/unity-catalog/prompts/${encodeURIComponent(name)}/versions/search`,
+      { max_results: 1 },
+    )
+    const pv = res?.prompt_versions?.[0]
+    if (!pv) return null
+    return { version: String(pv.version), tags: pv.tags ?? [] }
+  }
+}

package/lib/telemetry/mlflow/exporter/MlflowExporter.js

@@ -0,0 +1,115 @@
+import cds from "@sap/cds"
+
+const LOG = cds.log("agents")
+
+export class MlflowExporter {
+  constructor(creds) {
+    this._creds = creds
+  }
+
+  async _fetch(path, body, method = "POST") {
+    const { host, getAuthHeaders } = this._creds
+    const headers = { ...(await getAuthHeaders()), "Content-Type": "application/json" }
+    try {
+      const res = await fetch(`${host}${path}`, {
+        method,
+        headers,
+        ...(body !== undefined && { body: JSON.stringify(body) }),
+      })
+      if (!res.ok) {
+        LOG.error(
+          `[mlflow] ${method} ${path} → ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`,
+        )
+        return null
+      }
+      return res.json().catch(() => null)
+    } catch (err) {
+      LOG.debug(`[mlflow] ${path} error: ${err.message}`)
+      return null
+    }
+  }
+
+  async createRun(experimentId, name) {
+    const data = await this._fetch("/api/2.0/mlflow/runs/create", {
+      experiment_id: experimentId,
+      run_name: name || `eval-${new Date().toISOString()}`,
+      start_time: Date.now(),
+      tags: [{ key: "mlflow.source.type", value: "LOCAL" }],
+    })
+    return data?.run?.info?.run_id ?? null
+  }
+
+  async closeRun(runId) {
+    await this._fetch("/api/2.0/mlflow/runs/update", {
+      run_id: runId,
+      status: "FINISHED",
+      end_time: Date.now(),
+    })
+  }
+
+  async logMetric(runId, key, value) {
+    await this._fetch("/api/2.0/mlflow/runs/log-metric", {
+      run_id: runId,
+      key,
+      value,
+      timestamp: Date.now(),
+      step: 0,
+    })
+  }
+
+  async postAssessment(
+    traceId,
+    score,
+    rationale,
+    assessmentName,
+    sourceId,
+    { metadata, sourceType } = {},
+  ) {
+    await this._fetch(`/api/3.0/mlflow/traces/tr-${traceId}/assessments`, {
+      assessment: {
+        trace_id: traceId,
+        assessment_name: assessmentName,
+        source: { source_type: sourceType ?? "LLM_JUDGE", source_id: sourceId ?? null },
+        feedback: { value: score },
+        rationale: String(rationale),
+        ...(metadata && { metadata }),
+      },
+    })
+  }
+
+  // Returns { tags: [{key,value}], latestVersion: {version, tags} | null }.
+  async ensurePrompt(name, description, registrationTags = []) {
+    let res = await this._fetch(
+      `/api/2.0/mlflow/registered-models/get?name=${encodeURIComponent(name)}`,
+      undefined,
+      "GET",
+    )
+    if (!res) {
+      res = await this._fetch("/api/2.0/mlflow/registered-models/create", {
+        name,
+        description,
+        tags: registrationTags,
+      })
+    }
+    const rm = res?.registered_model
+    const lv = rm?.latest_versions?.[0]
+    return {
+      tags: rm?.tags ?? [],
+      latestVersion: lv ? { version: String(lv.version), tags: lv.tags ?? [] } : null,
+    }
+  }
+
+  async createPromptVersion(name, description, tags = [], _template = "") {
+    const res = await this._fetch("/api/2.0/mlflow/model-versions/create", {
+      name,
+      source: "dummy-source", // required by API but unused for prompts
+      description,
+      tags,
+    })
+    return res?.model_version?.version ? String(res.model_version.version) : null
+  }
+
+  async setRegisteredModelTag(name, key, value) {
+    await this._fetch("/api/2.0/mlflow/registered-models/set-tag", { name, key, value })
+  }
+}

package/lib/telemetry/mlflow/exporter/index.js

@@ -0,0 +1,23 @@
+import cds from "@sap/cds"
+import { resolveMlflowCredentials } from "../credentials.js"
+import { MlflowExporter } from "./MlflowExporter.js"
+import { DatabricksExporter } from "./DatabricksExporter.js"
+
+export { MlflowExporter } from "./MlflowExporter.js"
+export { DatabricksExporter } from "./DatabricksExporter.js"
+
+let _instance = null
+let _instanceHost = null // track which host the singleton was built for
+
+// Returns the singleton exporter, rebuilding it when credentials change.
+// Returns null when MLflow is disabled or credentials are missing.
+export function getMlflowExporter() {
+  if (!cds.env.agents?.mlflow) return null
+  const creds = resolveMlflowCredentials()
+  if (!creds) return null
+  // Rebuild if the host changed (e.g. between tests or env reloads)
+  if (_instance && _instanceHost === creds.host) return _instance
+  _instance = creds.uc ? new DatabricksExporter(creds) : new MlflowExporter(creds)
+  _instanceHost = creds.host
+  return _instance
+}

package/lib/telemetry/mlflow/index.js

@@ -0,0 +1,16 @@
+export {
+  mlflowAttrs,
+  mlflowTraceAttrs,
+  setSpanAttrs,
+  setupMlflowExporter,
+  flushMlflowTraces,
+  RoutingSpanProcessor,
+} from "./tracing.js"
+export { postMlflowAssessment, createEvalRun, closeEvalRun } from "./evaluation.js"
+export {
+  syncPromptVersion,
+  syncSystemPrompt,
+  resolvePromptName,
+  linkedPromptsAttr,
+  hashPrompt,
+} from "./prompts.js"

package/lib/telemetry/mlflow/prompts.js

@@ -0,0 +1,123 @@
+import cds from "@sap/cds"
+import { createHash } from "node:crypto"
+import { join, relative } from "node:path"
+import { getMlflowExporter } from "./exporter/index.js"
+import { resolveAgentDir } from "../../utils/markdown.js"
+import { resolveExperimentId } from "./credentials.js"
+
+const _cache = new Map() // promptName → { version, hash }
+
+// Register or reuse the prompt version in MLflow. Cache hit skips all I/O.
+export async function syncPromptVersion(name, template) {
+  const exporter = getMlflowExporter()
+  if (!exporter) return null
+
+  const hash = hashPrompt(template)
+  const cached = _cache.get(name)
+  if (cached?.hash === hash) return { name, version: cached.version }
+
+  try {
+    const prompt = await exporter.ensurePrompt(name, _description(name), _registrationTags())
+    let version
+    if (_extractHash(prompt?.latestVersion?.tags) === hash) {
+      version = prompt?.latestVersion?.version
+    } else {
+      version = await exporter.createPromptVersion(
+        name,
+        `Hash ${hash}`,
+        _versionTags(template, hash),
+        template,
+      )
+    }
+    if (!version) return null
+    _cache.set(name, { hash, version })
+    _linkToExperiment(exporter, name, prompt?.tags).catch(() => {})
+    return { name, version }
+  } catch {
+    return null
+  }
+}
+
+// JSON for mlflow.traceTag.mlflow.linkedPrompts span attribute, or null if not cached.
+export function linkedPromptsAttr(promptName) {
+  const cached = _cache.get(promptName)
+  if (!cached) return null
+  return JSON.stringify([{ name: promptName, version: cached.version }])
+}
+
+// Extracts the SystemMessage from prepared LLM messages and syncs it to MLflow.
+export function syncSystemPrompt(messages) {
+  if (!cds.env.agents?.mlflow) return
+  const srvName = cds.context?.["agent.service"]
+  if (!srvName) return
+  const sysMsg = messages?.find((m) => m.type === "system")
+  if (!sysMsg) return
+  const text =
+    typeof sysMsg.content === "string"
+      ? sysMsg.content
+      : Array.isArray(sysMsg.content)
+        ? sysMsg.content
+            .filter((b) => b?.type === "text")
+            .map((b) => b.text ?? "")
+            .join("")
+        : null
+  if (!text) return
+  const srv = cds.services[srvName]
+  if (!srv) return
+  syncPromptVersion(resolvePromptName(srv), text).catch(() => {})
+}
+
+// AGENTS.md path relative to cds.root, or srv.name when no AGENTS.md exists.
+export function resolvePromptName(srv) {
+  if (!srv?.name) return ""
+  const agentDir = resolveAgentDir(srv)
+  if (!agentDir) return srv.name
+  const root = cds.root ?? process.cwd()
+  return relative(root, join(agentDir, "AGENTS.md")).replace(/\\/g, "/")
+}
+
+export function hashPrompt(text) {
+  return createHash("sha256").update(text).digest("hex").slice(0, 16)
+}
+
+export function getCachedPromptVersion(name) {
+  return _cache.get(name) ?? null
+}
+
+function _description(name) {
+  return `System prompt for CAP agent service "${name.split("/")[0]}"`
+}
+
+function _registrationTags() {
+  return [{ key: "mlflow.prompt.is_prompt", value: "true" }]
+}
+
+function _versionTags(template, hash) {
+  return [
+    { key: "mlflow.prompt.is_prompt", value: "true" }, // triggers prompt path on server → allows dummy-source
+    { key: "mlflow.prompt.text", value: template },
+    { key: "_mlflow_prompt_type", value: "text" },
+    { key: "_cap_prompt_hash", value: hash },
+  ]
+}
+
+function _extractHash(tags) {
+  return tags?.find((t) => t.key === "_cap_prompt_hash")?.value ?? null
+}
+
+// Append experiment ID to the model-level "_mlflow_experiment_ids" tags so prompt appears in UI prompts tab.
+async function _linkToExperiment(exporter, name, promptTags) {
+  const experimentId = resolveExperimentId()
+  if (!experimentId) return
+  const current = promptTags?.find((t) => t.key === "_mlflow_experiment_ids")?.value
+  const ids = current
+    ? current
+        .replace(/^,|,$/g, "")
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean)
+    : []
+  if (ids.includes(String(experimentId))) return
+  ids.push(String(experimentId))
+  await exporter.setRegisteredModelTag(name, "_mlflow_experiment_ids", `,${ids.join(",")},`)
+}