@cap-js/agents 0.9.2 → 0.9.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js/agents",
-  "version": "0.9.2",
+  "version": "0.9.4",
   "description": "CDS plugin for building agents",
   "author": "SAP SE (https://www.sap.com)",
   "license": "Apache-2.0",
@@ -11,16 +11,14 @@
   },
   "type": "module",
   "exports": {
-    ".": "./index.js",
     "./package.json": "./package.json",
     "./cds-plugin": "./cds-plugin.js",
-    "./index.cds": "./index.cds",
+    "./eval": "./lib/eval/index.js",
+    "./lib/index.cjs": "./lib/index.cjs",
     "./lib/models/*": "./lib/models/*.js",
-    "./srv/langgraph-executor-srv": "./srv/langgraph-executor-srv.js",
-    "./srv/*": "./srv/*",
-    "./lib/*": "./lib/*"
+    "./srv/entities.cds": "./srv/entities.cds",
+    "./srv/push-notification-srv.js": "./srv/push-notification-srv.js"
   },
-  "main": "index.js",
   "scripts": {
     "lint": "npx eslint .",
     "test": "CDS_ENV=test npx vitest run --config vitest.config.js",
@@ -30,15 +28,15 @@
     "watch:hybrid": "DEBUG=agents cds bind --exec -- cds w tests/projects/bookshop",
     "watch:with-claude": "cds w tests/projects/bookshop --profile with-claude",
     "watch:deep-agent": "cds w tests/projects/deep-agent --profile hybrid",
+    "docs:audit": "node .scripts/generate-audit-docs.js",
+    "docs:audit:check": "node .scripts/generate-audit-docs.js --check",
     "prettier": "npx -y prettier@3 --write .",
     "prettier:check": "npx -y prettier@3 --check ."
   },
   "files": [
-    "index.js",
-    "index.cds",
+    "cds-plugin.js",
     "lib",
     "srv",
-    "cds-plugin.js",
     "_i18n"
   ],
   "dependencies": {
@@ -52,7 +50,6 @@
     "@sap-ai-sdk/langchain": "^2.13.0",
     "@sap-ai-sdk/orchestration": "^2.11.0",
     "@sap-cloud-sdk/connectivity": "^4.7.0",
-    "@sap-cloud-sdk/resilience": "^4.7.0",
     "deepagents": "^1.10.0",
     "langchain": "^1.5.0",
     "marked": "^18",
@@ -63,12 +60,15 @@
     "@cap-js/sqlite": "^3",
     "@sap/cds-mtxs": "^4",
     "@toon-format/toon": ">=2.3",
-    "deepagents": "^1.10.2"
+    "acorn": "^8.18.0",
+    "deepagents": "^1.10.2",
+    "openevals": "^0.2.0"
   },
   "peerDependencies": {
     "@cap-js/audit-logging": ">=1",
     "@cap-js/telemetry": ">=1",
-    "@sap/cds": ">=9"
+    "@sap/cds": ">=9",
+    "openevals": ">=0.2"
   },
   "peerDependenciesMeta": {
     "@cap-js/telemetry": {
@@ -76,6 +76,9 @@
     },
     "@cap-js/audit-logging": {
       "optional": true
+    },
+    "openevals": {
+      "optional": true
     }
   },
   "engines": {
@@ -113,6 +116,12 @@
       },
       "persistAllCheckpointWrites": false,
       "activeUsersInterval": "24h",
+      "circuitBreaker": {
+        "errorThresholdPercentage": 50,
+        "volumeThreshold": 10,
+        "resetTimeout": 30000,
+        "rollingCountTimeout": 10000
+      },
       "params": {
         "max_tokens": 4096,
         "temperature": 0
@@ -156,7 +165,7 @@
     },
     "requires": {
       "agents": {
-        "model": "@cap-js/agents"
+        "model": "@cap-js/agents/srv/entities.cds"
       },
       "agent-push-notifications": {
         "impl": "@cap-js/agents/srv/push-notification-srv.js"
@@ -208,6 +217,9 @@
           "_javaHcqlCompat": true
         }
       }
+    },
+    "folders": {
+      "srvs": "srv/*"
     }
   },
   "workspaces": [

package/srv/handlers/chat.js

@@ -0,0 +1,278 @@
+import cds from "@sap/cds"
+import { startCollection } from "../../lib/eval/span-collector.js"
+import { metricsFromSpans } from "../../lib/eval/metrics.js"
+import { getActiveRunState, logMlflowMetricsForResult } from "../../lib/eval/eval-run.js"
+
+export const COLLECT_RESULT = Symbol.for("@cap-js/agents:chat:collect-result")
+
+class NoopEventBus {
+  constructor() {
+    this.events = []
+    this[COLLECT_RESULT] = true
+    this._graphResult = null
+    this._done = false
+    this.finished$ = new Promise((resolve, reject) => {
+      this._resolve = resolve
+      this._reject = reject
+    })
+  }
+
+  publish(event) {
+    this.events.push(event)
+  }
+
+  finished() {
+    this._done = true
+    this._resolve?.()
+  }
+
+  error(err) {
+    this._reject?.(err)
+  }
+
+  getFinalText() {
+    for (let i = this.events.length - 1; i >= 0; i--) {
+      const e = this.events[i]
+      if (e.kind === "artifact-update" && e.lastChunk === true) {
+        const text = e.artifact?.parts
+          ?.filter((p) => p.kind === "text")
+          .map((p) => p.text)
+          .join("")
+        if (text) return text
+      }
+    }
+    for (let i = this.events.length - 1; i >= 0; i--) {
+      const e = this.events[i]
+      if (e.kind === "status-update" && e.status?.message?.parts) {
+        const text = e.status.message.parts
+          .filter((p) => p.kind === "text")
+          .map((p) => p.text)
+          .join("")
+        if (text) return text
+      }
+    }
+    return ""
+  }
+
+  getStatus() {
+    const e = this.events.at(-1)
+    if (e && e.kind === "status-update" && e.status?.state) {
+      const state = e.status.state
+      const msg = e.status?.message?.parts?.find((p) => p.kind === "text")?.text ?? ""
+      return { status: state, description: msg }
+    }
+    return { status: "completed", description: "" }
+  }
+}
+
+// Derive toolCalls from graph messages by pairing AIMessage.tool_calls with ToolMessage results.
+function toolCallsFromMessages(messages) {
+  if (!Array.isArray(messages) || messages.length === 0) return []
+
+  const resultById = new Map()
+  for (const msg of messages) {
+    if (msg.tool_call_id && msg.type === "tool") {
+      const content = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
+      resultById.set(msg.tool_call_id, { content, ...msg })
+    }
+  }
+
+  const entries = []
+  for (const msg of messages) {
+    if (!Array.isArray(msg.tool_calls) || msg.tool_calls.length === 0) continue
+    for (const tc of msg.tool_calls) {
+      const raw = resultById.get(tc.id)
+      let toolResult
+      try {
+        toolResult = raw?.content !== undefined ? JSON.parse(raw.content) : undefined
+      } catch {
+        toolResult = raw?.content
+      }
+      const entry = {
+        tool: tc.name,
+        args: tc.args ?? {},
+        outcome: raw.status,
+        ...(toolResult !== undefined && { result: toolResult }),
+      }
+      attachCqn(entry)
+      entries.push(entry)
+    }
+  }
+  return entries
+}
+
+/**
+ * If entry.args.cql attaches cqn as hidden property.
+ * No-op when cql is absent or faulty.
+ */
+function attachCqn(entry) {
+  const cql = entry.args?.cql
+  if (typeof cql !== "string") return
+  try {
+    const cqn = cds.parse.cql(cql)
+    Object.defineProperty(entry, "cqn", {
+      value: cqn,
+      enumerable: false,
+      writable: false,
+      configurable: true,
+    })
+  } catch {
+    /* unparseable CQL — skip */
+  }
+}
+
+function buildRequestContext(query, opts = {}) {
+  const taskId = opts.taskId || cds.utils.uuid()
+  const contextId = opts.contextId || cds.utils.uuid()
+  const parts =
+    opts.parts ??
+    (typeof query === "string"
+      ? [{ kind: "text", text: query }]
+      : (query?.parts ?? [{ kind: "text", text: String(query) }]))
+  return {
+    taskId,
+    contextId,
+    task: opts.task ?? null,
+    userMessage: {
+      kind: "message",
+      messageId: cds.utils.uuid(),
+      role: "user",
+      taskId,
+      contextId,
+      parts,
+    },
+  }
+}
+
+/** Install srv.chat(query, previous?) on an @agent service for eval tests. */
+export function registerChat(srv) {
+  srv.chat = async function chat(query, previous) {
+    // Resolve the small eval helper API:
+    //   chat(query)
+    //   chat(query, prevResult) — extract contextId + taskId for HITL resume
+    //   chat(query, { _details: true }) — internal/test escape hatch outside test profile
+    let opts = {}
+    if (typeof previous === "string") {
+      throw new TypeError(
+        "agent.chat: second argument must be a previous chat result object or options object",
+      )
+    } else if (previous && typeof previous === "object") {
+      if ("text" in previous || "contextId" in previous) {
+        // prior chat() result — extract conversation ids and HITL state
+        opts = {
+          contextId: previous.contextId,
+          // mark as resume when prior result was input-required
+          ...(previous.status === "input-required" && {
+            taskId: previous.taskId,
+            task: { id: previous.taskId, status: { state: "input-required" } },
+          }),
+        }
+      } else {
+        opts = { _details: previous._details === true }
+      }
+    }
+
+    const includeDetails = shouldIncludeChatDetails(opts)
+    const runState = includeDetails ? getActiveRunState() : null
+
+    // Open span collection session before execution so the processor
+    // captures all spans from this trace.
+    const collection = includeDetails ? await startCollection() : null
+
+    const { LangGraphExecutor } = await import("../langgraph-executor-srv.js")
+    const executor = LangGraphExecutor.for(srv)
+    const requestContext = buildRequestContext(query, opts)
+    const eventBus = new NoopEventBus()
+    const mlflowRunId = runState?.mlflowRunId
+    let traceId
+
+    const runInContext = async () => {
+      // Link the trace to the MLflow eval run — graph-executor reads this to set mlflow.sourceRun.
+      if (mlflowRunId && cds.context) cds.context["_mlflow.evalRunId"] = mlflowRunId
+
+      let execError = null
+      const execPromise = executor.execute(requestContext, eventBus).catch((err) => {
+        execError = err
+        if (!eventBus._done) eventBus.finished()
+      })
+      await Promise.race([eventBus.finished$, execPromise])
+      if (!eventBus._done) eventBus.finished()
+      if (execError) throw execError
+
+      if (includeDetails) {
+        const rootSpan = cds.context?.["_mlflow.rootSpan"]
+        if (rootSpan) traceId = rootSpan.spanContext?.()?.traceId
+      }
+    }
+
+    if (cds.context) {
+      await runInContext()
+    } else {
+      await cds._with(
+        new cds.EventContext({ tenant: "t0", user: new cds.User.Privileged() }),
+        runInContext,
+      )
+    }
+
+    const { status, description } = eventBus.getStatus()
+    if (status === "failed")
+      throw new Error(`agent.chat: task failed — ${description || "no message"}`)
+
+    const text = eventBus.getFinalText()
+    const result = {
+      text,
+      contextId: requestContext.contextId,
+      taskId: requestContext.taskId,
+      status,
+    }
+
+    if (includeDetails) {
+      const allMessages = eventBus._graphResult?.messages ?? []
+
+      // Find last HumanMessage whose text matches current query; return from there.
+      const queryText = String(query)
+      let turnStart = 0
+      for (let i = allMessages.length - 1; i >= 0; i--) {
+        const m = allMessages[i]
+        if (m.getType?.() === "human" || m._getType?.() === "human" || m.type === "human") {
+          const content = typeof m.content === "string" ? m.content : (m.content?.[0]?.text ?? "")
+          if (content.startsWith(queryText) || queryText.startsWith(content.trim())) {
+            turnStart = i
+            break
+          }
+        }
+      }
+      const messages = allMessages.slice(turnStart)
+      const toolCalls = toolCallsFromMessages(messages)
+
+      const allSpans = collection.collect()
+      const spans = traceId
+        ? allSpans.filter((s) => s.spanContext?.()?.traceId === traceId)
+        : allSpans
+
+      result.query = String(query)
+      result.traceId = traceId
+      result.toolCalls = toolCalls
+      result.messages = messages.map((m) => {
+        // Eval prompts expect text content, not content-block objects.
+        if (m.type === "ai") {
+          if (Array.isArray(m.content) && m.content[0]?.text) {
+            m.content = m.content[0].text
+          }
+        }
+        return m
+      })
+      result.spans = spans
+      result.metrics = metricsFromSpans(spans)
+      if (runState) result._evalState = runState
+      // Post metrics to MLflow ootb — fire-and-forget.
+      logMlflowMetricsForResult(result, runState).catch(() => {})
+    }
+
+    return result
+  }
+}
+
+export function shouldIncludeChatDetails(opts = {}) {
+  return cds.env.profiles?.includes("test") || opts._details === true
+}

package/srv/handlers/graph-executor/hitl.js

@@ -0,0 +1,263 @@
+import cds from "@sap/cds"
+import { agentMessage, firstDataPart, partsToText } from "../../../lib/utils/message-handling.js"
+import { audit, short } from "../../../lib/utils/utils.js"
+
+const LOG = cds.log("agents")
+
+export const HITL_METADATA_KEY = "sap.cds.agents.hitl"
+export const requiresHitl = (result) =>
+  result?.__interrupt__?.length > 0 || result?.interrupts?.length > 0
+
+export function parseResumeDecision(userText) {
+  const t = userText.trim()
+  if (/^(approve|yes|confirm|ok)$/i.test(t)) return { decisions: [{ type: "approve" }] }
+  if (/^edit$/i.test(t)) return { decisions: [{ type: "edit" }] }
+  return {
+    decisions: [
+      {
+        type: "reject",
+        message: `The user rejected this particular tool invocation with the reason: ${userText}`,
+      },
+    ],
+  }
+}
+
+function decisionsForAudit(resume, actionRequests = []) {
+  if (!Array.isArray(resume?.decisions)) return [{ action: null, decision: resume }]
+  return resume.decisions.map((decision, index) => ({
+    action: actionRequests[index] ?? { index: index + 1 },
+    decision,
+  }))
+}
+
+function extractInterruptDescription(resultOrErr) {
+  const interrupt = resultOrErr.__interrupt__?.[0] || resultOrErr.interrupts?.[0]
+  const payload = interrupt?.value
+  if (!payload) return "This action requires your approval. Reply 'approve' or 'reject'."
+  if (payload.actionRequests?.length > 0) {
+    return (
+      payload.actionRequests[0].description || `Approve action: ${payload.actionRequests[0].name}?`
+    )
+  }
+  return typeof payload === "string" ? payload : JSON.stringify(payload)
+}
+
+export function extractInterruptData(resultOrErr) {
+  const interrupt = resultOrErr.__interrupt__?.[0] || resultOrErr.interrupts?.[0]
+  const payload = interrupt?.value
+  if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined
+  const actionRequests = mergeReviewConfigs(payload.actionRequests, payload.reviewConfigs)
+  return actionRequests === payload.actionRequests ? payload : { ...payload, actionRequests }
+}
+
+function mergeReviewConfigs(actionRequests, reviewConfigs) {
+  if (!Array.isArray(actionRequests) || !Array.isArray(reviewConfigs)) return actionRequests
+  const configsByAction = new Map(reviewConfigs.map((config) => [config.actionName, config]))
+  let changed = false
+  const requests = actionRequests.map((request) => {
+    const config = configsByAction.get(request.name)
+    if (!config) return request
+    changed = true
+    const { allowedDecisions, argsSchema } = config
+    return {
+      ...request,
+      ...(allowedDecisions === undefined ? {} : { allowedDecisions }),
+      ...(argsSchema === undefined ? {} : { argsSchema }),
+    }
+  })
+  return changed ? requests : actionRequests
+}
+
+function interruptActionCount(resultOrErr) {
+  const interrupts = resultOrErr.__interrupt__ || resultOrErr.interrupts || []
+  return interrupts.reduce(
+    (count, interrupt) => count + (interrupt?.value?.actionRequests?.length || 0),
+    0,
+  )
+}
+
+function interruptActionRequests(resultOrErr) {
+  const interrupts = resultOrErr.__interrupt__ || resultOrErr.interrupts || []
+  return interrupts.flatMap((interrupt) => interrupt?.value?.actionRequests || [])
+}
+
+export function composeHitlDecisionNote(actionRequests, resume) {
+  const decisions = resume?.decisions
+  if (!Array.isArray(decisions) || decisions.length === 0) return undefined
+  if (decisions.every((decision) => decision?.type === "approve")) return undefined
+  const consumed = new Set()
+  const takeByName = (name) => {
+    for (let index = 0; index < actionRequests.length; index++) {
+      if (!consumed.has(index) && actionRequests[index]?.name === name) {
+        consumed.add(index)
+        return actionRequests[index]
+      }
+    }
+    return undefined
+  }
+  const action = (request) =>
+    "`" + (request?.name ?? "unknown action") + "(" + JSON.stringify(request?.args ?? {}) + ")`"
+  const lines = []
+  for (const [index, decision] of decisions.entries()) {
+    const original = actionRequests[index]
+    if (decision?.type === "edit") {
+      const matched = takeByName(decision.editedAction?.name) ?? original
+      lines.push("- User edited " + action(matched) + " to " + action(decision.editedAction) + ".")
+      continue
+    }
+  }
+  if (!lines.length) {
+    return undefined
+  }
+  return ["User HITL decisions (not tool failures):", ...lines].join("\n")
+}
+
+async function getPreInterruptToolCalls(graph, config) {
+  try {
+    if (typeof graph.getState !== "function") return []
+    const state = await graph.getState(config)
+    const messages = state?.values?.messages ?? []
+    for (let i = messages.length - 1; i >= 0; i--) {
+      const message = messages[i]
+      if (message?.tool_calls?.length) {
+        return message.tool_calls.map((call) => ({ id: call.id, name: call.name, args: call.args }))
+      }
+    }
+    return []
+  } catch {
+    return []
+  }
+}
+
+async function getPendingHitlActionCount(graph, config) {
+  if (typeof graph.getState !== "function") {
+    throw new Error("Cannot resume HITL: graph state is unavailable.")
+  }
+  const state = await graph.getState(config)
+  const interrupts = state?.tasks?.flatMap((task) => task.interrupts || []) || []
+  const count = interrupts.reduce(
+    (total, interrupt) => total + (interrupt?.value?.actionRequests?.length || 1),
+    0,
+  )
+  if (count < 1) throw new Error("Cannot resume HITL: no pending actions found.")
+  return count
+}
+
+function pendingHitlFromTask(task) {
+  const pending = task?.status?.message?.metadata?.[HITL_METADATA_KEY]
+  if (!Number.isInteger(pending?.actionCount) || pending.actionCount < 1) return undefined
+  if (!Array.isArray(pending.decisions)) return undefined
+  return pending
+}
+
+function pendingActionRequests(task, pending) {
+  return (
+    pending?.actionRequests || firstDataPart(task?.status?.message?.parts)?.actionRequests || []
+  )
+}
+
+function interruptDescriptionFromTask(task, pending) {
+  const action = pendingActionRequests(task, pending)[pending?.decisions?.length || 0]
+  if (action) return action.description || `Approve action: ${action.name}?`
+  return (
+    task?.status?.message?.parts?.find((part) => part.kind === "text")?.text ||
+    "This action requires your approval. Reply 'approve' or 'reject'."
+  )
+}
+
+function publishInputRequired({ requestContext, eventBus, description, interruptData, pending }) {
+  const { taskId, contextId } = requestContext
+  eventBus.publish({
+    kind: "status-update",
+    taskId,
+    contextId,
+    status: {
+      state: "input-required",
+      message: agentMessage(description, interruptData, { [HITL_METADATA_KEY]: pending }),
+      timestamp: new Date().toISOString(),
+    },
+    final: true,
+  })
+  eventBus.finished()
+}
+
+export async function resumeHitl({ requestContext, graph, config, eventBus, stream, signal }) {
+  const { taskId, contextId } = requestContext
+  const dataPart = firstDataPart(requestContext.userMessage?.parts)
+  const userText = partsToText(requestContext.userMessage?.parts)
+  if (dataPart === undefined && !userText.trim()) {
+    throw new Error(cds.i18n.messages.at("RESUME_REQUIRES_TEXT"))
+  }
+  const { Command } = await import("@langchain/langgraph")
+  let resume = dataPart !== undefined ? dataPart : parseResumeDecision(userText)
+  let actionRequests = []
+
+  if (Array.isArray(resume?.decisions)) {
+    const pending = pendingHitlFromTask(requestContext.task)
+    const actionCount = pending?.actionCount ?? (await getPendingHitlActionCount(graph, config))
+    actionRequests = pendingActionRequests(requestContext.task, pending)
+    const decisions = [...(pending?.decisions || []), ...resume.decisions]
+    if (decisions.length < actionCount) {
+      const interruptData = firstDataPart(requestContext.task?.status?.message?.parts)
+      const nextPending = { ...pending, actionCount, decisions }
+      publishInputRequired({
+        requestContext,
+        eventBus,
+        description: interruptDescriptionFromTask(requestContext.task, nextPending),
+        interruptData,
+        pending: {
+          actionCount,
+          decisions,
+          actionRequests: pendingActionRequests(requestContext.task, nextPending),
+        },
+      })
+      return undefined
+    }
+    resume = { ...resume, decisions }
+  }
+
+  const decisions = decisionsForAudit(resume, actionRequests)
+  LOG.debug("resuming", { conversation: short(contextId), decisions })
+  audit("AgentTaskResumed", {
+    data: { taskId, contextId, service: cds.context?.["agent.service"], decisions },
+  })
+
+  const originalActions = actionRequests.length
+    ? actionRequests
+    : await getPreInterruptToolCalls(graph, config)
+  const decisionNote = composeHitlDecisionNote(originalActions, resume)
+  const commandArgs = { resume }
+  if (decisionNote) commandArgs.update = { _hitlDecisionNote: decisionNote }
+  const resumed = await stream(new Command(commandArgs), signal)
+  return resumed.state
+}
+
+export function handleHitlInterrupt({
+  result,
+  requestContext,
+  eventBus,
+  serviceName,
+  duration,
+  onInputRequired,
+}) {
+  const { taskId, contextId } = requestContext
+  const description = extractInterruptDescription(result)
+  const interruptData = extractInterruptData(result)
+  LOG.info("input-required", { conversation: short(contextId), service: serviceName, duration })
+  onInputRequired?.(description)
+  audit("AgentInputRequired", {
+    data: { taskId, contextId, service: serviceName, description, interruptData },
+  })
+  publishInputRequired({
+    requestContext,
+    eventBus,
+    description,
+    interruptData,
+    pending: {
+      actionCount: interruptActionCount(result),
+      decisions: [],
+      actionRequests: interruptData?.actionRequests || interruptActionRequests(result),
+    },
+  })
+  return true
+}