@cap-js/agents 0.9.1 → 0.9.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js/agents",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "description": "CDS plugin for building agents",
   "author": "SAP SE (https://www.sap.com)",
   "license": "Apache-2.0",
@@ -11,16 +11,14 @@
   },
   "type": "module",
   "exports": {
-    ".": "./index.js",
     "./package.json": "./package.json",
     "./cds-plugin": "./cds-plugin.js",
-    "./index.cds": "./index.cds",
+    "./eval": "./lib/eval/index.js",
+    "./lib/index.cjs": "./lib/index.cjs",
     "./lib/models/*": "./lib/models/*.js",
-    "./srv/langgraph-executor-srv": "./srv/langgraph-executor-srv.js",
-    "./srv/*": "./srv/*",
-    "./lib/*": "./lib/*"
+    "./srv/entities.cds": "./srv/entities.cds",
+    "./srv/push-notification-srv.js": "./srv/push-notification-srv.js"
   },
-  "main": "index.js",
   "scripts": {
     "lint": "npx eslint .",
     "test": "CDS_ENV=test npx vitest run --config vitest.config.js",
@@ -30,15 +28,15 @@
     "watch:hybrid": "DEBUG=agents cds bind --exec -- cds w tests/projects/bookshop",
     "watch:with-claude": "cds w tests/projects/bookshop --profile with-claude",
     "watch:deep-agent": "cds w tests/projects/deep-agent --profile hybrid",
+    "docs:audit": "node .scripts/generate-audit-docs.js",
+    "docs:audit:check": "node .scripts/generate-audit-docs.js --check",
     "prettier": "npx -y prettier@3 --write .",
     "prettier:check": "npx -y prettier@3 --check ."
   },
   "files": [
-    "index.js",
-    "index.cds",
+    "cds-plugin.js",
     "lib",
     "srv",
-    "cds-plugin.js",
     "_i18n"
   ],
   "dependencies": {
@@ -52,7 +50,6 @@
     "@sap-ai-sdk/langchain": "^2.13.0",
     "@sap-ai-sdk/orchestration": "^2.11.0",
     "@sap-cloud-sdk/connectivity": "^4.7.0",
-    "@sap-cloud-sdk/resilience": "^4.7.0",
     "deepagents": "^1.10.0",
     "langchain": "^1.5.0",
     "marked": "^18",
@@ -63,12 +60,15 @@
     "@cap-js/sqlite": "^3",
     "@sap/cds-mtxs": "^4",
     "@toon-format/toon": ">=2.3",
-    "deepagents": "^1.10.2"
+    "acorn": "^8.18.0",
+    "deepagents": "^1.10.2",
+    "openevals": "^0.2.0"
   },
   "peerDependencies": {
     "@cap-js/audit-logging": ">=1",
     "@cap-js/telemetry": ">=1",
-    "@sap/cds": ">=9"
+    "@sap/cds": ">=9",
+    "openevals": ">=0.2"
   },
   "peerDependenciesMeta": {
     "@cap-js/telemetry": {
@@ -76,6 +76,9 @@
     },
     "@cap-js/audit-logging": {
       "optional": true
+    },
+    "openevals": {
+      "optional": true
     }
   },
   "engines": {
@@ -87,6 +90,7 @@
       "trace_langchain": true,
       "mlflow": false,
       "connect": "auto",
+      "retention": "30d",
       "[development]": {
         "pushNotifications": {
           "allowedDomains": [
@@ -112,11 +116,16 @@
       },
       "persistAllCheckpointWrites": false,
       "activeUsersInterval": "24h",
+      "circuitBreaker": {
+        "errorThresholdPercentage": 50,
+        "volumeThreshold": 10,
+        "resetTimeout": 30000,
+        "rollingCountTimeout": 10000
+      },
       "params": {
         "max_tokens": 4096,
         "temperature": 0
       },
-      "contentFilter": true,
       "fileIO": {
         "enabled": false,
         "maxInputFileSizeBytes": 2097152,
@@ -156,7 +165,7 @@
     },
     "requires": {
       "agents": {
-        "model": "@cap-js/agents"
+        "model": "@cap-js/agents/srv/entities.cds"
       },
       "agent-push-notifications": {
         "impl": "@cap-js/agents/srv/push-notification-srv.js"

package/{index.cds → srv/entities.cds}

@@ -50,12 +50,23 @@ entity Tasks : managed {
        * Files written by agent via /outputs/ path for this task.
        */
       outputFiles    : Composition of many Attachments;
+
+      /** LangGraph checkpoints created by this task. Cascade-deleted. */
+      checkpoints    : Composition of many Checkpoints
+                         on checkpoints.task_id = taskId;
+
+      /** LangGraph checkpoint writes tied to this task. Cascade-deleted. */
+      checkpointWrites : Composition of many CheckpointWrites
+                           on checkpointWrites.task_id = taskId;
 }
 
 entity Checkpoints : managed {
   key thread_id            : String;
   key checkpoint_ns        : String default '';
   key checkpoint_id        : String;
+      task_id              : String;
+      task                 : Association to one Tasks
+                               on task.taskId = task_id;
       parent_checkpoint_id : String;
       parent               : Association to one Checkpoints
                                on parent.checkpoint_id = parent_checkpoint_id;

package/srv/handlers/chat.js

@@ -0,0 +1,278 @@
+import cds from "@sap/cds"
+import { startCollection } from "../../lib/eval/span-collector.js"
+import { metricsFromSpans } from "../../lib/eval/metrics.js"
+import { getActiveRunState, logMlflowMetricsForResult } from "../../lib/eval/eval-run.js"
+
+export const COLLECT_RESULT = Symbol.for("@cap-js/agents:chat:collect-result")
+
+class NoopEventBus {
+  constructor() {
+    this.events = []
+    this[COLLECT_RESULT] = true
+    this._graphResult = null
+    this._done = false
+    this.finished$ = new Promise((resolve, reject) => {
+      this._resolve = resolve
+      this._reject = reject
+    })
+  }
+
+  publish(event) {
+    this.events.push(event)
+  }
+
+  finished() {
+    this._done = true
+    this._resolve?.()
+  }
+
+  error(err) {
+    this._reject?.(err)
+  }
+
+  getFinalText() {
+    for (let i = this.events.length - 1; i >= 0; i--) {
+      const e = this.events[i]
+      if (e.kind === "artifact-update" && e.lastChunk === true) {
+        const text = e.artifact?.parts
+          ?.filter((p) => p.kind === "text")
+          .map((p) => p.text)
+          .join("")
+        if (text) return text
+      }
+    }
+    for (let i = this.events.length - 1; i >= 0; i--) {
+      const e = this.events[i]
+      if (e.kind === "status-update" && e.status?.message?.parts) {
+        const text = e.status.message.parts
+          .filter((p) => p.kind === "text")
+          .map((p) => p.text)
+          .join("")
+        if (text) return text
+      }
+    }
+    return ""
+  }
+
+  getStatus() {
+    const e = this.events.at(-1)
+    if (e && e.kind === "status-update" && e.status?.state) {
+      const state = e.status.state
+      const msg = e.status?.message?.parts?.find((p) => p.kind === "text")?.text ?? ""
+      return { status: state, description: msg }
+    }
+    return { status: "completed", description: "" }
+  }
+}
+
+// Derive toolCalls from graph messages by pairing AIMessage.tool_calls with ToolMessage results.
+function toolCallsFromMessages(messages) {
+  if (!Array.isArray(messages) || messages.length === 0) return []
+
+  const resultById = new Map()
+  for (const msg of messages) {
+    if (msg.tool_call_id && msg.type === "tool") {
+      const content = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
+      resultById.set(msg.tool_call_id, content)
+    }
+  }
+
+  const entries = []
+  for (const msg of messages) {
+    if (!Array.isArray(msg.tool_calls) || msg.tool_calls.length === 0) continue
+    for (const tc of msg.tool_calls) {
+      const raw = resultById.get(tc.id)
+      let toolResult
+      try {
+        toolResult = raw !== undefined ? JSON.parse(raw) : undefined
+      } catch {
+        toolResult = raw
+      }
+      const entry = {
+        tool: tc.name,
+        args: tc.args ?? {},
+        outcome: "success",
+        ...(toolResult !== undefined && { result: toolResult }),
+      }
+      attachCqn(entry)
+      entries.push(entry)
+    }
+  }
+  return entries
+}
+
+/**
+ * If entry.args.cql attaches cqn as hidden property.
+ * No-op when cql is absent or faulty.
+ */
+function attachCqn(entry) {
+  const cql = entry.args?.cql
+  if (typeof cql !== "string") return
+  try {
+    const cqn = cds.parse.cql(cql)
+    Object.defineProperty(entry, "cqn", {
+      value: cqn,
+      enumerable: false,
+      writable: false,
+      configurable: true,
+    })
+  } catch {
+    /* unparseable CQL — skip */
+  }
+}
+
+function buildRequestContext(query, opts = {}) {
+  const taskId = opts.taskId || cds.utils.uuid()
+  const contextId = opts.contextId || cds.utils.uuid()
+  const parts =
+    opts.parts ??
+    (typeof query === "string"
+      ? [{ kind: "text", text: query }]
+      : (query?.parts ?? [{ kind: "text", text: String(query) }]))
+  return {
+    taskId,
+    contextId,
+    task: opts.task ?? null,
+    userMessage: {
+      kind: "message",
+      messageId: cds.utils.uuid(),
+      role: "user",
+      taskId,
+      contextId,
+      parts,
+    },
+  }
+}
+
+/** Install srv.chat(query, previous?) on an @agent service for eval tests. */
+export function registerChat(srv) {
+  srv.chat = async function chat(query, previous) {
+    // Resolve the small eval helper API:
+    //   chat(query)
+    //   chat(query, prevResult) — extract contextId + taskId for HITL resume
+    //   chat(query, { _details: true }) — internal/test escape hatch outside test profile
+    let opts = {}
+    if (typeof previous === "string") {
+      throw new TypeError(
+        "agent.chat: second argument must be a previous chat result object or options object",
+      )
+    } else if (previous && typeof previous === "object") {
+      if ("text" in previous || "contextId" in previous) {
+        // prior chat() result — extract conversation ids and HITL state
+        opts = {
+          contextId: previous.contextId,
+          // mark as resume when prior result was input-required
+          ...(previous.status === "input-required" && {
+            taskId: previous.taskId,
+            task: { id: previous.taskId, status: { state: "input-required" } },
+          }),
+        }
+      } else {
+        opts = { _details: previous._details === true }
+      }
+    }
+
+    const includeDetails = shouldIncludeChatDetails(opts)
+    const runState = includeDetails ? getActiveRunState() : null
+
+    // Open span collection session before execution so the processor
+    // captures all spans from this trace.
+    const collection = includeDetails ? await startCollection() : null
+
+    const { LangGraphExecutor } = await import("../langgraph-executor-srv.js")
+    const executor = LangGraphExecutor.for(srv)
+    const requestContext = buildRequestContext(query, opts)
+    const eventBus = new NoopEventBus()
+    const mlflowRunId = runState?.mlflowRunId
+    let traceId
+
+    const runInContext = async () => {
+      // Link the trace to the MLflow eval run — graph-executor reads this to set mlflow.sourceRun.
+      if (mlflowRunId && cds.context) cds.context["_mlflow.evalRunId"] = mlflowRunId
+
+      let execError = null
+      const execPromise = executor.execute(requestContext, eventBus).catch((err) => {
+        execError = err
+        if (!eventBus._done) eventBus.finished()
+      })
+      await Promise.race([eventBus.finished$, execPromise])
+      if (!eventBus._done) eventBus.finished()
+      if (execError) throw execError
+
+      if (includeDetails) {
+        const rootSpan = cds.context?.["_mlflow.rootSpan"]
+        if (rootSpan) traceId = rootSpan.spanContext?.()?.traceId
+      }
+    }
+
+    if (cds.context) {
+      await runInContext()
+    } else {
+      await cds._with(
+        new cds.EventContext({ tenant: "t0", user: new cds.User.Privileged() }),
+        runInContext,
+      )
+    }
+
+    const { status, description } = eventBus.getStatus()
+    if (status === "failed")
+      throw new Error(`agent.chat: task failed — ${description || "no message"}`)
+
+    const text = eventBus.getFinalText()
+    const result = {
+      text,
+      contextId: requestContext.contextId,
+      taskId: requestContext.taskId,
+      status,
+    }
+
+    if (includeDetails) {
+      const allMessages = eventBus._graphResult?.messages ?? []
+
+      // Find last HumanMessage whose text matches current query; return from there.
+      const queryText = String(query)
+      let turnStart = 0
+      for (let i = allMessages.length - 1; i >= 0; i--) {
+        const m = allMessages[i]
+        if (m.getType?.() === "human" || m._getType?.() === "human" || m.type === "human") {
+          const content = typeof m.content === "string" ? m.content : (m.content?.[0]?.text ?? "")
+          if (content.startsWith(queryText) || queryText.startsWith(content.trim())) {
+            turnStart = i
+            break
+          }
+        }
+      }
+      const messages = allMessages.slice(turnStart)
+      const toolCalls = toolCallsFromMessages(messages)
+
+      const allSpans = collection.collect()
+      const spans = traceId
+        ? allSpans.filter((s) => s.spanContext?.()?.traceId === traceId)
+        : allSpans
+
+      result.query = String(query)
+      result.traceId = traceId
+      result.toolCalls = toolCalls
+      result.messages = messages.map((m) => {
+        // Eval prompts expect text content, not content-block objects.
+        if (m.type === "ai") {
+          if (Array.isArray(m.content) && m.content[0]?.text) {
+            m.content = m.content[0].text
+          }
+        }
+        return m
+      })
+      result.spans = spans
+      result.metrics = metricsFromSpans(spans)
+      if (runState) result._evalState = runState
+      // Post metrics to MLflow ootb — fire-and-forget.
+      logMlflowMetricsForResult(result, runState).catch(() => {})
+    }
+
+    return result
+  }
+}
+
+export function shouldIncludeChatDetails(opts = {}) {
+  return cds.env.profiles?.includes("test") || opts._details === true
+}

package/srv/handlers/graph-executor.js

@@ -2,10 +2,13 @@ import cds from "@sap/cds"
 import { short, audit, ms4 } from "../../lib/utils/utils.js"
 import { partsToText, buildChatMessages, firstDataPart } from "../../lib/utils/message-handling.js"
 import * as metrics from "../../lib/telemetry/metrics.js"
-import { mlflowAttrs, mlflowTraceAttrs, setSpanAttrs } from "../../lib/telemetry/mlflow.js"
+import { mlflowAttrs, mlflowTraceAttrs, setSpanAttrs } from "../../lib/telemetry/mlflow/index.js"
 import { CdsFileStore } from "../../lib/protocol/persistence/file-store.js"
 import { formatFileSize, sanitizeFilename } from "./tools.js"
 import { convertUsageData } from "../../lib/telemetry/chat-tracing.js"
+import { triggerCleanup } from "../../lib/protocol/persistence/cleanup.js"
+import { COLLECT_RESULT } from "./chat.js"
+import { linkTraceToPrompt } from "../../lib/telemetry/mlflow/tracing.js"
 
 const LOG = cds.log("agents")
 
@@ -333,16 +336,14 @@ class GraphExecutor {
 
     let tokenCount = 0
     let finalState = null
-    // Track the current turn (langchain message id) and whether it has emitted a
-    // tool call. Anthropic-style turns can stream a text preamble BEFORE their
-    // tool_use block ("Let me first look up …"); we can't tell in advance that
-    // such a turn is planning rather than the final answer, so we stream those
-    // tokens optimistically. Once we see tool_call_chunks for the same turn, we
-    // know retrospectively that the preamble was planning — emit an authoritative
-    // event-level replace with empty text to wipe the leaked preamble, then skip
-    // all further text from this turn.
+    // Track the current turn (langchain message id). Each new turn opens a fresh
+    // bubble on the client via `append:false`; subsequent tokens of the same turn
+    // are `append:true` (accumulate). Anthropic-style turns can stream a text
+    // preamble BEFORE their tool_use block ("Let me first look up …") — we let
+    // that reasoning text stream normally; the client is responsible for the
+    // final visual (collapse to the last turn's bubble at task completion).
     let currentMsgId = null
-    let turnHasToolCall = false
+    let thinkingCount = 0
 
     try {
       if (typeof graph.stream !== "function" || cds.env.agents?.streaming === false) {
@@ -378,30 +379,11 @@ class GraphExecutor {
 
           if (msgChunk.id && msgChunk.id !== currentMsgId) {
             currentMsgId = msgChunk.id
-            turnHasToolCall = false
             tokenCount = 0
           }
 
-          // Retroactively invalidate a leaked planning preamble. In a ReAct loop
-          // the model can emit "Let me look this up …" before its tool_use block
-          if (msgChunk.tool_call_chunks?.length && !turnHasToolCall) {
-            turnHasToolCall = true
-            if (tokenCount > 0) {
-              eventBus.publish({
-                kind: "artifact-update",
-                taskId,
-                contextId,
-                append: false,
-                lastChunk: false,
-                artifact: {
-                  artifactId: "response",
-                  parts: [{ kind: "text", text: "" }],
-                },
-              })
-              tokenCount = 0
-            }
-          }
-          if (turnHasToolCall) continue
+          const lastChunk =
+            !!msgChunk.additional_kwargs?.intermediate_results?.llm?.choices[0].finish_reason
 
           const text = messageText(msgChunk?.content)
           if (!text) continue
@@ -414,12 +396,13 @@ class GraphExecutor {
             taskId,
             contextId,
             append: tokenCount > 0,
-            lastChunk: false,
+            lastChunk: lastChunk,
             artifact: {
-              artifactId: "response",
+              artifactId: `thinking-${thinkingCount}`,
               parts: [{ kind: "text", text }],
             },
           })
+          if (lastChunk) thinkingCount++
           tokenCount++
         } else if (mode === "updates") {
           // The updates stream yields per-node deltas — { <node>: { messages: [oneNewMessage] } },
@@ -589,6 +572,10 @@ class GraphExecutor {
       audit("AgentTaskStarted", {
         data: { taskId, contextId, service: serviceName, userMessage: requestContext.userMessage },
       })
+      // Lazy scheduling task deletion
+      cds.spawn({}, async () => {
+        await triggerCleanup(serviceName)
+      })
     }
 
     // ── File I/O: persist incoming FileParts to cap.agent.Tasks.inputFiles ──
@@ -694,6 +681,12 @@ class GraphExecutor {
           }),
         )
         setSpanAttrs(rootSpan, mlflowTraceAttrs())
+        // Required for MLFLow run linking
+        const evalRunId = cds.context?.["_mlflow.evalRunId"]
+        if (evalRunId) {
+          rootSpan.setAttribute("mlflow.sourceRun", evalRunId)
+          wfSpan.setAttribute("mlflow.sourceRun", evalRunId)
+        }
       }
 
       let usageData
@@ -748,7 +741,6 @@ class GraphExecutor {
               contextId,
               service: serviceName,
               decision,
-              userMessage: requestContext.userMessage,
             },
           })
           // On edit, stash a diff note in state; the injector middleware prepends it next turn.
@@ -818,7 +810,7 @@ class GraphExecutor {
               contextId,
               service: serviceName,
               description,
-              userMessage: requestContext.userMessage,
+              interruptData,
             },
           })
 
@@ -873,8 +865,7 @@ class GraphExecutor {
             service: serviceName,
             duration,
             tokenUsage: usageData,
-            toolCalls: totalToolCalls(result.messages),
-            output: output?.slice(0, 2000),
+            toolCalls: toolCallsShortened(result.messages),
             task: requestContext.task,
           },
         })
@@ -1073,6 +1064,12 @@ class GraphExecutor {
           })
         }
 
+        // Programmatic .chat() path: stash graph result on eventBus so chat.js
+        // can read messages without an extra checkpoint roundtrip.
+        if (eventBus[COLLECT_RESULT]) {
+          eventBus._graphResult = { messages: result.messages || [] }
+        }
+
         eventBus.publish({
           kind: "status-update",
           taskId,
@@ -1240,6 +1237,10 @@ class GraphExecutor {
           final: true,
         })
       } finally {
+        // setSpanAttrs must happen at the end for the linking as else prompt might not yet have been created
+        const rootSpan = cds.context["_mlflow.rootSpan"]
+        setSpanAttrs(rootSpan, linkTraceToPrompt())
+
         this._abortControllers.delete(taskId)
         metrics.concurrentExecutions.add(-1, mAttrs)
 
@@ -1367,3 +1368,13 @@ function totalToolCalls(messages) {
     return acc
   }, 0)
 }
+
+/**
+ * @param {[import('@langchain/core/messages').Message]} messages
+ */
+function toolCallsShortened(messages) {
+  return messages.reduce((acc, val) => {
+    if (val.type === "tool") acc.push(val.name)
+    return acc
+  }, [])
+}

package/srv/handlers/index.js

@@ -3,6 +3,8 @@ import { generateTools, createReadFileTool } from "./tools.js"
 import { buildSystemPrompt } from "./system-prompt.js"
 import buildMiddleware from "../../lib/agents/middleware/index.js"
 import { partsToText } from "../../lib/utils/message-handling.js"
+import { cleanupExpiredTasks } from "../../lib/protocol/persistence/cleanup.js"
+import { registerChat } from "./chat.js"
 
 const LOG = cds.log("agents")
 
@@ -170,4 +172,10 @@ export default function registerDefaultAgentHandlers(srv) {
       },
     })
   })
+
+  srv.on("cleanupTasks", async () => {
+    await cleanupExpiredTasks(srv.name)
+  })
+
+  registerChat(srv)
 }