@cap-js/agents 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/cds-plugin.js +2 -2
- package/lib/agents/markdown/backends/outputs-backend.js +16 -12
- package/lib/agents/markdown/backends/readonly-backend.js +48 -0
- package/lib/agents/markdown/backends/uploads-backend.js +30 -13
- package/lib/agents/markdown/deep-agent.js +15 -26
- package/lib/agents/middleware/content-filter.js +3 -2
- package/lib/agents/middleware/index.js +11 -0
- package/lib/agents/middleware/remote-mcp.js +62 -0
- package/lib/agents/middleware/tool-wrap.js +52 -0
- package/lib/compile.js +6 -2
- package/lib/eval/Judge.js +239 -0
- package/lib/eval/eval-describe.js +69 -0
- package/lib/eval/eval-run.js +147 -0
- package/lib/eval/index.js +6 -0
- package/lib/eval/metrics.js +53 -0
- package/lib/eval/span-collector.js +50 -0
- package/lib/index.js +4 -3
- package/lib/models/aicore.js +23 -6
- package/lib/preview/chat.html +394 -69
- package/lib/protocol/agent-card.js +6 -2
- package/lib/protocol/persistence/checkpoint-saver.js +4 -1
- package/lib/protocol/persistence/cleanup.js +84 -0
- package/lib/sidecar.js +1 -1
- package/lib/telemetry/chat-tracing.js +61 -8
- package/lib/telemetry/mlflow/credentials.js +79 -0
- package/lib/telemetry/mlflow/evaluation.js +38 -0
- package/lib/telemetry/mlflow/exporter/DatabricksExporter.js +75 -0
- package/lib/telemetry/mlflow/exporter/MlflowExporter.js +115 -0
- package/lib/telemetry/mlflow/exporter/index.js +23 -0
- package/lib/telemetry/mlflow/index.js +16 -0
- package/lib/telemetry/mlflow/prompts.js +123 -0
- package/lib/telemetry/mlflow/tracing.js +264 -0
- package/lib/telemetry/tool-tracing.js +27 -54
- package/lib/telemetry/tracing.js +3 -1
- package/lib/utils/markdown.js +1 -11
- package/lib/utils/resilience.js +133 -0
- package/lib/utils/utils.js +17 -0
- package/package.json +24 -15
- package/{index.cds → srv/entities.cds} +11 -0
- package/srv/handlers/chat.js +278 -0
- package/srv/handlers/graph-executor.js +48 -37
- package/srv/handlers/index.js +8 -0
- package/srv/handlers/mcp-tools.js +11 -47
- package/srv/handlers/sub-agent-tools.js +12 -3
- package/srv/handlers/tools.js +9 -5
- package/index.js +0 -0
- package/lib/telemetry/mlflow.js +0 -290
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import cds from "@sap/cds"
|
|
2
|
+
import { ms4 } from "../../utils/utils.js"
|
|
3
|
+
|
|
4
|
+
const LOG = cds.log("agent")
|
|
5
|
+
|
|
6
|
+
const TASKS = "cap.agent.Tasks"
|
|
7
|
+
|
|
8
|
+
// Throttle: last cleanup timestamp per tenant & service
|
|
9
|
+
const lastDeletionTriggerMap = new Map()
|
|
10
|
+
|
|
11
|
+
/** Test-only: reset throttle map. */
|
|
12
|
+
export function _resetCleanupThrottle() {
|
|
13
|
+
lastDeletionTriggerMap.clear()
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// TTL from cds.env.agents.retention., `false`/0 → disabled.
|
|
17
|
+
function resolveTtlMs() {
|
|
18
|
+
const cfg = cds.env.agents?.retention
|
|
19
|
+
if (cfg === false || cfg === 0) return 0
|
|
20
|
+
const value = cfg === true || cfg
|
|
21
|
+
if (typeof value === "number") return value
|
|
22
|
+
return ms4(String(value))
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const MS_OF_A_DAY = ms4("1d")
|
|
26
|
+
|
|
27
|
+
export async function triggerCleanup(serviceName) {
|
|
28
|
+
const ttlMs = resolveTtlMs()
|
|
29
|
+
if (!ttlMs) {
|
|
30
|
+
LOG.debug(`cds.agents.retention is not configured. Skipping cleanup of old Tasks.`)
|
|
31
|
+
return
|
|
32
|
+
}
|
|
33
|
+
const tenant = cds.context?.tenant
|
|
34
|
+
if (!lastDeletionTriggerMap.has(tenant)) lastDeletionTriggerMap.set(tenant, new Map())
|
|
35
|
+
const serviceMap = lastDeletionTriggerMap.get(tenant)
|
|
36
|
+
const lastDeletionTriggered = serviceMap.get(serviceName)
|
|
37
|
+
if (lastDeletionTriggered > Date.now() - MS_OF_A_DAY) {
|
|
38
|
+
LOG.debug(
|
|
39
|
+
`Skip scheduling deletion of tasks for ${serviceName} because the last scheduled deletion was triggered within the last 24h.`,
|
|
40
|
+
)
|
|
41
|
+
return
|
|
42
|
+
}
|
|
43
|
+
const srv = cds.services[serviceName]
|
|
44
|
+
if (!srv) {
|
|
45
|
+
LOG.warn(`triggerCleanup: service "${serviceName}" not found in cds.services, skipping.`)
|
|
46
|
+
return
|
|
47
|
+
}
|
|
48
|
+
if (typeof srv.schedule !== "function") {
|
|
49
|
+
LOG.debug(`triggerCleanup: srv.schedule not available (CDS < 9). Skipping cleanup scheduling.`)
|
|
50
|
+
return
|
|
51
|
+
}
|
|
52
|
+
serviceMap.set(serviceName, Date.now())
|
|
53
|
+
const MAX_TIMEOUT = 2_147_483_647
|
|
54
|
+
const delay = Math.min(ttlMs + MS_OF_A_DAY, MAX_TIMEOUT)
|
|
55
|
+
await srv.schedule("cleanupTasks", {}).after(delay)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* GC of expired tasks per service.
|
|
60
|
+
* Compositions cascade automatically: inputFiles, outputFiles, pushConfigs,
|
|
61
|
+
* checkpoints, checkpointWrites.
|
|
62
|
+
*/
|
|
63
|
+
export async function cleanupExpiredTasks(serviceName) {
|
|
64
|
+
const ttlMs = resolveTtlMs()
|
|
65
|
+
if (!ttlMs) {
|
|
66
|
+
LOG.debug(`cds.agents.retention is not configured. Skipping cleanup of old Tasks.`)
|
|
67
|
+
return
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const tenant = cds.context?.tenant
|
|
71
|
+
const now = Date.now()
|
|
72
|
+
const cutoff = new Date(now - ttlMs).toISOString()
|
|
73
|
+
|
|
74
|
+
const taskResult = await DELETE.from(TASKS).where({
|
|
75
|
+
modifiedAt: { "<": cutoff },
|
|
76
|
+
agentService: serviceName,
|
|
77
|
+
})
|
|
78
|
+
// affectedRows since cds10
|
|
79
|
+
const deletedTasks = typeof taskResult === "number" ? taskResult : taskResult?.affectedRows || 0
|
|
80
|
+
|
|
81
|
+
if (deletedTasks > 0) {
|
|
82
|
+
LOG.debug("Cleanup", { tenant, deletedTasks, cutoff })
|
|
83
|
+
}
|
|
84
|
+
}
|
package/lib/sidecar.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import cds from "@sap/cds"
|
|
2
2
|
import A2AProtocolAdapter, { authContext } from "./index.js"
|
|
3
|
-
import { slugified } from "./utils/
|
|
3
|
+
import { slugified } from "./utils/utils.js"
|
|
4
4
|
import registerDefaultAgentHandlers from "../srv/handlers/index.js"
|
|
5
5
|
|
|
6
6
|
const LOG = cds.log("a2a")
|
|
@@ -1,12 +1,23 @@
|
|
|
1
1
|
import cds from "@sap/cds"
|
|
2
2
|
import * as metrics from "./metrics.js"
|
|
3
|
-
import { mlflowAttrs, setSpanAttrs } from "./mlflow.js"
|
|
3
|
+
import { mlflowAttrs, setSpanAttrs } from "./mlflow/index.js"
|
|
4
4
|
import { audit } from "../utils/utils.js"
|
|
5
5
|
|
|
6
6
|
const LOG = cds.log("agents")
|
|
7
7
|
|
|
8
8
|
const PATCHED = Symbol.for("@cap-js/agents:patched")
|
|
9
9
|
const SPAN_KIND_CLIENT = 3
|
|
10
|
+
const MODEL_PARAM_KEYS = [
|
|
11
|
+
"max_tokens",
|
|
12
|
+
"temperature",
|
|
13
|
+
"frequency_penalty",
|
|
14
|
+
"presence_penalty",
|
|
15
|
+
"top_p",
|
|
16
|
+
"n",
|
|
17
|
+
"stop",
|
|
18
|
+
"reasoning_effort",
|
|
19
|
+
"tool_choice",
|
|
20
|
+
]
|
|
10
21
|
|
|
11
22
|
// ─── Public API ──────────────────────────────────────────────────────────────
|
|
12
23
|
|
|
@@ -48,7 +59,7 @@ export function _patchChatModelProto(proto) {
|
|
|
48
59
|
if (!tracer) return original.call(this, input, opts)
|
|
49
60
|
|
|
50
61
|
const model = this.options?.model || this.model || this.constructor.name
|
|
51
|
-
const provider =
|
|
62
|
+
const provider = _resolveProvider()
|
|
52
63
|
const node = opts?.runName || "agent"
|
|
53
64
|
const messages = Array.isArray(input) ? input : undefined
|
|
54
65
|
const cacheControl = _detectCacheControl(messages)
|
|
@@ -71,6 +82,28 @@ export function _patchChatModelProto(proto) {
|
|
|
71
82
|
try {
|
|
72
83
|
const result = await original.call(this, input, opts)
|
|
73
84
|
const duration = Date.now() - t0
|
|
85
|
+
const ir = result.additional_kwargs?.intermediate_results
|
|
86
|
+
if (span && ir) {
|
|
87
|
+
if (ir.input_filtering) span.setAttribute("gen_ai.orchestration.input_filtering", true)
|
|
88
|
+
if (ir.output_filtering) span.setAttribute("gen_ai.orchestration.output_filtering", true)
|
|
89
|
+
if (ir.input_masking) span.setAttribute("gen_ai.orchestration.input_masking", true)
|
|
90
|
+
const appliedFilterAmount = (filtering) => {
|
|
91
|
+
let res = []
|
|
92
|
+
for (const entry of filtering?.data?.choices ?? []) {
|
|
93
|
+
Object.keys(entry).forEach((e) => {
|
|
94
|
+
if (e !== "index") {
|
|
95
|
+
res = res.concat(Object.keys(entry[e]).map((filter) => `${e}_${filter}`))
|
|
96
|
+
}
|
|
97
|
+
})
|
|
98
|
+
}
|
|
99
|
+
return res
|
|
100
|
+
}
|
|
101
|
+
let ic = appliedFilterAmount(ir.input_filtering)
|
|
102
|
+
let oc = appliedFilterAmount(ir.output_filtering)
|
|
103
|
+
if (ic.length) span.setAttribute("gen_ai.orchestration.input_filter_services", ic)
|
|
104
|
+
if (oc.length) span.setAttribute("gen_ai.orchestration.output_filter_services", oc)
|
|
105
|
+
}
|
|
106
|
+
|
|
74
107
|
/** @type {import('@langchain/core/messages').UsageMetadata} */
|
|
75
108
|
const usage = result.usage_metadata
|
|
76
109
|
const finishReason =
|
|
@@ -113,10 +146,11 @@ export function _patchChatModelProto(proto) {
|
|
|
113
146
|
|
|
114
147
|
// ─── Internal helpers ────────────────────────────────────────────────────────
|
|
115
148
|
|
|
116
|
-
/**
|
|
117
|
-
function
|
|
118
|
-
|
|
119
|
-
return "
|
|
149
|
+
/** Resolve LLM provider from cds.env.requires. */
|
|
150
|
+
function _resolveProvider() {
|
|
151
|
+
const kind = cds.env.requires?.llm?.kind || "langchain"
|
|
152
|
+
if (kind === "ai-core" || kind === "aicore") return "sap-ai-core"
|
|
153
|
+
return kind.replace(/^llm-/, "")
|
|
120
154
|
}
|
|
121
155
|
|
|
122
156
|
function _metricAttrs(model, node) {
|
|
@@ -162,15 +196,19 @@ function _handleSuccess(
|
|
|
162
196
|
|
|
163
197
|
const taskId = opts?.configurable?._taskId || cds.context?.["agent.task.id"]
|
|
164
198
|
if (taskId) {
|
|
199
|
+
const modelParams = pickModelParams(params)
|
|
165
200
|
audit("AgentDecision", {
|
|
166
201
|
data: {
|
|
167
202
|
taskId,
|
|
168
203
|
contextId:
|
|
169
204
|
opts?.configurable?.thread_id?.split(":")[1] || cds.context?.["agent.context.id"],
|
|
170
205
|
service: opts?.configurable?._service || cds.context?.["agent.service"],
|
|
171
|
-
|
|
206
|
+
provider,
|
|
207
|
+
model: response?.model,
|
|
208
|
+
...(modelParams ? { modelParams } : {}),
|
|
209
|
+
finishReason: response?.finishReason,
|
|
172
210
|
iteration: opts?.configurable?._iteration ?? cds.context?.["agent.iteration"],
|
|
173
|
-
toolCalls: response?.toolCalls,
|
|
211
|
+
toolCalls: response?.toolCalls.map((t) => t.name),
|
|
174
212
|
tokenUsage,
|
|
175
213
|
duration,
|
|
176
214
|
},
|
|
@@ -178,6 +216,21 @@ function _handleSuccess(
|
|
|
178
216
|
}
|
|
179
217
|
}
|
|
180
218
|
|
|
219
|
+
function pickModelParams(params) {
|
|
220
|
+
if (!params || typeof params !== "object" || Array.isArray(params)) return undefined
|
|
221
|
+
const result = {}
|
|
222
|
+
for (const key of MODEL_PARAM_KEYS) {
|
|
223
|
+
const value = params[key]
|
|
224
|
+
if (value === undefined || typeof value === "function" || typeof value === "symbol") continue
|
|
225
|
+
try {
|
|
226
|
+
result[key] = structuredClone(value)
|
|
227
|
+
} catch {
|
|
228
|
+
result[key] = String(value)
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return Object.keys(result).length ? result : undefined
|
|
232
|
+
}
|
|
233
|
+
|
|
181
234
|
/** Handle LLM error: content-filter warnings, metrics, span error attrs. */
|
|
182
235
|
function _handleError(span, { err, mAttrs, model, node, messages }) {
|
|
183
236
|
const status = err.rootCause?.status
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import cds from "@sap/cds"
|
|
2
|
+
|
|
3
|
+
/** @returns {{ host: string, uc: {catalog,schema,tablePrefix}|null, warehouseId: string|undefined, getAuthHeaders: () => Promise<Record<string,string>> } | null} */
|
|
4
|
+
export function resolveMlflowCredentials() {
|
|
5
|
+
const creds = cds.env.requires?.mlflow?.credentials || {}
|
|
6
|
+
const host = (creds.MLFLOW_HOST || "").replace(/\/$/, "")
|
|
7
|
+
if (!host) return null
|
|
8
|
+
|
|
9
|
+
const ucCatalog = creds.UC_CATALOG
|
|
10
|
+
const ucSchema = creds.UC_SCHEMA
|
|
11
|
+
const ucTablePrefix = creds.UC_TABLE_PREFIX
|
|
12
|
+
const uc =
|
|
13
|
+
ucCatalog && ucSchema && ucTablePrefix
|
|
14
|
+
? { catalog: ucCatalog, schema: ucSchema, tablePrefix: ucTablePrefix }
|
|
15
|
+
: null
|
|
16
|
+
|
|
17
|
+
const warehouseId = creds.DATABRICKS_SQL_WAREHOUSE_ID || undefined
|
|
18
|
+
|
|
19
|
+
const getAuthHeaders = _buildAuthHeaderFactory(creds)
|
|
20
|
+
|
|
21
|
+
return { host, uc, warehouseId, getAuthHeaders }
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Resolution order: @Core.SchemaVersion annotation → credentials → env.
|
|
25
|
+
// MLflow experiment IDs must be numeric strings (int64).
|
|
26
|
+
export function resolveExperimentId() {
|
|
27
|
+
const srvName = cds.context?.["agent.service"]
|
|
28
|
+
if (srvName) {
|
|
29
|
+
const def = cds.context?.model?.definitions?.[srvName] || cds.services?.[srvName]?.definition
|
|
30
|
+
const annotated = def?.["@Core.SchemaVersion"]
|
|
31
|
+
if (annotated) return String(annotated)
|
|
32
|
+
}
|
|
33
|
+
const creds = cds.env.requires?.mlflow?.credentials || {}
|
|
34
|
+
return creds.MLFLOW_EXPERIMENT_ID || process.env.MLFLOW_EXPERIMENT_ID || null
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Build an async factory that returns { Authorization: "Bearer ..." } headers.
|
|
39
|
+
* OAuth tokens are cached and refreshed 60 s before expiry.
|
|
40
|
+
* Returns an empty object factory when no auth is configured (local MLflow).
|
|
41
|
+
*/
|
|
42
|
+
function _buildAuthHeaderFactory(creds) {
|
|
43
|
+
if (creds.clientid && creds.clientsecret) {
|
|
44
|
+
const tokenUrl = `${(creds.url || creds.MLFLOW_HOST || "").replace(/\/$/, "")}/oidc/v1/token`
|
|
45
|
+
let cached = null
|
|
46
|
+
let expiresAt = 0
|
|
47
|
+
return async function fetchOAuthHeaders() {
|
|
48
|
+
if (cached && Date.now() < expiresAt) return cached
|
|
49
|
+
const res = await fetch(tokenUrl, {
|
|
50
|
+
method: "POST",
|
|
51
|
+
headers: {
|
|
52
|
+
Authorization:
|
|
53
|
+
"Basic " + Buffer.from(`${creds.clientid}:${creds.clientsecret}`).toString("base64"),
|
|
54
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
55
|
+
},
|
|
56
|
+
body: "grant_type=client_credentials&scope=all-apis",
|
|
57
|
+
signal: AbortSignal.timeout(10_000),
|
|
58
|
+
})
|
|
59
|
+
if (!res.ok) {
|
|
60
|
+
const body = await res.text().catch(() => "")
|
|
61
|
+
throw new Error(
|
|
62
|
+
`MLflow OAuth token fetch failed: HTTP ${res.status} — ${body.slice(0, 200)}`,
|
|
63
|
+
)
|
|
64
|
+
}
|
|
65
|
+
const { access_token, expires_in } = await res.json()
|
|
66
|
+
cached = { Authorization: `Bearer ${access_token}` }
|
|
67
|
+
expiresAt = Date.now() + (expires_in - 60) * 1000
|
|
68
|
+
return cached
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
if (creds.MLFLOW_TOKEN) {
|
|
73
|
+
const headers = { Authorization: `Bearer ${creds.MLFLOW_TOKEN}` }
|
|
74
|
+
return async () => headers
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Local / unauthenticated MLflow
|
|
78
|
+
return async () => ({})
|
|
79
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import cds from "@sap/cds"
|
|
2
|
+
import { getMlflowExporter } from "./exporter/index.js"
|
|
3
|
+
|
|
4
|
+
export async function createEvalRun({ name } = {}) {
|
|
5
|
+
const exporter = getMlflowExporter()
|
|
6
|
+
if (!exporter) return null
|
|
7
|
+
const creds = cds.env.requires?.mlflow?.credentials || {}
|
|
8
|
+
const experimentId = creds.MLFLOW_EXPERIMENT_ID || process.env.MLFLOW_EXPERIMENT_ID || "0"
|
|
9
|
+
return exporter.createRun(experimentId, name)
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export async function closeEvalRun(runId) {
|
|
13
|
+
if (!runId) return
|
|
14
|
+
getMlflowExporter()?.closeRun(runId)
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// Log a flat metrics object; null/undefined values are skipped.
|
|
18
|
+
export async function logMlflowMetrics(runId, metrics) {
|
|
19
|
+
if (!runId) return
|
|
20
|
+
const exporter = getMlflowExporter()
|
|
21
|
+
if (!exporter) return
|
|
22
|
+
await Promise.allSettled(
|
|
23
|
+
Object.entries(metrics)
|
|
24
|
+
.filter(([, v]) => v != null)
|
|
25
|
+
.map(([key, value]) => exporter.logMetric(runId, key, value)),
|
|
26
|
+
)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export async function postMlflowAssessment(
|
|
30
|
+
traceId,
|
|
31
|
+
score,
|
|
32
|
+
rationale,
|
|
33
|
+
assessmentName,
|
|
34
|
+
sourceId,
|
|
35
|
+
opts,
|
|
36
|
+
) {
|
|
37
|
+
getMlflowExporter()?.postAssessment(traceId, score, rationale, assessmentName, sourceId, opts)
|
|
38
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { MlflowExporter } from "./MlflowExporter.js"
|
|
2
|
+
|
|
3
|
+
export class DatabricksExporter extends MlflowExporter {
|
|
4
|
+
async postAssessment(
|
|
5
|
+
traceId,
|
|
6
|
+
score,
|
|
7
|
+
rationale,
|
|
8
|
+
assessmentName,
|
|
9
|
+
sourceId,
|
|
10
|
+
{ metadata, sourceType } = {},
|
|
11
|
+
) {
|
|
12
|
+
const { uc, warehouseId } = this._creds
|
|
13
|
+
const ucPrefix = `${uc.catalog}.${uc.schema}.${uc.tablePrefix}`
|
|
14
|
+
let url = `/api/4.0/mlflow/traces/${encodeURIComponent(ucPrefix)}/tr-${traceId}/assessments`
|
|
15
|
+
if (warehouseId) url += `?sql_warehouse_id=${encodeURIComponent(warehouseId)}`
|
|
16
|
+
await this._fetch(url, {
|
|
17
|
+
trace_id: traceId,
|
|
18
|
+
assessment_name: assessmentName,
|
|
19
|
+
source: { source_type: sourceType ?? "LLM_JUDGE", source_id: sourceId ?? null },
|
|
20
|
+
feedback: { value: score },
|
|
21
|
+
rationale: String(rationale),
|
|
22
|
+
...(metadata && { metadata }),
|
|
23
|
+
trace_location: {
|
|
24
|
+
type: "UC_TABLE_PREFIX",
|
|
25
|
+
uc_table_prefix: {
|
|
26
|
+
catalog_name: uc.catalog,
|
|
27
|
+
schema_name: uc.schema,
|
|
28
|
+
table_prefix: uc.tablePrefix,
|
|
29
|
+
},
|
|
30
|
+
},
|
|
31
|
+
})
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Returns { tags: [{key,value}], latestVersion: {version, tags} | null }.
|
|
35
|
+
async ensurePrompt(name, description) {
|
|
36
|
+
let res = await this._fetch(
|
|
37
|
+
`/mlflow/unity-catalog/prompts/${encodeURIComponent(name)}`,
|
|
38
|
+
undefined,
|
|
39
|
+
"GET",
|
|
40
|
+
)
|
|
41
|
+
if (!res) {
|
|
42
|
+
res = await this._fetch("/mlflow/unity-catalog/prompts", { name, prompt: { description } })
|
|
43
|
+
}
|
|
44
|
+
const latestVersion = await this._getLatestUcVersion(name)
|
|
45
|
+
return { tags: res?.tags ?? [], latestVersion }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async createPromptVersion(name, description, tags = [], template = "") {
|
|
49
|
+
const res = await this._fetch(
|
|
50
|
+
`/mlflow/unity-catalog/prompts/${encodeURIComponent(name)}/versions`,
|
|
51
|
+
{
|
|
52
|
+
prompt_version: { template, description, tags },
|
|
53
|
+
},
|
|
54
|
+
)
|
|
55
|
+
return res?.version ? String(res.version) : null
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async setRegisteredModelTag(name, key, value) {
|
|
59
|
+
await this._fetch(`/mlflow/unity-catalog/prompts/${encodeURIComponent(name)}/tags`, {
|
|
60
|
+
key,
|
|
61
|
+
value,
|
|
62
|
+
})
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Returns { version, tags } of the latest UC prompt version, or null.
|
|
66
|
+
async _getLatestUcVersion(name) {
|
|
67
|
+
const res = await this._fetch(
|
|
68
|
+
`/mlflow/unity-catalog/prompts/${encodeURIComponent(name)}/versions/search`,
|
|
69
|
+
{ max_results: 1 },
|
|
70
|
+
)
|
|
71
|
+
const pv = res?.prompt_versions?.[0]
|
|
72
|
+
if (!pv) return null
|
|
73
|
+
return { version: String(pv.version), tags: pv.tags ?? [] }
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import cds from "@sap/cds"
|
|
2
|
+
|
|
3
|
+
const LOG = cds.log("agents")
|
|
4
|
+
|
|
5
|
+
export class MlflowExporter {
|
|
6
|
+
constructor(creds) {
|
|
7
|
+
this._creds = creds
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
async _fetch(path, body, method = "POST") {
|
|
11
|
+
const { host, getAuthHeaders } = this._creds
|
|
12
|
+
const headers = { ...(await getAuthHeaders()), "Content-Type": "application/json" }
|
|
13
|
+
try {
|
|
14
|
+
const res = await fetch(`${host}${path}`, {
|
|
15
|
+
method,
|
|
16
|
+
headers,
|
|
17
|
+
...(body !== undefined && { body: JSON.stringify(body) }),
|
|
18
|
+
})
|
|
19
|
+
if (!res.ok) {
|
|
20
|
+
LOG.error(
|
|
21
|
+
`[mlflow] ${method} ${path} → ${res.status}: ${(await res.text().catch(() => "")).slice(0, 200)}`,
|
|
22
|
+
)
|
|
23
|
+
return null
|
|
24
|
+
}
|
|
25
|
+
return res.json().catch(() => null)
|
|
26
|
+
} catch (err) {
|
|
27
|
+
LOG.debug(`[mlflow] ${path} error: ${err.message}`)
|
|
28
|
+
return null
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async createRun(experimentId, name) {
|
|
33
|
+
const data = await this._fetch("/api/2.0/mlflow/runs/create", {
|
|
34
|
+
experiment_id: experimentId,
|
|
35
|
+
run_name: name || `eval-${new Date().toISOString()}`,
|
|
36
|
+
start_time: Date.now(),
|
|
37
|
+
tags: [{ key: "mlflow.source.type", value: "LOCAL" }],
|
|
38
|
+
})
|
|
39
|
+
return data?.run?.info?.run_id ?? null
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async closeRun(runId) {
|
|
43
|
+
await this._fetch("/api/2.0/mlflow/runs/update", {
|
|
44
|
+
run_id: runId,
|
|
45
|
+
status: "FINISHED",
|
|
46
|
+
end_time: Date.now(),
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async logMetric(runId, key, value) {
|
|
51
|
+
await this._fetch("/api/2.0/mlflow/runs/log-metric", {
|
|
52
|
+
run_id: runId,
|
|
53
|
+
key,
|
|
54
|
+
value,
|
|
55
|
+
timestamp: Date.now(),
|
|
56
|
+
step: 0,
|
|
57
|
+
})
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async postAssessment(
|
|
61
|
+
traceId,
|
|
62
|
+
score,
|
|
63
|
+
rationale,
|
|
64
|
+
assessmentName,
|
|
65
|
+
sourceId,
|
|
66
|
+
{ metadata, sourceType } = {},
|
|
67
|
+
) {
|
|
68
|
+
await this._fetch(`/api/3.0/mlflow/traces/tr-${traceId}/assessments`, {
|
|
69
|
+
assessment: {
|
|
70
|
+
trace_id: traceId,
|
|
71
|
+
assessment_name: assessmentName,
|
|
72
|
+
source: { source_type: sourceType ?? "LLM_JUDGE", source_id: sourceId ?? null },
|
|
73
|
+
feedback: { value: score },
|
|
74
|
+
rationale: String(rationale),
|
|
75
|
+
...(metadata && { metadata }),
|
|
76
|
+
},
|
|
77
|
+
})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Returns { tags: [{key,value}], latestVersion: {version, tags} | null }.
|
|
81
|
+
async ensurePrompt(name, description, registrationTags = []) {
|
|
82
|
+
let res = await this._fetch(
|
|
83
|
+
`/api/2.0/mlflow/registered-models/get?name=${encodeURIComponent(name)}`,
|
|
84
|
+
undefined,
|
|
85
|
+
"GET",
|
|
86
|
+
)
|
|
87
|
+
if (!res) {
|
|
88
|
+
res = await this._fetch("/api/2.0/mlflow/registered-models/create", {
|
|
89
|
+
name,
|
|
90
|
+
description,
|
|
91
|
+
tags: registrationTags,
|
|
92
|
+
})
|
|
93
|
+
}
|
|
94
|
+
const rm = res?.registered_model
|
|
95
|
+
const lv = rm?.latest_versions?.[0]
|
|
96
|
+
return {
|
|
97
|
+
tags: rm?.tags ?? [],
|
|
98
|
+
latestVersion: lv ? { version: String(lv.version), tags: lv.tags ?? [] } : null,
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async createPromptVersion(name, description, tags = [], _template = "") {
|
|
103
|
+
const res = await this._fetch("/api/2.0/mlflow/model-versions/create", {
|
|
104
|
+
name,
|
|
105
|
+
source: "dummy-source", // required by API but unused for prompts
|
|
106
|
+
description,
|
|
107
|
+
tags,
|
|
108
|
+
})
|
|
109
|
+
return res?.model_version?.version ? String(res.model_version.version) : null
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async setRegisteredModelTag(name, key, value) {
|
|
113
|
+
await this._fetch("/api/2.0/mlflow/registered-models/set-tag", { name, key, value })
|
|
114
|
+
}
|
|
115
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import cds from "@sap/cds"
|
|
2
|
+
import { resolveMlflowCredentials } from "../credentials.js"
|
|
3
|
+
import { MlflowExporter } from "./MlflowExporter.js"
|
|
4
|
+
import { DatabricksExporter } from "./DatabricksExporter.js"
|
|
5
|
+
|
|
6
|
+
export { MlflowExporter } from "./MlflowExporter.js"
|
|
7
|
+
export { DatabricksExporter } from "./DatabricksExporter.js"
|
|
8
|
+
|
|
9
|
+
let _instance = null
|
|
10
|
+
let _instanceHost = null // track which host the singleton was built for
|
|
11
|
+
|
|
12
|
+
// Returns the singleton exporter, rebuilding it when credentials change.
|
|
13
|
+
// Returns null when MLflow is disabled or credentials are missing.
|
|
14
|
+
export function getMlflowExporter() {
|
|
15
|
+
if (!cds.env.agents?.mlflow) return null
|
|
16
|
+
const creds = resolveMlflowCredentials()
|
|
17
|
+
if (!creds) return null
|
|
18
|
+
// Rebuild if the host changed (e.g. between tests or env reloads)
|
|
19
|
+
if (_instance && _instanceHost === creds.host) return _instance
|
|
20
|
+
_instance = creds.uc ? new DatabricksExporter(creds) : new MlflowExporter(creds)
|
|
21
|
+
_instanceHost = creds.host
|
|
22
|
+
return _instance
|
|
23
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export {
|
|
2
|
+
mlflowAttrs,
|
|
3
|
+
mlflowTraceAttrs,
|
|
4
|
+
setSpanAttrs,
|
|
5
|
+
setupMlflowExporter,
|
|
6
|
+
flushMlflowTraces,
|
|
7
|
+
RoutingSpanProcessor,
|
|
8
|
+
} from "./tracing.js"
|
|
9
|
+
export { postMlflowAssessment, createEvalRun, closeEvalRun } from "./evaluation.js"
|
|
10
|
+
export {
|
|
11
|
+
syncPromptVersion,
|
|
12
|
+
syncSystemPrompt,
|
|
13
|
+
resolvePromptName,
|
|
14
|
+
linkedPromptsAttr,
|
|
15
|
+
hashPrompt,
|
|
16
|
+
} from "./prompts.js"
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import cds from "@sap/cds"
|
|
2
|
+
import { createHash } from "node:crypto"
|
|
3
|
+
import { join, relative } from "node:path"
|
|
4
|
+
import { getMlflowExporter } from "./exporter/index.js"
|
|
5
|
+
import { resolveAgentDir } from "../../utils/markdown.js"
|
|
6
|
+
import { resolveExperimentId } from "./credentials.js"
|
|
7
|
+
|
|
8
|
+
const _cache = new Map() // promptName → { version, hash }
|
|
9
|
+
|
|
10
|
+
// Register or reuse the prompt version in MLflow. Cache hit skips all I/O.
|
|
11
|
+
export async function syncPromptVersion(name, template) {
|
|
12
|
+
const exporter = getMlflowExporter()
|
|
13
|
+
if (!exporter) return null
|
|
14
|
+
|
|
15
|
+
const hash = hashPrompt(template)
|
|
16
|
+
const cached = _cache.get(name)
|
|
17
|
+
if (cached?.hash === hash) return { name, version: cached.version }
|
|
18
|
+
|
|
19
|
+
try {
|
|
20
|
+
const prompt = await exporter.ensurePrompt(name, _description(name), _registrationTags())
|
|
21
|
+
let version
|
|
22
|
+
if (_extractHash(prompt?.latestVersion?.tags) === hash) {
|
|
23
|
+
version = prompt?.latestVersion?.version
|
|
24
|
+
} else {
|
|
25
|
+
version = await exporter.createPromptVersion(
|
|
26
|
+
name,
|
|
27
|
+
`Hash ${hash}`,
|
|
28
|
+
_versionTags(template, hash),
|
|
29
|
+
template,
|
|
30
|
+
)
|
|
31
|
+
}
|
|
32
|
+
if (!version) return null
|
|
33
|
+
_cache.set(name, { hash, version })
|
|
34
|
+
_linkToExperiment(exporter, name, prompt?.tags).catch(() => {})
|
|
35
|
+
return { name, version }
|
|
36
|
+
} catch {
|
|
37
|
+
return null
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// JSON for mlflow.traceTag.mlflow.linkedPrompts span attribute, or null if not cached.
|
|
42
|
+
export function linkedPromptsAttr(promptName) {
|
|
43
|
+
const cached = _cache.get(promptName)
|
|
44
|
+
if (!cached) return null
|
|
45
|
+
return JSON.stringify([{ name: promptName, version: cached.version }])
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Extracts the SystemMessage from prepared LLM messages and syncs it to MLflow.
|
|
49
|
+
export function syncSystemPrompt(messages) {
|
|
50
|
+
if (!cds.env.agents?.mlflow) return
|
|
51
|
+
const srvName = cds.context?.["agent.service"]
|
|
52
|
+
if (!srvName) return
|
|
53
|
+
const sysMsg = messages?.find((m) => m.type === "system")
|
|
54
|
+
if (!sysMsg) return
|
|
55
|
+
const text =
|
|
56
|
+
typeof sysMsg.content === "string"
|
|
57
|
+
? sysMsg.content
|
|
58
|
+
: Array.isArray(sysMsg.content)
|
|
59
|
+
? sysMsg.content
|
|
60
|
+
.filter((b) => b?.type === "text")
|
|
61
|
+
.map((b) => b.text ?? "")
|
|
62
|
+
.join("")
|
|
63
|
+
: null
|
|
64
|
+
if (!text) return
|
|
65
|
+
const srv = cds.services[srvName]
|
|
66
|
+
if (!srv) return
|
|
67
|
+
syncPromptVersion(resolvePromptName(srv), text).catch(() => {})
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// AGENTS.md path relative to cds.root, or srv.name when no AGENTS.md exists.
|
|
71
|
+
export function resolvePromptName(srv) {
|
|
72
|
+
if (!srv?.name) return ""
|
|
73
|
+
const agentDir = resolveAgentDir(srv)
|
|
74
|
+
if (!agentDir) return srv.name
|
|
75
|
+
const root = cds.root ?? process.cwd()
|
|
76
|
+
return relative(root, join(agentDir, "AGENTS.md")).replace(/\\/g, "/")
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function hashPrompt(text) {
|
|
80
|
+
return createHash("sha256").update(text).digest("hex").slice(0, 16)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function getCachedPromptVersion(name) {
|
|
84
|
+
return _cache.get(name) ?? null
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function _description(name) {
|
|
88
|
+
return `System prompt for CAP agent service "${name.split("/")[0]}"`
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function _registrationTags() {
|
|
92
|
+
return [{ key: "mlflow.prompt.is_prompt", value: "true" }]
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function _versionTags(template, hash) {
|
|
96
|
+
return [
|
|
97
|
+
{ key: "mlflow.prompt.is_prompt", value: "true" }, // triggers prompt path on server → allows dummy-source
|
|
98
|
+
{ key: "mlflow.prompt.text", value: template },
|
|
99
|
+
{ key: "_mlflow_prompt_type", value: "text" },
|
|
100
|
+
{ key: "_cap_prompt_hash", value: hash },
|
|
101
|
+
]
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function _extractHash(tags) {
|
|
105
|
+
return tags?.find((t) => t.key === "_cap_prompt_hash")?.value ?? null
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Append experiment ID to the model-level "_mlflow_experiment_ids" tags so prompt appears in UI prompts tab.
|
|
109
|
+
async function _linkToExperiment(exporter, name, promptTags) {
|
|
110
|
+
const experimentId = resolveExperimentId()
|
|
111
|
+
if (!experimentId) return
|
|
112
|
+
const current = promptTags?.find((t) => t.key === "_mlflow_experiment_ids")?.value
|
|
113
|
+
const ids = current
|
|
114
|
+
? current
|
|
115
|
+
.replace(/^,|,$/g, "")
|
|
116
|
+
.split(",")
|
|
117
|
+
.map((s) => s.trim())
|
|
118
|
+
.filter(Boolean)
|
|
119
|
+
: []
|
|
120
|
+
if (ids.includes(String(experimentId))) return
|
|
121
|
+
ids.push(String(experimentId))
|
|
122
|
+
await exporter.setRegisteredModelTag(name, "_mlflow_experiment_ids", `,${ids.join(",")},`)
|
|
123
|
+
}
|