@cap-js/agents 0.9.6 → 0.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +2 -131
  2. package/_i18n/messages.properties +2 -0
  3. package/_i18n/messages_ar.properties +48 -0
  4. package/_i18n/messages_bg.properties +48 -0
  5. package/_i18n/messages_cs.properties +48 -0
  6. package/_i18n/messages_da.properties +48 -0
  7. package/_i18n/messages_de.properties +48 -0
  8. package/_i18n/messages_el.properties +48 -0
  9. package/_i18n/messages_en.properties +48 -0
  10. package/_i18n/messages_es.properties +48 -0
  11. package/_i18n/messages_es_MX.properties +48 -0
  12. package/_i18n/messages_fi.properties +48 -0
  13. package/_i18n/messages_fr.properties +48 -0
  14. package/_i18n/messages_he.properties +48 -0
  15. package/_i18n/messages_hr.properties +48 -0
  16. package/_i18n/messages_hu.properties +48 -0
  17. package/_i18n/messages_it.properties +48 -0
  18. package/_i18n/messages_ja.properties +48 -0
  19. package/_i18n/messages_kk.properties +48 -0
  20. package/_i18n/messages_ko.properties +48 -0
  21. package/_i18n/messages_ms.properties +48 -0
  22. package/_i18n/messages_nl.properties +48 -0
  23. package/_i18n/messages_no.properties +48 -0
  24. package/_i18n/messages_pl.properties +48 -0
  25. package/_i18n/messages_pt.properties +48 -0
  26. package/_i18n/messages_ro.properties +48 -0
  27. package/_i18n/messages_ru.properties +48 -0
  28. package/_i18n/messages_sh.properties +48 -0
  29. package/_i18n/messages_sk.properties +48 -0
  30. package/_i18n/messages_sl.properties +48 -0
  31. package/_i18n/messages_sv.properties +48 -0
  32. package/_i18n/messages_th.properties +48 -0
  33. package/_i18n/messages_tr.properties +48 -0
  34. package/_i18n/messages_uk.properties +48 -0
  35. package/_i18n/messages_vi.properties +48 -0
  36. package/_i18n/messages_zh_CN.properties +48 -0
  37. package/_i18n/messages_zh_TW.properties +48 -0
  38. package/cds-plugin.js +15 -10
  39. package/lib/agents/middleware/content-filter.js +4 -0
  40. package/lib/agents/middleware/hitl-decision-note-injector.js +18 -6
  41. package/lib/agents/middleware/hitl.js +17 -6
  42. package/lib/agents/middleware/index.js +3 -1
  43. package/lib/agents/middleware/masking.js +127 -0
  44. package/lib/agents/middleware/remote-mcp.js +10 -5
  45. package/lib/agents/middleware/status-update.js +209 -76
  46. package/lib/agents/middleware/tool-wrap.js +1 -2
  47. package/lib/agents/summarize-on-timeout.js +5 -2
  48. package/lib/config/local.js +121 -8
  49. package/lib/eval/eval-run.js +50 -4
  50. package/lib/masking/index.js +90 -0
  51. package/lib/masking/store.js +80 -0
  52. package/lib/masking/structured/findElements.js +459 -0
  53. package/lib/masking/structured/index.js +170 -0
  54. package/lib/masking/unstructured/dpi.js +84 -0
  55. package/lib/masking/unstructured/hana.js +72 -0
  56. package/lib/masking/unstructured/index.js +52 -0
  57. package/lib/models/aicore.js +16 -3
  58. package/lib/models/openai.js +9 -0
  59. package/lib/preview/chat.html +623 -119
  60. package/lib/protocol/persistence/cleanup.js +13 -4
  61. package/lib/telemetry/chat-tracing.js +9 -2
  62. package/lib/telemetry/mlflow/evaluation.js +83 -4
  63. package/lib/telemetry/mlflow/exporter/DatabricksExporter.js +39 -5
  64. package/lib/telemetry/mlflow/exporter/MlflowExporter.js +43 -4
  65. package/lib/telemetry/mlflow/index.js +1 -7
  66. package/lib/telemetry/mlflow/prompts.js +15 -11
  67. package/lib/telemetry/mlflow/tracing.js +3 -3
  68. package/lib/telemetry/span-masking.js +165 -0
  69. package/lib/telemetry/tool-tracing.js +4 -4
  70. package/lib/utils/markdown.js +3 -10
  71. package/lib/utils/resilience.js +2 -1
  72. package/lib/utils/toml.js +67 -0
  73. package/lib/utils/usage.js +34 -0
  74. package/lib/utils/utils.js +43 -1
  75. package/package.json +33 -26
  76. package/srv/handlers/graph-executor/crash-handler.js +47 -0
  77. package/srv/handlers/graph-executor/hitl.js +27 -0
  78. package/srv/handlers/graph-executor.js +69 -20
  79. package/srv/handlers/index.js +6 -1
  80. package/srv/handlers/mcp-tools.js +11 -2
  81. package/srv/handlers/subagent-tools.js +13 -4
  82. package/srv/handlers/system-prompt.js +30 -20
  83. package/srv/handlers/tools.js +14 -13
@@ -0,0 +1,90 @@
1
+ import cds from "@sap/cds"
2
+ import { randomBytes } from "node:crypto"
3
+ import { toolName as normaliseName } from "../utils/utils.js"
4
+ import { PseudonymStore } from "./store.js"
5
+ import { pseudonymizeToolResult } from "./structured/index.js"
6
+ import anonymizeUnstructured from "./unstructured/index.js"
7
+
8
+ export function resolvePseudonyms(text) {
9
+ if (typeof text !== "string") return text
10
+ return cds.context?.["agent.pseudonyms"]?.resolveText(text) ?? text
11
+ }
12
+
13
+ /**
14
+ * Input: { data, type, seed?, metadata? }
15
+ * Output: { data, mappings?, metadata: { textAnalysisResults?, seed, type } }
16
+ */
17
+ export async function pseudonymize({ data, type, seed, metadata }, srv) {
18
+ const effectiveSeed = seed ?? randomBytes(16).toString("hex")
19
+ const result = { metadata: { seed: effectiveSeed, type } }
20
+
21
+ if (type === "unstructured") {
22
+ const r = await anonymizeUnstructured(data, effectiveSeed)
23
+ result.data = r.text
24
+ if (r.mappings?.length) result.mappings = r.mappings
25
+ if (r.textAnalysisResults) result.metadata.textAnalysisResults = r.textAnalysisResults
26
+ } else {
27
+ // data is a decoded object (caller handles TOON/JSON decode+encode)
28
+ const toolName = metadata?.toolName
29
+ const srvName = metadata?.serviceName ?? srv?.name
30
+ const targetSrv = cds.services?.[srvName] ?? srv
31
+ const model = targetSrv?.model
32
+
33
+ const store = new PseudonymStore(effectiveSeed)
34
+
35
+ pseudonymizeToolResult({
36
+ decoded: data,
37
+ toolName,
38
+ cql: metadata?.cql,
39
+ strictMasking: metadata?.strictMasking ?? false,
40
+ session: store,
41
+ srv: targetSrv,
42
+ model,
43
+ })
44
+ result.data = data
45
+
46
+ if (store._hashToOriginal.size) result.mappings = [...store._hashToOriginal]
47
+ }
48
+
49
+ return result
50
+ }
51
+
52
+ export async function ensureSession(checkpointer, threadId) {
53
+ if (cds.context?.["agent.pseudonyms"]) return
54
+ const { seed, hashToOriginal } = await readMaskingState(checkpointer, threadId)
55
+ const session = new PseudonymStore(seed, hashToOriginal)
56
+ cds.context["agent.pseudonyms"] = session
57
+ }
58
+
59
+ // REVISIT: Should not read checkpointer, should move into a beforeAgent hook
60
+ async function readMaskingState(checkpointer, thread_id) {
61
+ if (!checkpointer || !thread_id) return {}
62
+ try {
63
+ const tuple = await checkpointer.getTuple({ configurable: { thread_id } })
64
+ const values = tuple?.checkpoint?.channel_values
65
+ if (!values) return {}
66
+ return {
67
+ seed: values.seed,
68
+ hashToOriginal: values.hashToOriginal,
69
+ }
70
+ } catch {
71
+ return {}
72
+ }
73
+ }
74
+
75
+ // For a remote MCP tool name like "catalogservice_query", strip the service prefix
76
+ // and return { bareName, remoteSrv }. Returns null for local tools.
77
+ export function resolveRemoteMcpTool(prefixedName) {
78
+ const cache = cds.context?.__mcpDynamicTools
79
+ if (!cache) return null
80
+ for (const { serviceName, tools } of Object.values(cache)) {
81
+ const prefix = normaliseName(`${serviceName}_`)
82
+ if (!tools?.some((t) => t.name === prefixedName)) continue
83
+ // If multiple CAP MCPs are included its possible that one of the CAP MCPs owns query if the agent itself does not expose any entities
84
+ return {
85
+ bareName: prefixedName.startsWith(prefix) ? prefixedName.slice(prefix.length) : prefixedName,
86
+ remoteSrv: cds.services?.[serviceName],
87
+ }
88
+ }
89
+ return null
90
+ }
@@ -0,0 +1,80 @@
1
+ import { createHash, randomBytes } from "node:crypto"
2
+
3
+ export function generatePseudonymTag(seed, value, category) {
4
+ const hash = createHash("sha256")
5
+ .update(seed + String(value))
6
+ .digest("hex")
7
+ .slice(0, 16)
8
+ return `${category}-${hash}`
9
+ }
10
+
11
+ export class PseudonymStore {
12
+ constructor(seed, existing = new Map()) {
13
+ this._seed = seed ?? randomBytes(16).toString("hex")
14
+ this._hashToOriginal = new Map(existing)
15
+ this._originalToHash = new Map()
16
+ // Sort by descending key length so tags ("name-abc12345") are processed
17
+ // before bare hashes ("abc12345") and win in _originalToHash.
18
+ for (const [hash, original] of [...existing].sort((a, b) => b[0].length - a[0].length)) {
19
+ const str = String(original)
20
+ if (!this._originalToHash.has(str)) this._originalToHash.set(str, hash)
21
+ }
22
+ this._sortedOriginalPairs = null
23
+ this._sortedHashPairs = null
24
+ }
25
+
26
+ pseudonymize(value, propertyName) {
27
+ const str = String(value)
28
+ if (this._originalToHash.has(str)) return this._originalToHash.get(str)
29
+ const tag = generatePseudonymTag(this._seed, str, propertyName)
30
+ const hash = tag.slice(propertyName.length + 1)
31
+ this._hashToOriginal.set(tag, str)
32
+ this._hashToOriginal.set(hash, str)
33
+ this._originalToHash.set(str, tag)
34
+ // Invalidate sorted cache; rebuilt lazily on next scrub.
35
+ this._sortedOriginalPairs = null
36
+ this._sortedHashPairs = null
37
+ return tag
38
+ }
39
+
40
+ addMappings(mappings) {
41
+ if (!mappings?.length) return
42
+ // Sort by descending key length so tags ("name-abc12345") win over bare hashes ("abc12345").
43
+ const sorted = [...mappings].sort((a, b) => b[0].length - a[0].length)
44
+ for (const [hash, original] of sorted) {
45
+ const str = String(original)
46
+ const key = String(hash)
47
+ if (!str || !key || str === key) continue
48
+ if (!this._hashToOriginal.has(key)) this._hashToOriginal.set(key, str)
49
+ if (!this._originalToHash.has(str)) this._originalToHash.set(str, key)
50
+ }
51
+ this._sortedOriginalPairs = null
52
+ this._sortedHashPairs = null
53
+ }
54
+
55
+ resolve(hash) {
56
+ return this._hashToOriginal.get(hash) ?? hash
57
+ }
58
+
59
+ resolveText(text) {
60
+ if (!text || !this._hashToOriginal.size) return text
61
+ // tag and ID are registered. First go for tags then ID, else artefacts are left in text
62
+ this._sortedHashPairs ??= [...this._hashToOriginal].sort((a, b) => b[0].length - a[0].length)
63
+ let result = String(text)
64
+ for (const [hash, original] of this._sortedHashPairs) result = result.replaceAll(hash, original)
65
+ return result
66
+ }
67
+
68
+ scrubText(text) {
69
+ if (!text || !this._originalToHash.size) return text
70
+ // Replace longest originals first so a shorter original that is a substring
71
+ // of a longer one (e.g. "Emily" vs "Emily Brontë") does not corrupt it.
72
+ this._sortedOriginalPairs ??= [...this._originalToHash].sort(
73
+ (a, b) => b[0].length - a[0].length,
74
+ )
75
+ let result = String(text)
76
+ for (const [original, hash] of this._sortedOriginalPairs)
77
+ result = result.replaceAll(original, hash)
78
+ return result
79
+ }
80
+ }
@@ -0,0 +1,459 @@
1
+ import cds from "@sap/cds"
2
+
3
+ const PERSONAL_DATA_CACHE = Symbol.for("@cap-js/agents:personalDataPaths")
4
+
5
+ const STRING_TYPES = new Set(["cds.String", "cds.UUID", "cds.LargeString"])
6
+ const NUMERIC_TYPES = new Set(["cds.Integer", "cds.Integer64", "cds.Decimal", "cds.Double"])
7
+
8
+ const PD_TAGS = [
9
+ "@PersonalData.IsPotentiallyPersonal",
10
+ "@PersonalData.IsPotentiallySensitive",
11
+ "@Common.Masked",
12
+ ]
13
+
14
+ export function hasPersonalDataAnnotation(element) {
15
+ return (
16
+ PD_TAGS.some((a) => element[a] != null && element[a] !== false) ||
17
+ element["@PersonalData.FieldSemantics"] === "DataSubjectID" ||
18
+ element["@PersonalData.FieldSemantics"] === "UserID"
19
+ )
20
+ }
21
+
22
+ export function shouldHash(el) {
23
+ if (STRING_TYPES.has(el.type)) return true
24
+ if (NUMERIC_TYPES.has(el.type)) return !!(el.key || el._foreignKey4)
25
+ return false
26
+ }
27
+
28
+ // Returns the set of hashable @PersonalData element names for an entity.
29
+ // strictMasking=true: also includes @Common.Masked:false fields (traces must not expose them).
30
+ //
31
+ // Without a `model`, only top-level scalar/scalar-array elements are returned as plain
32
+ // names (used by join/nav/expand resolution helpers that key on single element names).
33
+ //
34
+ // With a `model`, complex-type elements are resolved recursively and their nested PII is
35
+ // returned as PATH ARRAYS (e.g. ["address","street"], ["address","geo","lat"]), and
36
+ // arrayed structs likewise (["contacts","email"]). Scalar-array PII (annotation on the
37
+ // arrayed element itself, e.g. `nicknames : many String`) is returned as a plain name.
38
+ // pseudonymizeData traverses path arrays and hashes scalar-array elements.
39
+ export function personalDataElements(entityDef, strictMasking = false, model = null) {
40
+ if (!entityDef?.elements) return new Set()
41
+ const result = new Set()
42
+ _collectPaths(entityDef, strictMasking, model, [], result, 0)
43
+ return result
44
+ }
45
+
46
+ function _collectPaths(entityDef, strictMasking, model, prefix, out, depth) {
47
+ if (!entityDef?.elements || depth > 20) return
48
+ for (const [name, el] of Object.entries(entityDef.elements)) {
49
+ if (!strictMasking && el["@Common.Masked"] === false) continue
50
+ const path = [...prefix, name]
51
+
52
+ // Scalar (String/UUID or numeric key/FK): hash when annotated PII.
53
+ if (shouldHash(el)) {
54
+ if (hasPersonalDataAnnotation(el)) out.add(path.length === 1 ? name : path)
55
+ continue
56
+ }
57
+
58
+ if (!model) continue // flat mode: only top-level scalars
59
+
60
+ // Resolve the element's structural type (named type, inline struct, or arrayed).
61
+ const resolved = _structOf(el, model)
62
+ if (resolved?.elements) {
63
+ _collectPaths(resolved, strictMasking, model, path, out, depth + 1)
64
+ } else if (resolved?.items) {
65
+ // Arrayed: struct items recurse; scalar items annotated on the arrayed element → plain path.
66
+ const itemStruct = _structOf(resolved.items, model)
67
+ if (itemStruct?.elements) {
68
+ _collectPaths(itemStruct, strictMasking, model, path, out, depth + 1)
69
+ } else if (hasPersonalDataAnnotation(el) && shouldHash(resolved.items)) {
70
+ out.add(path.length === 1 ? name : path)
71
+ }
72
+ }
73
+ }
74
+ }
75
+
76
+ // Resolve an element/type node to its structural definition:
77
+ // named type reference → model definition; inline struct/array → itself.
78
+ function _structOf(node, model) {
79
+ if (!node) return null
80
+ if (node.elements || node.items) return node
81
+ if (node.type) return model?.definitions?.[node.type] ?? null
82
+ return null
83
+ }
84
+
85
+ // Model-level memoization; invalidates automatically when the model object changes.
86
+ function cache(model, key, compute) {
87
+ model[PERSONAL_DATA_CACHE] ??= {}
88
+ if (model[PERSONAL_DATA_CACHE][key] !== undefined) return model[PERSONAL_DATA_CACHE][key]
89
+ return (model[PERSONAL_DATA_CACHE][key] = compute())
90
+ }
91
+
92
+ // Returns the `returns` type node of an action/function.
93
+ // Case-insensitive fallback handles tool names normalised by toolName() (lowercased).
94
+ export function actionReturnType(model, srv, actionName) {
95
+ const exact = model.definitions?.[`${srv.name}.${actionName}`]
96
+ if (exact) return exact.returns ?? null
97
+ const lower = `${srv.name}.${actionName}`.toLowerCase()
98
+ const key = Object.keys(model.definitions ?? {}).find((k) => k.toLowerCase() === lower)
99
+ return key ? (model.definitions[key].returns ?? null) : null
100
+ }
101
+
102
+ export function hasPersonalDataAnnotations(model, serviceName) {
103
+ return cache(model, `hasAny:${serviceName}`, () =>
104
+ Object.entries(model.definitions ?? {}).some(([name, def]) => {
105
+ if (!name.startsWith(serviceName + ".") && name !== serviceName) return false
106
+ if (!def.elements) return false
107
+ return Object.values(def.elements).some((el) => hasPersonalDataAnnotation(el))
108
+ }),
109
+ )
110
+ }
111
+
112
+ export function discoverElementsToBeMasked(model, srv, cql, strictMasking) {
113
+ if (!cql) return new Set()
114
+ return cache(model, `query:${srv.name}:${cql}:${strictMasking}`, () => {
115
+ let cqn
116
+ try {
117
+ cqn = cds.parse.cql(cql)
118
+ } catch {
119
+ return new Set()
120
+ }
121
+ return _discoverFromCqn(model, srv, cqn, strictMasking)
122
+ })
123
+ }
124
+
125
+ // Resolve { ref, as? } to its entity def + alias. Returns null when unresolvable.
126
+ function _resolveSource(model, srv, node) {
127
+ const ref = node?.ref
128
+ const raw = typeof ref?.[0] === "string" ? ref[0] : ref?.[0]?.id
129
+ if (!raw) return null
130
+ const localName = raw.includes(".") ? raw : `${srv.name}.${raw}`
131
+ const entityDef = model.definitions?.[localName] ?? model.definitions?.[raw]
132
+ if (!entityDef) return null
133
+ const alias = node.as ?? raw.split(".").pop()
134
+ return { alias, entityDef }
135
+ }
136
+
137
+ // Collect all source entities from a FROM clause into a Map keyed by alias.
138
+ // Handles: plain ref, JOIN ({ join, args }), subquery ({ SELECT }), union-in-FROM ({ SET }).
139
+ // The primary (leading) source is additionally registered under the key `null`, so
140
+ // callers can resolve unqualified refs and unqualified expand roots without knowing
141
+ // the alias — even in joins where the primary source carries an alias.
142
+ function _collectSources(model, srv, from, out = new Map()) {
143
+ if (!from) return out
144
+ if (Array.isArray(from.args)) {
145
+ for (const arg of from.args) _collectSources(model, srv, arg, out)
146
+ return out
147
+ }
148
+ if (from.SELECT) return _collectSources(model, srv, from.SELECT.from, out)
149
+ if (from.SET) {
150
+ for (const branch of from.SET.args ?? [])
151
+ if (branch.SELECT) _collectSources(model, srv, branch.SELECT.from, out)
152
+ return out
153
+ }
154
+ const src = _resolveSource(model, srv, from)
155
+ if (src) {
156
+ out.set(src.alias, src)
157
+ if (!out.has(null)) out.set(null, src) // first resolved source is the primary
158
+ }
159
+ return out
160
+ }
161
+
162
+ // Extract all ref arrays from xpr/func.args nodes (not into SELECT — see _extractSubselects).
163
+ function _extractRefs(node) {
164
+ if (!node) return []
165
+ if (Array.isArray(node)) return node.flatMap(_extractRefs)
166
+ if (typeof node !== "object") return []
167
+ if (node.SELECT) return []
168
+ if (node.ref) return [node.ref]
169
+ if (node.xpr) return _extractRefs(node.xpr)
170
+ if (node.args) return _extractRefs(node.args)
171
+ return []
172
+ }
173
+
174
+ // Extract all scalar subselect nodes ({ SELECT }) from xpr/func.args.
175
+ function _extractSubselects(node) {
176
+ if (!node) return []
177
+ if (Array.isArray(node)) return node.flatMap(_extractSubselects)
178
+ if (typeof node !== "object") return []
179
+ if (node.SELECT) return [node]
180
+ const found = []
181
+ if (node.xpr) found.push(..._extractSubselects(node.xpr))
182
+ if (node.args) found.push(..._extractSubselects(node.args))
183
+ return found
184
+ }
185
+
186
+ // True when a ref resolves to a PII element via annotatedByAlias or nav-path traversal.
187
+ function _isRefPii(ref, annotatedByAlias, primaryEntity, model) {
188
+ if (!ref?.length) return false
189
+ const element = ref[ref.length - 1]
190
+
191
+ // Unqualified: binds to whichever source has the element.
192
+ if (ref.length === 1) {
193
+ for (const annotated of annotatedByAlias.values()) if (annotated.has(element)) return true
194
+ return false
195
+ }
196
+
197
+ // Qualified: ref[0] is a table alias, or an association path on the primary source.
198
+ let annotated = annotatedByAlias.get(ref[0])
199
+ if (!annotated && primaryEntity) {
200
+ let entityDef = primaryEntity
201
+ for (let i = 0; i < ref.length - 1 && entityDef; i++) {
202
+ const assocEl = entityDef.elements?.[ref[i]]
203
+ entityDef = assocEl?.target ? model.definitions?.[assocEl.target] : undefined
204
+ }
205
+ if (entityDef) annotated = personalDataElements(entityDef)
206
+ }
207
+ return annotated?.has(element) ?? false
208
+ }
209
+
210
+ // Recursively walk expand columns under `expandCols` rooted at `entityDef`.
211
+ // Emits path arrays like ["author","contact","email"] into `out` for every PII leaf.
212
+ // assocPath: path segments accumulated so far (e.g. ["author","contact"]).
213
+ function _expandPiiPaths(model, srv, entityDef, expandCols, assocPath, strictMasking, out) {
214
+ if (!entityDef || !expandCols) return
215
+ const annotated = personalDataElements(entityDef, strictMasking)
216
+ for (const expandCol of expandCols) {
217
+ if (expandCol.SELECT && expandCol.as) {
218
+ if (_discoverFromCqn(model, srv, expandCol, strictMasking).size > 0)
219
+ out.add([...assocPath, expandCol.as])
220
+ continue
221
+ }
222
+ const colName = expandCol.ref?.[expandCol.ref.length - 1]
223
+ if (!colName) continue
224
+ if (expandCol.expand) {
225
+ const assocEl = entityDef.elements?.[colName]
226
+ const targetDef = assocEl?.target ? model.definitions?.[assocEl.target] : undefined
227
+ _expandPiiPaths(
228
+ model,
229
+ srv,
230
+ targetDef,
231
+ expandCol.expand,
232
+ [...assocPath, expandCol.as ?? colName],
233
+ strictMasking,
234
+ out,
235
+ )
236
+ } else if (annotated.has(colName)) {
237
+ out.add([...assocPath, expandCol.as ?? colName])
238
+ }
239
+ }
240
+ }
241
+
242
+ // Map a set of inner PII field names through the outer SELECT's columns.
243
+ // `byInnerName`: true → look up col.ref[last] (subquery rename), result key = col.as ?? innerName.
244
+ // false → look up col.as ?? col.ref[last] directly (union, no rename).
245
+ // SELECT * (or no columns) passes the inner set through unchanged.
246
+ function _mapThroughColumns(columns, innerPii, byInnerName) {
247
+ if (!columns || columns.some((c) => c === "*")) return innerPii
248
+ const result = new Set()
249
+ for (const col of columns) {
250
+ const lastRef = col.ref?.[col.ref.length - 1]
251
+ if (byInnerName) {
252
+ if (lastRef && innerPii.has(lastRef)) result.add(col.as ?? lastRef)
253
+ } else {
254
+ const name = col.as ?? lastRef
255
+ if (name && innerPii.has(name)) result.add(name)
256
+ }
257
+ }
258
+ return result
259
+ }
260
+
261
+ // Resolve a plain ref column to the PII result keys it contributes.
262
+ // Returns an array: [] none, [name] a scalar/scalar-array PII column, or one-or-more
263
+ // path arrays when the column is a complex/arrayed type carrying nested PII
264
+ // (e.g. SELECT address → [["address","street"], ["address","geo","lat"], ...]).
265
+ // `ctx` carries the per-query lookup state.
266
+ function _resolvePlainRef(col, ctx) {
267
+ const { model, sources, primaryEntity, strictMasking } = ctx
268
+ const ref = col.ref
269
+ const element = ref[ref.length - 1]
270
+
271
+ // Find the source entity the column binds to.
272
+ let entityDef
273
+ if (ref.length === 1) {
274
+ // Unqualified: binds to whichever source has the element.
275
+ for (const [alias, src] of sources) {
276
+ if (alias === null) continue
277
+ if (src.entityDef.elements?.[element]) {
278
+ entityDef = src.entityDef
279
+ break
280
+ }
281
+ }
282
+ } else {
283
+ // Qualified: ref[0] is a table alias, or an association path on the primary source.
284
+ entityDef = sources.get(ref[0])?.entityDef
285
+ if (!entityDef && primaryEntity) {
286
+ let walk = primaryEntity
287
+ for (let i = 0; i < ref.length - 1 && walk; i++) {
288
+ const assocEl = walk.elements?.[ref[i]]
289
+ walk = assocEl?.target ? model.definitions?.[assocEl.target] : undefined
290
+ }
291
+ entityDef = walk
292
+ }
293
+ }
294
+ const elDef = entityDef?.elements?.[element]
295
+ if (!elDef) return []
296
+ if (!strictMasking && elDef["@Common.Masked"] === false) return []
297
+
298
+ const outKey = col.as ?? element
299
+
300
+ // Scalar (incl. scalar array annotated on the element): single result key.
301
+ if (shouldHash(elDef)) {
302
+ return hasPersonalDataAnnotation(elDef) ? [outKey] : []
303
+ }
304
+
305
+ // Complex / arrayed type: collect nested PII as paths under the column's output key.
306
+ const nested = new Set()
307
+ _collectComplexPaths(elDef, strictMasking, model, [outKey], nested, 0)
308
+ return [...nested]
309
+ }
310
+
311
+ // Walk a single complex/arrayed element's type, emitting path arrays for nested PII.
312
+ function _collectComplexPaths(elDef, strictMasking, model, prefix, out, depth) {
313
+ if (depth > 20) return
314
+ const resolved = _structOf(elDef, model)
315
+ if (resolved?.elements) {
316
+ for (const [name, child] of Object.entries(resolved.elements)) {
317
+ if (!strictMasking && child["@Common.Masked"] === false) continue
318
+ const path = [...prefix, name]
319
+ if (shouldHash(child)) {
320
+ if (hasPersonalDataAnnotation(child)) out.add(path)
321
+ } else {
322
+ _collectComplexPaths(child, strictMasking, model, path, out, depth + 1)
323
+ }
324
+ }
325
+ } else if (resolved?.items) {
326
+ const itemStruct = _structOf(resolved.items, model)
327
+ if (itemStruct?.elements) {
328
+ _collectComplexPaths(
329
+ { elements: itemStruct.elements },
330
+ strictMasking,
331
+ model,
332
+ prefix,
333
+ out,
334
+ depth + 1,
335
+ )
336
+ } else if (hasPersonalDataAnnotation(elDef) && shouldHash(resolved.items)) {
337
+ out.add(prefix)
338
+ }
339
+ }
340
+ }
341
+
342
+ // Resolve the target entity def reached by walking a ref path from the sources.
343
+ // Unqualified expand root (author { ... }) binds to the primary source.
344
+ function _resolveExpandTarget(col, ctx) {
345
+ const { model, sources, primaryEntity } = ctx
346
+ const firstSeg = col.ref.length > 1 ? col.ref[0] : null
347
+ let targetDef =
348
+ firstSeg !== null ? (sources.get(firstSeg)?.entityDef ?? primaryEntity) : primaryEntity
349
+ for (const seg of col.ref) {
350
+ const assocEl = targetDef?.elements?.[seg]
351
+ targetDef = assocEl?.target ? model.definitions?.[assocEl.target] : undefined
352
+ }
353
+ return targetDef
354
+ }
355
+
356
+ function _discoverFromCqn(model, srv, cqn, strictMasking) {
357
+ // Top-level SET: union results from every branch.
358
+ if (cqn.SET) {
359
+ const all = new Set()
360
+ for (const branch of cqn.SET.args ?? [])
361
+ for (const f of _discoverFromCqn(model, srv, branch, strictMasking)) all.add(f)
362
+ return all
363
+ }
364
+
365
+ const from = cqn.SELECT?.from
366
+ const columns = cqn.SELECT?.columns
367
+
368
+ // SET in FROM (SELECT name FROM (Books UNION Authors)): union branch results,
369
+ // then map through outer columns by their output name (no rename at union level).
370
+ if (from?.SET) {
371
+ const innerPii = new Set()
372
+ for (const branch of from.SET.args ?? [])
373
+ for (const f of _discoverFromCqn(model, srv, branch, strictMasking)) innerPii.add(f)
374
+ return _mapThroughColumns(columns, innerPii, false)
375
+ }
376
+
377
+ // Subquery FROM: recurse into the inner SELECT, then map through outer columns
378
+ // by the inner (renamed) field name.
379
+ if (from?.SELECT) {
380
+ const innerPii = _discoverFromCqn(model, srv, from, strictMasking)
381
+ if (innerPii.size) return _mapThroughColumns(columns, innerPii, true)
382
+ }
383
+
384
+ const sources = _collectSources(model, srv, from)
385
+ const primaryEntity = sources.get(null)?.entityDef
386
+ // Flat sets (top-level scalar names) for join/nav column resolution.
387
+ const annotatedByAlias = new Map()
388
+ for (const [alias, { entityDef }] of sources) {
389
+ if (alias === null) continue // primary is also stored under its real alias
390
+ annotatedByAlias.set(alias, personalDataElements(entityDef, strictMasking))
391
+ }
392
+
393
+ const anyAnnotated = [...annotatedByAlias.values()].some((s) => s.size)
394
+ const hasNavigationOrExpr =
395
+ columns &&
396
+ columns.some((c) => {
397
+ if (!c || typeof c !== "object") return false
398
+ return (
399
+ (Array.isArray(c?.ref) && (c.ref.length > 1 || c.expand)) || c.xpr || c.func || c.SELECT
400
+ )
401
+ })
402
+ // A source may have PII only inside complex/arrayed elements (not caught by the flat
403
+ // set above) — recompute with the model to detect those before bailing out.
404
+ const anyDeepAnnotated =
405
+ anyAnnotated ||
406
+ [...sources].some(
407
+ ([alias, { entityDef }]) =>
408
+ alias !== null && personalDataElements(entityDef, strictMasking, model).size,
409
+ )
410
+ if (!anyDeepAnnotated && !hasNavigationOrExpr) return new Set()
411
+
412
+ // SELECT * — every source's annotated elements, resolving complex/arrayed types
413
+ // into path arrays so nested PII is masked too.
414
+ if (!columns || columns.some((c) => c === "*")) {
415
+ const all = new Set()
416
+ for (const [alias, { entityDef }] of sources) {
417
+ if (alias === null) continue
418
+ for (const n of personalDataElements(entityDef, strictMasking, model)) all.add(n)
419
+ }
420
+ return all
421
+ }
422
+
423
+ const ctx = { model, srv, strictMasking, sources, annotatedByAlias, primaryEntity }
424
+
425
+ const resultNames = new Set()
426
+ for (const col of columns) {
427
+ if (col.expand && col.ref?.length) {
428
+ // Expand: SELECT author { name } — emits path arrays e.g. ["author","name"]
429
+ const assocPath = col.as ? [col.as] : [...col.ref]
430
+ _expandPiiPaths(
431
+ model,
432
+ srv,
433
+ _resolveExpandTarget(col, ctx),
434
+ col.expand,
435
+ assocPath,
436
+ strictMasking,
437
+ resultNames,
438
+ )
439
+ } else if (col.ref?.length) {
440
+ // Plain ref: scalar → name; complex/arrayed → one or more path arrays.
441
+ for (const key of _resolvePlainRef(col, ctx)) resultNames.add(key)
442
+ } else if (col.SELECT && col.as) {
443
+ // Scalar subselect as column
444
+ if (_discoverFromCqn(model, srv, col, strictMasking).size > 0) resultNames.add(col.as)
445
+ } else if ((col.xpr || col.func) && col.as) {
446
+ // Expression / function: check extracted refs, then nested subselects
447
+ const byRef = _extractRefs(col).some((ref) =>
448
+ _isRefPii(ref, annotatedByAlias, primaryEntity, model),
449
+ )
450
+ const bySub =
451
+ !byRef &&
452
+ _extractSubselects(col).some(
453
+ (sub) => _discoverFromCqn(model, srv, sub, strictMasking).size > 0,
454
+ )
455
+ if (byRef || bySub) resultNames.add(col.as)
456
+ }
457
+ }
458
+ return resultNames
459
+ }