@duckcodeailabs/dql-core 1.7.2 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/dashboard-document.d.ts +1 -1
- package/dist/apps/dashboard-document.d.ts.map +1 -1
- package/dist/apps/dashboard-document.js +1 -1
- package/dist/apps/dashboard-document.js.map +1 -1
- package/dist/apps/product-domain-context.d.ts +3 -0
- package/dist/apps/product-domain-context.d.ts.map +1 -1
- package/dist/apps/product-domain-context.js +1 -0
- package/dist/apps/product-domain-context.js.map +1 -1
- package/dist/manifest/builder.d.ts.map +1 -1
- package/dist/manifest/builder.js +77 -3
- package/dist/manifest/builder.js.map +1 -1
- package/dist/manifest/index.d.ts +3 -1
- package/dist/manifest/index.d.ts.map +1 -1
- package/dist/manifest/index.js +2 -0
- package/dist/manifest/index.js.map +1 -1
- package/dist/manifest/knowledge-graph.d.ts +15 -0
- package/dist/manifest/knowledge-graph.d.ts.map +1 -0
- package/dist/manifest/knowledge-graph.js +616 -0
- package/dist/manifest/knowledge-graph.js.map +1 -0
- package/dist/manifest/knowledge-skills.d.ts +42 -0
- package/dist/manifest/knowledge-skills.d.ts.map +1 -0
- package/dist/manifest/knowledge-skills.js +173 -0
- package/dist/manifest/knowledge-skills.js.map +1 -0
- package/dist/manifest/types.d.ts +115 -0
- package/dist/manifest/types.d.ts.map +1 -1
- package/dist/semantic/providers/dbt-provider.d.ts.map +1 -1
- package/dist/semantic/providers/dbt-provider.js +76 -23
- package/dist/semantic/providers/dbt-provider.js.map +1 -1
- package/dist/semantic/providers/provider.d.ts +6 -0
- package/dist/semantic/providers/provider.d.ts.map +1 -1
- package/dist/semantic/semantic-layer.d.ts.map +1 -1
- package/dist/semantic/semantic-layer.js +138 -16
- package/dist/semantic/semantic-layer.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
export const KNOWLEDGE_INDEX_SCHEMA_VERSION = 4;
|
|
3
|
+
/** Build the single qualified policy graph after every source object resolves. */
|
|
4
|
+
export function buildManifestKnowledgeGraph(input) {
|
|
5
|
+
const { manifest, skills } = input;
|
|
6
|
+
const knowledgeBlocks = input.blocks ?? Object.values(manifest.blocks);
|
|
7
|
+
const diagnostics = [];
|
|
8
|
+
const objects = {};
|
|
9
|
+
const edges = [];
|
|
10
|
+
const edgeKeys = new Set();
|
|
11
|
+
const domainAliases = buildDomainAliases(manifest);
|
|
12
|
+
const canonicalDomain = (value) => value ? domainAliases.get(value.toLowerCase()) ?? value : undefined;
|
|
13
|
+
const qualified = (kind, localId, domain) => `${domain ?? 'global'}::${kind}::${localId}`;
|
|
14
|
+
const addObject = (object) => {
|
|
15
|
+
const existing = objects[object.id];
|
|
16
|
+
if (existing && stable(existing) !== stable(object)) {
|
|
17
|
+
diagnostics.push({ kind: 'resolve', filePath: object.source.path, severity: 'error', message: `Knowledge object identity collision for "${object.id}".` });
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
objects[object.id] = object;
|
|
21
|
+
};
|
|
22
|
+
const addEdge = (kind, from, to, extra = {}) => {
|
|
23
|
+
const key = `${kind}:${from}:${to}:${stable(extra)}`;
|
|
24
|
+
if (edgeKeys.has(key))
|
|
25
|
+
return;
|
|
26
|
+
edgeKeys.add(key);
|
|
27
|
+
const fingerprint = digest(key);
|
|
28
|
+
edges.push({ id: `edge::${fingerprint.slice(0, 20)}`, kind, from, to, ...extra, fingerprint });
|
|
29
|
+
};
|
|
30
|
+
for (const [key, domain] of Object.entries(manifest.domains ?? {})) {
|
|
31
|
+
const domainId = domain.id ?? key;
|
|
32
|
+
addObject(object('domain', `domain::${domainId}`, domainId, domain.filePath, {
|
|
33
|
+
domainId,
|
|
34
|
+
owner: domain.owner,
|
|
35
|
+
aliases: unique([domain.name, key, domainId]),
|
|
36
|
+
payload: {
|
|
37
|
+
name: domain.name,
|
|
38
|
+
parent: domain.parent,
|
|
39
|
+
boundedContext: domain.boundedContext,
|
|
40
|
+
description: domain.description,
|
|
41
|
+
inScope: domain.inScope ?? [],
|
|
42
|
+
outOfScope: domain.outOfScope ?? [],
|
|
43
|
+
sourceSystems: domain.sourceSystems ?? [],
|
|
44
|
+
primaryTerms: domain.primaryTerms ?? [],
|
|
45
|
+
},
|
|
46
|
+
}));
|
|
47
|
+
if (domain.parent)
|
|
48
|
+
addEdge('parent_domain', `domain::${canonicalDomain(domain.parent)}`, `domain::${domainId}`);
|
|
49
|
+
}
|
|
50
|
+
for (const area of Object.values(manifest.modeling?.areas ?? {})) {
|
|
51
|
+
addObject(object('model_area', area.qualifiedId, area.localId, area.sourcePath, {
|
|
52
|
+
domainId: area.domain,
|
|
53
|
+
aliases: unique([area.name, area.localId, area.qualifiedId]),
|
|
54
|
+
payload: { description: area.description, intentExamples: area.intentExamples, referencedEntityIds: area.referencedEntityIds },
|
|
55
|
+
}));
|
|
56
|
+
addEdge('contains', `domain::${area.domain}`, area.qualifiedId);
|
|
57
|
+
}
|
|
58
|
+
const blockIds = new Map();
|
|
59
|
+
for (const block of knowledgeBlocks) {
|
|
60
|
+
const domainId = canonicalDomain(block.domain);
|
|
61
|
+
const id = qualified('block', block.name, domainId);
|
|
62
|
+
addLookup(blockIds, block.name, id, domainId);
|
|
63
|
+
addObject(object('block', id, block.name, block.filePath, {
|
|
64
|
+
domainId,
|
|
65
|
+
owner: block.owner,
|
|
66
|
+
status: block.status,
|
|
67
|
+
aliases: unique([block.name, block.description ?? '']),
|
|
68
|
+
payload: {
|
|
69
|
+
description: block.description,
|
|
70
|
+
grain: block.grain,
|
|
71
|
+
entities: block.entities ?? [],
|
|
72
|
+
metricRefs: unique([...(block.metricsRef ?? []), ...(block.metricRef ? [block.metricRef] : []), ...(block.metricRefs ?? [])]),
|
|
73
|
+
dimensions: block.dimensions ?? [],
|
|
74
|
+
allowedFilters: block.allowedFilters ?? [],
|
|
75
|
+
declaredOutputs: block.declaredOutputs ?? [],
|
|
76
|
+
outputContract: block.outputContract ?? [],
|
|
77
|
+
dataState: block.dataState,
|
|
78
|
+
},
|
|
79
|
+
}));
|
|
80
|
+
if (domainId)
|
|
81
|
+
addEdge('contains', `domain::${domainId}`, id);
|
|
82
|
+
}
|
|
83
|
+
const termIds = new Map();
|
|
84
|
+
for (const term of Object.values(manifest.terms ?? {})) {
|
|
85
|
+
const domainId = canonicalDomain(term.domain);
|
|
86
|
+
const id = qualified('term', term.name, domainId);
|
|
87
|
+
addLookup(termIds, term.name, id, domainId);
|
|
88
|
+
addObject(object('term', id, term.name, term.filePath, {
|
|
89
|
+
domainId,
|
|
90
|
+
owner: term.owner,
|
|
91
|
+
status: term.status,
|
|
92
|
+
aliases: unique([term.name, ...(term.identifiers ?? []), ...(term.synonyms ?? [])]),
|
|
93
|
+
payload: { description: term.description, businessRules: term.businessRules ?? [], caveats: term.caveats ?? [] },
|
|
94
|
+
}));
|
|
95
|
+
if (domainId)
|
|
96
|
+
addEdge('contains', `domain::${domainId}`, id);
|
|
97
|
+
}
|
|
98
|
+
const viewIds = new Map();
|
|
99
|
+
for (const view of Object.values(manifest.businessViews ?? {})) {
|
|
100
|
+
const domainId = canonicalDomain(view.domain);
|
|
101
|
+
const id = qualified('business_view', view.name, domainId);
|
|
102
|
+
addLookup(viewIds, view.name, id, domainId);
|
|
103
|
+
addObject(object('business_view', id, view.name, view.filePath, {
|
|
104
|
+
domainId,
|
|
105
|
+
owner: view.owner,
|
|
106
|
+
status: view.status,
|
|
107
|
+
aliases: unique([view.name, view.description ?? '']),
|
|
108
|
+
payload: { description: view.description, caveats: view.caveats ?? [] },
|
|
109
|
+
}));
|
|
110
|
+
if (domainId)
|
|
111
|
+
addEdge('contains', `domain::${domainId}`, id);
|
|
112
|
+
}
|
|
113
|
+
const metricIds = new Map();
|
|
114
|
+
for (const metric of Object.values(manifest.metrics ?? {})) {
|
|
115
|
+
const domainId = canonicalDomain(metric.domain);
|
|
116
|
+
const id = qualified('metric', metric.name, domainId);
|
|
117
|
+
addLookup(metricIds, metric.name, id, domainId);
|
|
118
|
+
addObject(object('metric', id, metric.name, metric.filePath, {
|
|
119
|
+
domainId,
|
|
120
|
+
owner: metric.owner,
|
|
121
|
+
status: metric.status,
|
|
122
|
+
aliases: unique([metric.name, metric.label ?? '', metric.description ?? '']),
|
|
123
|
+
sourceSystem: 'semantic',
|
|
124
|
+
payload: { description: metric.description, type: metric.type, table: metric.table },
|
|
125
|
+
}));
|
|
126
|
+
if (domainId)
|
|
127
|
+
addEdge('contains', `domain::${domainId}`, id);
|
|
128
|
+
}
|
|
129
|
+
const dimensionIds = new Map();
|
|
130
|
+
for (const dimension of Object.values(manifest.dimensions ?? {})) {
|
|
131
|
+
const domainId = canonicalDomain(dimension.domain);
|
|
132
|
+
const id = qualified('dimension', dimension.name, domainId);
|
|
133
|
+
addLookup(dimensionIds, dimension.name, id, domainId);
|
|
134
|
+
addObject(object('dimension', id, dimension.name, dimension.filePath, {
|
|
135
|
+
domainId,
|
|
136
|
+
owner: dimension.owner,
|
|
137
|
+
status: dimension.status,
|
|
138
|
+
aliases: unique([dimension.name, dimension.label ?? '', dimension.description ?? '']),
|
|
139
|
+
sourceSystem: 'semantic',
|
|
140
|
+
payload: { description: dimension.description, type: dimension.type, table: dimension.table },
|
|
141
|
+
}));
|
|
142
|
+
if (domainId)
|
|
143
|
+
addEdge('contains', `domain::${domainId}`, id);
|
|
144
|
+
}
|
|
145
|
+
for (const source of Object.values(manifest.sources ?? {})) {
|
|
146
|
+
const nativeId = source.dbtModel?.uniqueId;
|
|
147
|
+
const kind = nativeId?.startsWith('source.') ? 'dbt_source' : nativeId ? 'dbt_model' : 'source_table';
|
|
148
|
+
const id = nativeId ? `dbt::${nativeId}` : `source::${source.name.toLowerCase()}`;
|
|
149
|
+
addObject(object(kind, id, source.name, undefined, {
|
|
150
|
+
sourceSystem: source.origin === 'dbt' ? 'dbt' : 'dql',
|
|
151
|
+
nativeId,
|
|
152
|
+
aliases: [source.name],
|
|
153
|
+
payload: { origin: source.origin, referencedBy: source.referencedBy },
|
|
154
|
+
}));
|
|
155
|
+
}
|
|
156
|
+
for (const node of Object.values(manifest.dbtProvenance?.nodes ?? {})) {
|
|
157
|
+
const id = `dbt::${node.uniqueId}`;
|
|
158
|
+
addObject(object(node.resourceType === 'source' ? 'dbt_source' : 'dbt_model', id, node.name, node.sourcePath, {
|
|
159
|
+
sourceSystem: 'dbt',
|
|
160
|
+
nativeId: node.uniqueId,
|
|
161
|
+
sourceFingerprint: node.identityFingerprint,
|
|
162
|
+
aliases: unique([node.name, node.relation ?? '', node.uniqueId]),
|
|
163
|
+
payload: { packageName: node.packageName, relation: node.relation, available: node.available },
|
|
164
|
+
}));
|
|
165
|
+
}
|
|
166
|
+
for (const metric of Object.values(manifest.dbtProvenance?.metricFlow ?? {})) {
|
|
167
|
+
const id = `semantic::${metric.uniqueId}`;
|
|
168
|
+
addObject(object('metric', id, metric.name, metric.sourcePath, {
|
|
169
|
+
sourceSystem: 'semantic',
|
|
170
|
+
nativeId: metric.uniqueId,
|
|
171
|
+
sourceFingerprint: metric.fingerprint,
|
|
172
|
+
aliases: unique([metric.name, metric.uniqueId]),
|
|
173
|
+
payload: { semanticModel: metric.semanticModel },
|
|
174
|
+
}));
|
|
175
|
+
}
|
|
176
|
+
const entityIds = new Map();
|
|
177
|
+
for (const entity of Object.values(manifest.modeling?.entities ?? {})) {
|
|
178
|
+
const id = entity.qualifiedId;
|
|
179
|
+
addLookup(entityIds, entity.localId, id, entity.domain);
|
|
180
|
+
addLookup(entityIds, entity.id, id, entity.domain);
|
|
181
|
+
addObject(object('entity', id, entity.localId, entity.sourcePath, {
|
|
182
|
+
domainId: entity.domain,
|
|
183
|
+
owner: entity.owner,
|
|
184
|
+
status: entity.status,
|
|
185
|
+
modelAreaIds: entity.areaId ? [entity.areaId] : [],
|
|
186
|
+
aliases: unique([entity.localId, entity.businessName ?? '', entity.dbtUniqueId]),
|
|
187
|
+
sourceFingerprint: entity.identityFingerprint,
|
|
188
|
+
payload: { businessName: entity.businessName, businessContext: entity.businessContext, grain: entity.grain, keys: entity.keys, analyticalRole: entity.analyticalRole, dbtUniqueId: entity.dbtUniqueId },
|
|
189
|
+
}));
|
|
190
|
+
addEdge('contains', `domain::${entity.domain}`, id);
|
|
191
|
+
if (entity.areaId)
|
|
192
|
+
addEdge('contains', entity.areaId, id);
|
|
193
|
+
addEdge('binds_to', id, `dbt::${entity.dbtUniqueId}`);
|
|
194
|
+
}
|
|
195
|
+
const relationshipIds = new Map();
|
|
196
|
+
for (const relationship of Object.values(manifest.modeling?.relationships ?? {})) {
|
|
197
|
+
const id = relationship.qualifiedId;
|
|
198
|
+
addLookup(relationshipIds, relationship.localId, id, relationship.ownerDomain);
|
|
199
|
+
addLookup(relationshipIds, relationship.id, id, relationship.ownerDomain);
|
|
200
|
+
const fromId = resolveModelEntity(manifest, relationship.from);
|
|
201
|
+
const toId = resolveModelEntity(manifest, relationship.to);
|
|
202
|
+
const fromDomain = modelEntity(manifest, relationship.from)?.domain;
|
|
203
|
+
const toDomain = modelEntity(manifest, relationship.to)?.domain;
|
|
204
|
+
const state = relationship.staleCertification
|
|
205
|
+
? 'stale'
|
|
206
|
+
: relationship.automaticJoinAllowed
|
|
207
|
+
? 'authorized'
|
|
208
|
+
: 'blocked';
|
|
209
|
+
addObject(object('relationship', id, relationship.localId, relationship.sourcePath, {
|
|
210
|
+
domainId: relationship.ownerDomain,
|
|
211
|
+
owner: relationship.owner,
|
|
212
|
+
status: relationship.staleCertification ? 'stale' : relationship.status,
|
|
213
|
+
modelAreaIds: relationship.areaId ? [relationship.areaId] : [],
|
|
214
|
+
sourceFingerprint: relationship.fingerprint,
|
|
215
|
+
aliases: unique([relationship.localId, relationship.verb ?? '', relationship.description ?? '']),
|
|
216
|
+
payload: { ...relationship },
|
|
217
|
+
}));
|
|
218
|
+
addEdge('proves_join', fromId, id, { state, domainPair: pair(fromDomain, toDomain), evidenceRefs: relationship.validation ? [relationship.certificationFingerprint ?? relationship.fingerprint] : [], reasonCodes: relationshipReasonCodes(relationship) });
|
|
219
|
+
addEdge('proves_join', id, toId, { state, domainPair: pair(fromDomain, toDomain), evidenceRefs: relationship.validation ? [relationship.certificationFingerprint ?? relationship.fingerprint] : [], reasonCodes: relationshipReasonCodes(relationship) });
|
|
220
|
+
if (relationship.areaId)
|
|
221
|
+
addEdge('contains', relationship.areaId, id);
|
|
222
|
+
}
|
|
223
|
+
const contractIds = new Map();
|
|
224
|
+
for (const contract of Object.values(manifest.modeling?.contracts ?? {})) {
|
|
225
|
+
const id = contract.qualifiedId;
|
|
226
|
+
addLookup(contractIds, contract.localId, id, contract.domain);
|
|
227
|
+
addLookup(contractIds, contract.id, id, contract.domain);
|
|
228
|
+
addObject(object('contract', id, contract.localId, contract.sourcePath, {
|
|
229
|
+
domainId: contract.domain,
|
|
230
|
+
owner: contract.owner,
|
|
231
|
+
status: contract.status,
|
|
232
|
+
payload: { ...contract },
|
|
233
|
+
}));
|
|
234
|
+
addEdge('contains', `domain::${contract.domain}`, id);
|
|
235
|
+
for (const entity of contract.entities)
|
|
236
|
+
addEdge('governed_by', resolveModelEntity(manifest, entity), id);
|
|
237
|
+
for (const block of contract.blocks)
|
|
238
|
+
for (const blockId of lookup(blockIds, block, contract.domain))
|
|
239
|
+
addEdge('governed_by', blockId, id);
|
|
240
|
+
}
|
|
241
|
+
const exportObjects = new Map();
|
|
242
|
+
for (const [key, exported] of Object.entries(manifest.modeling?.interfaces?.exports ?? {})) {
|
|
243
|
+
const id = exported.qualifiedId;
|
|
244
|
+
addObject(object('domain_export', id, exported.localId, exported.sourcePath, { domainId: exported.domain, owner: exported.owner, status: exported.status, sourceFingerprint: exported.fingerprint, payload: { ...exported } }));
|
|
245
|
+
addEdge('exports', `domain::${exported.domain}`, id);
|
|
246
|
+
if (exported.entity)
|
|
247
|
+
addEdge('exports', resolveModelEntity(manifest, exported.entity), id);
|
|
248
|
+
for (const alias of exportAliases(key, exported))
|
|
249
|
+
exportObjects.set(alias, id);
|
|
250
|
+
}
|
|
251
|
+
const importObjects = new Map();
|
|
252
|
+
for (const [key, imported] of Object.entries(manifest.modeling?.interfaces?.imports ?? {})) {
|
|
253
|
+
const id = imported.qualifiedId;
|
|
254
|
+
addObject(object('domain_import', id, imported.localId, imported.sourcePath, { domainId: imported.domain, owner: imported.owner, status: imported.status, payload: { ...imported } }));
|
|
255
|
+
addEdge('imports', `domain::${imported.domain}`, id);
|
|
256
|
+
const exported = exportObjects.get(imported.exportRef);
|
|
257
|
+
if (exported)
|
|
258
|
+
addEdge('imports', exported, id);
|
|
259
|
+
for (const alias of unique([key, imported.id, imported.localId, imported.qualifiedId]))
|
|
260
|
+
importObjects.set(alias, id);
|
|
261
|
+
}
|
|
262
|
+
for (const declaration of Object.values(manifest.modeling?.conformance ?? {})) {
|
|
263
|
+
addObject(object('conformance', declaration.qualifiedId, declaration.localId, declaration.sourcePath, { domainId: declaration.domain, payload: { ...declaration } }));
|
|
264
|
+
addEdge('contains', `domain::${declaration.domain}`, declaration.qualifiedId);
|
|
265
|
+
for (const entity of declaration.entities)
|
|
266
|
+
addEdge('conforms_to', resolveModelEntity(manifest, entity), declaration.qualifiedId);
|
|
267
|
+
}
|
|
268
|
+
for (const rule of Object.values(manifest.modeling?.rules ?? {})) {
|
|
269
|
+
addObject(object('policy', rule.qualifiedId, rule.localId, rule.sourcePath, { domainId: rule.domain, payload: { ...rule } }));
|
|
270
|
+
addEdge('contains', `domain::${rule.domain}`, rule.qualifiedId);
|
|
271
|
+
}
|
|
272
|
+
for (const skill of skills) {
|
|
273
|
+
addObject(object('skill', skill.qualifiedId, skill.localId, skill.sourcePath, {
|
|
274
|
+
domainId: skill.domain,
|
|
275
|
+
owner: skill.owner,
|
|
276
|
+
status: skill.status,
|
|
277
|
+
modelAreaIds: skill.modelAreaRefs,
|
|
278
|
+
sourceFingerprint: skill.contentHash,
|
|
279
|
+
aliases: unique([skill.localId, skill.description ?? '', ...skill.triggers, ...Object.keys(skill.vocabulary)]),
|
|
280
|
+
payload: { ...skill },
|
|
281
|
+
}));
|
|
282
|
+
for (const domain of skill.domains)
|
|
283
|
+
addEdge('contains', `domain::${domain}`, skill.qualifiedId);
|
|
284
|
+
for (const area of skill.modelAreaRefs)
|
|
285
|
+
addEdge('contains', area, skill.qualifiedId);
|
|
286
|
+
}
|
|
287
|
+
for (const [path, notebook] of Object.entries(manifest.notebooks ?? {})) {
|
|
288
|
+
const id = `global::notebook::${path}`;
|
|
289
|
+
addObject(object('notebook', id, path, notebook.filePath, { domainId: canonicalDomain(notebook.ownerDomain), aliases: [notebook.title, path], payload: { title: notebook.title, ...productPayload(notebook) } }));
|
|
290
|
+
connectProduct(id, notebook, canonicalDomain, addEdge);
|
|
291
|
+
}
|
|
292
|
+
for (const app of Object.values(manifest.apps ?? {})) {
|
|
293
|
+
const id = `global::app::${app.id}`;
|
|
294
|
+
addObject(object('app', id, app.id, app.filePath, { domainId: canonicalDomain(app.ownerDomain ?? app.domain), owner: app.owners[0], status: app.lifecycle, aliases: [app.id, app.name], payload: { name: app.name, ...productPayload(app) } }));
|
|
295
|
+
connectProduct(id, app, canonicalDomain, addEdge);
|
|
296
|
+
}
|
|
297
|
+
for (const dashboard of Object.values(manifest.dashboards ?? {})) {
|
|
298
|
+
const id = `global::dashboard::${dashboard.qualifiedId}`;
|
|
299
|
+
const domainId = canonicalDomain(dashboard.domain);
|
|
300
|
+
addObject(object('dashboard', id, dashboard.qualifiedId, dashboard.filePath, { domainId, aliases: [dashboard.id, dashboard.qualifiedId, dashboard.title], payload: { title: dashboard.title, appId: dashboard.appId } }));
|
|
301
|
+
if (domainId)
|
|
302
|
+
addEdge('consumed_by', `domain::${domainId}`, id);
|
|
303
|
+
addEdge('contains', `global::app::${dashboard.appId}`, id);
|
|
304
|
+
}
|
|
305
|
+
// Resolve the free references only after every potential target exists.
|
|
306
|
+
for (const block of knowledgeBlocks) {
|
|
307
|
+
const domainId = canonicalDomain(block.domain);
|
|
308
|
+
const id = qualified('block', block.name, domainId);
|
|
309
|
+
for (const term of block.termRefs ?? [])
|
|
310
|
+
for (const target of lookup(termIds, term, domainId))
|
|
311
|
+
addEdge('defines', target, id);
|
|
312
|
+
for (const metric of unique([...(block.metricsRef ?? []), ...(block.metricRef ? [block.metricRef] : []), ...(block.metricRefs ?? [])]))
|
|
313
|
+
for (const target of lookup(metricIds, metric, domainId))
|
|
314
|
+
addEdge('implements', id, target);
|
|
315
|
+
for (const entity of block.entities ?? [])
|
|
316
|
+
for (const target of lookup(entityIds, entity, domainId))
|
|
317
|
+
addEdge('implements', id, target);
|
|
318
|
+
for (const ref of block.refDependencies ?? [])
|
|
319
|
+
for (const target of lookup(blockIds, ref, domainId))
|
|
320
|
+
addEdge('depends_on', target, id);
|
|
321
|
+
for (const table of block.tableDependencies ?? []) {
|
|
322
|
+
const target = sourceObjectId(manifest, table);
|
|
323
|
+
if (target)
|
|
324
|
+
addEdge('reads_from', target, id);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
for (const view of Object.values(manifest.businessViews ?? {})) {
|
|
328
|
+
const domainId = canonicalDomain(view.domain);
|
|
329
|
+
const id = qualified('business_view', view.name, domainId);
|
|
330
|
+
for (const term of view.termRefs ?? [])
|
|
331
|
+
for (const target of lookup(termIds, term, domainId))
|
|
332
|
+
addEdge('defines', target, id);
|
|
333
|
+
for (const block of view.blockRefs ?? [])
|
|
334
|
+
for (const target of lookup(blockIds, block, domainId))
|
|
335
|
+
addEdge('implements', target, id);
|
|
336
|
+
for (const child of view.businessViewRefs ?? [])
|
|
337
|
+
for (const target of lookup(viewIds, child, domainId))
|
|
338
|
+
addEdge('implements', target, id);
|
|
339
|
+
}
|
|
340
|
+
for (const skill of skills) {
|
|
341
|
+
for (const ref of skill.preferredMetrics)
|
|
342
|
+
for (const target of lookup(metricIds, ref, skill.domain))
|
|
343
|
+
addEdge('guided_by', target, skill.qualifiedId);
|
|
344
|
+
for (const ref of skill.preferredBlocks)
|
|
345
|
+
for (const target of lookup(blockIds, ref, skill.domain))
|
|
346
|
+
addEdge('guided_by', target, skill.qualifiedId);
|
|
347
|
+
for (const ref of skill.preferredDimensions)
|
|
348
|
+
for (const target of lookup(dimensionIds, ref, skill.domain))
|
|
349
|
+
addEdge('guided_by', target, skill.qualifiedId);
|
|
350
|
+
}
|
|
351
|
+
// Preserve dbt transformation flow as observed context, never join proof.
|
|
352
|
+
for (const edge of manifest.lineage.edges) {
|
|
353
|
+
if (edge.type !== 'depends_on')
|
|
354
|
+
continue;
|
|
355
|
+
const from = lineageObjectId(edge.source, objects);
|
|
356
|
+
const to = lineageObjectId(edge.target, objects);
|
|
357
|
+
if (from && to)
|
|
358
|
+
addEdge('transforms', from, to, { state: 'observed', domainPair: pair(canonicalDomain(edge.sourceDomain), canonicalDomain(edge.targetDomain)) });
|
|
359
|
+
}
|
|
360
|
+
const crossDomainRoutes = buildCrossDomainRoutes(manifest, exportObjects, importObjects, contractIds, canonicalDomain);
|
|
361
|
+
const routePairs = new Set(crossDomainRoutes.map((route) => `${route.providerDomainId}:${route.consumerDomainId}`));
|
|
362
|
+
for (const flow of manifest.lineage.crossDomainFlows) {
|
|
363
|
+
const providerDomainId = canonicalDomain(flow.from) ?? flow.from;
|
|
364
|
+
const consumerDomainId = canonicalDomain(flow.to) ?? flow.to;
|
|
365
|
+
if (routePairs.has(`${providerDomainId}:${consumerDomainId}`))
|
|
366
|
+
continue;
|
|
367
|
+
const key = `observed:${providerDomainId}:${consumerDomainId}`;
|
|
368
|
+
crossDomainRoutes.push({ id: `route::${digest(key).slice(0, 20)}`, providerDomainId, consumerDomainId, purpose: '', relationshipId: `observed::${providerDomainId}::${consumerDomainId}`, state: 'observed', reasonCodes: ['OBSERVED_DEPENDENCY_ONLY'], path: [`domain::${providerDomainId}`, `domain::${consumerDomainId}`], fingerprint: digest(key) });
|
|
369
|
+
}
|
|
370
|
+
const domainCapsules = buildDomainCapsules(manifest, skills, crossDomainRoutes, canonicalDomain, qualified, knowledgeBlocks);
|
|
371
|
+
edges.sort(edgeSort);
|
|
372
|
+
crossDomainRoutes.sort((a, b) => a.id.localeCompare(b.id));
|
|
373
|
+
const graphContent = { schemaVersion: 1, objects: sortRecord(objects), edges, domainCapsules, crossDomainRoutes };
|
|
374
|
+
return { ...graphContent, schemaVersion: 1, storageMode: 'inline', sourceFingerprint: digest(stable(graphContent)), diagnostics };
|
|
375
|
+
}
|
|
376
|
+
/** Compact control-plane projection; detailed rows live in metadata.sqlite. */
|
|
377
|
+
export function compactManifestKnowledgeGraph(graph) {
|
|
378
|
+
const objects = Object.values(graph.objects ?? {});
|
|
379
|
+
const edges = graph.edges ?? [];
|
|
380
|
+
const domainIds = [...new Set(objects.flatMap((item) => item.domainId ? [item.domainId] : []))].sort();
|
|
381
|
+
const shards = ['global', ...domainIds].map((domainId) => {
|
|
382
|
+
const shardObjects = objects.filter((item) => domainId === 'global' ? !item.domainId : item.domainId === domainId);
|
|
383
|
+
const ids = new Set(shardObjects.map((item) => item.id));
|
|
384
|
+
const shardEdges = edges.filter((edge) => ids.has(edge.from) || ids.has(edge.to));
|
|
385
|
+
const content = {
|
|
386
|
+
domainId: domainId === 'global' ? undefined : domainId,
|
|
387
|
+
objectIds: shardObjects.map((item) => item.id).sort(),
|
|
388
|
+
edgeIds: shardEdges.map((item) => item.id).sort(),
|
|
389
|
+
};
|
|
390
|
+
return {
|
|
391
|
+
id: domainId === 'global' ? 'global' : `domain:${domainId}`,
|
|
392
|
+
domainId: content.domainId,
|
|
393
|
+
fingerprint: digest(stable(content)),
|
|
394
|
+
objectCount: shardObjects.length,
|
|
395
|
+
edgeCount: shardEdges.length,
|
|
396
|
+
};
|
|
397
|
+
});
|
|
398
|
+
return {
|
|
399
|
+
schemaVersion: 2,
|
|
400
|
+
storageMode: 'indexed',
|
|
401
|
+
sourceFingerprint: graph.sourceFingerprint,
|
|
402
|
+
counts: {
|
|
403
|
+
objects: objects.length,
|
|
404
|
+
edges: edges.length,
|
|
405
|
+
skills: objects.filter((item) => item.kind === 'skill').length,
|
|
406
|
+
routes: graph.crossDomainRoutes.length,
|
|
407
|
+
},
|
|
408
|
+
shards,
|
|
409
|
+
index: { schemaVersion: KNOWLEDGE_INDEX_SCHEMA_VERSION, fingerprint: graph.sourceFingerprint },
|
|
410
|
+
objectRefs: objects.map(({ id, kind, localId, domainId, modelAreaIds, status, owner, source }) => ({
|
|
411
|
+
id, kind, localId, domainId, modelAreaIds, status, owner, source,
|
|
412
|
+
})).sort((a, b) => a.id.localeCompare(b.id)),
|
|
413
|
+
domainCapsules: graph.domainCapsules,
|
|
414
|
+
crossDomainRoutes: graph.crossDomainRoutes,
|
|
415
|
+
diagnostics: graph.diagnostics,
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
function object(kind, id, localId, sourcePath, options = {}) {
|
|
419
|
+
const source = { system: options.sourceSystem ?? 'dql', path: sourcePath, nativeId: options.nativeId, fingerprint: options.sourceFingerprint ?? digest(stable({ kind, id, sourcePath, payload: options.payload })) };
|
|
420
|
+
return { id, kind, localId, domainId: options.domainId, modelAreaIds: options.modelAreaIds?.length ? unique(options.modelAreaIds) : undefined, aliases: options.aliases?.length ? unique(options.aliases.filter(Boolean)) : undefined, status: options.status, owner: options.owner, source, payload: options.payload };
|
|
421
|
+
}
|
|
422
|
+
function buildDomainAliases(manifest) {
|
|
423
|
+
const aliases = new Map();
|
|
424
|
+
for (const [key, domain] of Object.entries(manifest.domains ?? {})) {
|
|
425
|
+
const id = domain.id ?? key;
|
|
426
|
+
for (const alias of unique([key, id, domain.name]))
|
|
427
|
+
aliases.set(alias.toLowerCase(), id);
|
|
428
|
+
}
|
|
429
|
+
for (const pkg of Object.values(manifest.modeling?.packages ?? {}))
|
|
430
|
+
aliases.set(pkg.id.toLowerCase(), pkg.id);
|
|
431
|
+
return aliases;
|
|
432
|
+
}
|
|
433
|
+
function buildCrossDomainRoutes(manifest, exportObjects, importObjects, contractIds, canonicalDomain) {
|
|
434
|
+
const routes = [];
|
|
435
|
+
const exports = Object.entries(manifest.modeling?.interfaces?.exports ?? {});
|
|
436
|
+
const imports = Object.entries(manifest.modeling?.interfaces?.imports ?? {});
|
|
437
|
+
for (const relationship of Object.values(manifest.modeling?.relationships ?? {})) {
|
|
438
|
+
if (!relationship.crossDomain)
|
|
439
|
+
continue;
|
|
440
|
+
const fromDomain = modelEntity(manifest, relationship.from)?.domain;
|
|
441
|
+
const toDomain = modelEntity(manifest, relationship.to)?.domain;
|
|
442
|
+
const matchingImports = imports.filter(([key, imported]) => {
|
|
443
|
+
// Relationship declarations normally cite the exported interface
|
|
444
|
+
// (`commerce.customer_identity@1`), while the consumer import has its own
|
|
445
|
+
// local identity. Both are valid references to the same governed hop.
|
|
446
|
+
const aliases = unique([key, imported.id, imported.localId, imported.qualifiedId, imported.exportRef]);
|
|
447
|
+
return relationship.importRefs?.length ? relationship.importRefs.some((ref) => aliases.includes(ref)) : imported.domain === fromDomain || imported.domain === toDomain;
|
|
448
|
+
});
|
|
449
|
+
if (matchingImports.length === 0) {
|
|
450
|
+
routes.push(routeRecord(relationship.qualifiedId, fromDomain ?? '', toDomain ?? '', '', relationship, undefined, undefined, undefined, ['MISSING_IMPORT']));
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
for (const [importKey, imported] of matchingImports) {
|
|
454
|
+
const exportEntry = exports.find(([key, exported]) => exportAliases(key, exported).includes(imported.exportRef));
|
|
455
|
+
const exported = exportEntry?.[1];
|
|
456
|
+
const providerDomainId = canonicalDomain(exported?.domain ?? (imported.domain === fromDomain ? toDomain : fromDomain)) ?? '';
|
|
457
|
+
const consumerDomainId = canonicalDomain(imported.domain) ?? imported.domain;
|
|
458
|
+
const contractRef = exported?.contract;
|
|
459
|
+
const contract = contractRef ? Object.values(manifest.modeling?.contracts ?? {}).find((item) => unique([item.id, item.localId, item.qualifiedId]).includes(contractRef)) : undefined;
|
|
460
|
+
const reasons = relationshipReasonCodes(relationship);
|
|
461
|
+
if (!exported)
|
|
462
|
+
reasons.push('MISSING_EXPORT');
|
|
463
|
+
if (imported.status !== 'certified')
|
|
464
|
+
reasons.push('IMPORT_NOT_CERTIFIED');
|
|
465
|
+
if (exported && exported.status !== 'certified')
|
|
466
|
+
reasons.push('EXPORT_NOT_CERTIFIED');
|
|
467
|
+
if (!contract)
|
|
468
|
+
reasons.push('MISSING_CONTRACT');
|
|
469
|
+
else if (contract.status !== 'certified')
|
|
470
|
+
reasons.push('CONTRACT_NOT_CERTIFIED');
|
|
471
|
+
if (exported && exported.purposes.length > 0 && !exported.purposes.includes(imported.purpose))
|
|
472
|
+
reasons.push('PURPOSE_NOT_ALLOWED');
|
|
473
|
+
if (exported && exported.consumerDomains.length > 0 && !exported.consumerDomains.includes(consumerDomainId))
|
|
474
|
+
reasons.push('CONSUMER_NOT_ALLOWED');
|
|
475
|
+
routes.push(routeRecord(relationship.qualifiedId, providerDomainId, consumerDomainId, imported.purpose, relationship, exported ? exportObjects.get(exportEntry?.[0] ?? '') ?? exported.qualifiedId : undefined, importObjects.get(importKey) ?? imported.qualifiedId, contract ? lookup(contractIds, contract.localId, contract.domain)[0] ?? contract.qualifiedId : undefined, unique(reasons)));
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
return routes;
|
|
479
|
+
}
|
|
480
|
+
function routeRecord(relationshipId, providerDomainId, consumerDomainId, purpose, relationship, exportId, importId, contractId, reasonCodes) {
|
|
481
|
+
const state = relationship.staleCertification
|
|
482
|
+
? 'stale'
|
|
483
|
+
: reasonCodes.length === 0 && relationship.automaticJoinAllowed
|
|
484
|
+
? 'authorized'
|
|
485
|
+
: 'blocked';
|
|
486
|
+
const path = unique([`domain::${providerDomainId}`, exportId ?? '', contractId ?? '', importId ?? '', relationshipId, `domain::${consumerDomainId}`].filter(Boolean));
|
|
487
|
+
const content = { providerDomainId, consumerDomainId, purpose, relationshipId, exportId, importId, contractId, state, reasonCodes: unique(reasonCodes).sort(), path };
|
|
488
|
+
const fingerprint = digest(stable(content));
|
|
489
|
+
return { id: `route::${fingerprint.slice(0, 20)}`, ...content, fingerprint };
|
|
490
|
+
}
|
|
491
|
+
function buildDomainCapsules(manifest, skills, routes, canonicalDomain, qualified, knowledgeBlocks) {
|
|
492
|
+
const capsules = {};
|
|
493
|
+
for (const [key, domain] of Object.entries(manifest.domains ?? {})) {
|
|
494
|
+
const domainId = domain.id ?? key;
|
|
495
|
+
const build = (areaId, areaName, intentExamples = [], description) => {
|
|
496
|
+
const terms = Object.values(manifest.terms ?? {}).filter((item) => canonicalDomain(item.domain) === domainId);
|
|
497
|
+
const domainSkills = skills.filter((item) => item.domains.includes(domainId) && (!areaId || item.modelAreaRefs.length === 0 || item.modelAreaRefs.includes(areaId)));
|
|
498
|
+
const entities = Object.values(manifest.modeling?.entities ?? {}).filter((item) => item.domain === domainId && (!areaId || item.areaId === areaId));
|
|
499
|
+
const metrics = Object.values(manifest.metrics ?? {}).filter((item) => canonicalDomain(item.domain) === domainId);
|
|
500
|
+
const blocks = knowledgeBlocks.filter((item) => canonicalDomain(item.domain) === domainId && item.status === 'certified');
|
|
501
|
+
const relevantRoutes = routes.filter((route) => route.providerDomainId === domainId || route.consumerDomainId === domainId);
|
|
502
|
+
const content = {
|
|
503
|
+
domainId,
|
|
504
|
+
modelAreaId: areaId,
|
|
505
|
+
name: areaName ?? domain.name,
|
|
506
|
+
description: description ?? domain.boundedContext ?? domain.description,
|
|
507
|
+
intentExamples,
|
|
508
|
+
exclusions: unique(domain.outOfScope ?? []),
|
|
509
|
+
termRefs: terms.map((item) => qualified('term', item.name, domainId)).sort(),
|
|
510
|
+
skillRefs: domainSkills.map((item) => item.qualifiedId).sort(),
|
|
511
|
+
entityRefs: entities.map((item) => item.qualifiedId).sort(),
|
|
512
|
+
metricRefs: metrics.map((item) => qualified('metric', item.name, domainId)).sort(),
|
|
513
|
+
blockRefs: blocks.map((item) => qualified('block', item.name, domainId)).sort(),
|
|
514
|
+
routeRefs: relevantRoutes.map((item) => item.id).sort(),
|
|
515
|
+
caveats: unique(terms.flatMap((item) => item.caveats ?? [])),
|
|
516
|
+
requiredFilters: unique(domainSkills.flatMap((item) => item.requiredFilters)),
|
|
517
|
+
};
|
|
518
|
+
const fingerprint = digest(stable(content));
|
|
519
|
+
const id = areaId ? `${areaId}::capsule` : `${domainId}::capsule`;
|
|
520
|
+
capsules[id] = { id, ...content, fingerprint };
|
|
521
|
+
};
|
|
522
|
+
build(undefined, undefined, [], domain.boundedContext ?? domain.description);
|
|
523
|
+
for (const area of Object.values(manifest.modeling?.areas ?? {}).filter((item) => item.domain === domainId))
|
|
524
|
+
build(area.qualifiedId, area.name, area.intentExamples, area.description);
|
|
525
|
+
}
|
|
526
|
+
return sortRecord(capsules);
|
|
527
|
+
}
|
|
528
|
+
function connectProduct(id, product, canonicalDomain, addEdge) {
|
|
529
|
+
for (const domain of unique([product.ownerDomain ?? '', ...(product.usesDomains ?? [])]).filter(Boolean)) {
|
|
530
|
+
const domainId = canonicalDomain(domain);
|
|
531
|
+
if (domainId)
|
|
532
|
+
addEdge('consumed_by', `domain::${domainId}`, id);
|
|
533
|
+
}
|
|
534
|
+
for (const skill of product.skillRefs ?? [])
|
|
535
|
+
addEdge('guided_by', id, skill);
|
|
536
|
+
}
|
|
537
|
+
function productPayload(product) {
|
|
538
|
+
return { ownerDomain: product.ownerDomain, usesDomains: product.usesDomains ?? [], purpose: product.purpose, requiredExports: product.requiredExports ?? [], skillRefs: product.skillRefs ?? [], classification: product.classification };
|
|
539
|
+
}
|
|
540
|
+
function relationshipReasonCodes(relationship) {
|
|
541
|
+
const reasons = [];
|
|
542
|
+
if (relationship.status !== 'certified')
|
|
543
|
+
reasons.push('RELATIONSHIP_NOT_CERTIFIED');
|
|
544
|
+
if (relationship.staleCertification)
|
|
545
|
+
reasons.push('STALE_PROOF');
|
|
546
|
+
if (!relationship.validation || relationship.validation.status !== 'passed')
|
|
547
|
+
reasons.push('VALIDATION_NOT_PASSED');
|
|
548
|
+
if (relationship.fanout === 'unsafe' || relationship.fanout === 'unknown')
|
|
549
|
+
reasons.push('UNSAFE_FANOUT');
|
|
550
|
+
if (relationship.keys.length === 0)
|
|
551
|
+
reasons.push('MISSING_KEYS');
|
|
552
|
+
return reasons;
|
|
553
|
+
}
|
|
554
|
+
function exportAliases(key, exported) {
|
|
555
|
+
return unique([key, exported.id, exported.localId, exported.qualifiedId, `${exported.domain}.${exported.localId}@${exported.version}`]);
|
|
556
|
+
}
|
|
557
|
+
function resolveModelEntity(manifest, ref) {
|
|
558
|
+
return modelEntity(manifest, ref)?.qualifiedId ?? ref;
|
|
559
|
+
}
|
|
560
|
+
function modelEntity(manifest, ref) {
|
|
561
|
+
return manifest.modeling?.entities[ref] ?? Object.values(manifest.modeling?.entities ?? {}).find((entity) => entity.id === ref || entity.localId === ref || entity.qualifiedId === ref);
|
|
562
|
+
}
|
|
563
|
+
function sourceObjectId(manifest, table) {
|
|
564
|
+
const normalized = table.toLowerCase();
|
|
565
|
+
const dbt = Object.values(manifest.dbtProvenance?.nodes ?? {}).find((node) => unique([node.name, node.relation ?? '', node.uniqueId]).some((value) => value.toLowerCase() === normalized));
|
|
566
|
+
if (dbt)
|
|
567
|
+
return `dbt::${dbt.uniqueId}`;
|
|
568
|
+
return Object.values(manifest.sources ?? {}).some((source) => source.name.toLowerCase() === normalized) ? `source::${normalized}` : undefined;
|
|
569
|
+
}
|
|
570
|
+
function lineageObjectId(id, objects) {
|
|
571
|
+
if (objects[id])
|
|
572
|
+
return id;
|
|
573
|
+
const [kind, ...rest] = id.split(':');
|
|
574
|
+
const value = rest.join(':');
|
|
575
|
+
if (kind === 'dbt_model' || kind === 'dbt_source') {
|
|
576
|
+
const found = Object.values(objects).find((object) => (object.kind === 'dbt_model' || object.kind === 'dbt_source') && (object.localId === value || object.source.nativeId === value));
|
|
577
|
+
return found?.id;
|
|
578
|
+
}
|
|
579
|
+
const found = Object.values(objects).filter((object) => object.kind === kind && (object.localId === value || object.aliases?.includes(value)));
|
|
580
|
+
return found.length === 1 ? found[0].id : undefined;
|
|
581
|
+
}
|
|
582
|
+
function addLookup(map, local, id, domain) {
|
|
583
|
+
for (const key of unique([local, id, domain ? `${domain}:${local}` : '']).filter(Boolean))
|
|
584
|
+
map.set(key, unique([...(map.get(key) ?? []), id]));
|
|
585
|
+
}
|
|
586
|
+
function lookup(map, ref, domain) {
|
|
587
|
+
// Local names may exist in several domain packages. Prefer the active
|
|
588
|
+
// domain's qualified lookup before falling back to the global alias.
|
|
589
|
+
return (domain ? map.get(`${domain}:${ref}`) : undefined) ?? map.get(ref) ?? [];
|
|
590
|
+
}
|
|
591
|
+
function pair(provider, consumer) {
|
|
592
|
+
return provider && consumer && provider !== consumer ? { provider, consumer } : undefined;
|
|
593
|
+
}
|
|
594
|
+
function edgeSort(a, b) {
|
|
595
|
+
return `${a.kind}:${a.from}:${a.to}:${a.id}`.localeCompare(`${b.kind}:${b.from}:${b.to}:${b.id}`);
|
|
596
|
+
}
|
|
597
|
+
function sortRecord(record) {
|
|
598
|
+
return Object.fromEntries(Object.entries(record).sort(([a], [b]) => a.localeCompare(b)));
|
|
599
|
+
}
|
|
600
|
+
function unique(values) {
|
|
601
|
+
return [...new Set(values.filter(Boolean))];
|
|
602
|
+
}
|
|
603
|
+
function stable(value) {
|
|
604
|
+
return JSON.stringify(sortValue(value));
|
|
605
|
+
}
|
|
606
|
+
function sortValue(value) {
|
|
607
|
+
if (Array.isArray(value))
|
|
608
|
+
return value.map(sortValue);
|
|
609
|
+
if (value && typeof value === 'object')
|
|
610
|
+
return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, sortValue(item)]));
|
|
611
|
+
return value;
|
|
612
|
+
}
|
|
613
|
+
function digest(value) {
|
|
614
|
+
return createHash('sha256').update(value).digest('hex');
|
|
615
|
+
}
|
|
616
|
+
//# sourceMappingURL=knowledge-graph.js.map
|