@duckcodeailabs/dql-core 0.8.9 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lineage/builder.d.ts +6 -17
- package/dist/lineage/builder.d.ts.map +1 -1
- package/dist/lineage/builder.js +68 -83
- package/dist/lineage/builder.js.map +1 -1
- package/dist/lineage/index.d.ts +1 -1
- package/dist/lineage/index.d.ts.map +1 -1
- package/dist/lineage/index.js +1 -1
- package/dist/lineage/index.js.map +1 -1
- package/dist/lineage/lineage-graph.d.ts +2 -2
- package/dist/lineage/lineage-graph.d.ts.map +1 -1
- package/dist/lineage/query.d.ts +1 -42
- package/dist/lineage/query.d.ts.map +1 -1
- package/dist/lineage/query.js +95 -190
- package/dist/lineage/query.js.map +1 -1
- package/dist/manifest/builder.d.ts.map +1 -1
- package/dist/manifest/builder.js +72 -82
- package/dist/manifest/builder.js.map +1 -1
- package/dist/manifest/types.d.ts +6 -5
- package/dist/manifest/types.d.ts.map +1 -1
- package/dist/parser/parser.d.ts.map +1 -1
- package/dist/parser/parser.js +3 -1
- package/dist/parser/parser.js.map +1 -1
- package/package.json +1 -1
package/dist/lineage/query.js
CHANGED
|
@@ -1,211 +1,116 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Lineage Query Engine — focused, searchable lineage views.
|
|
3
|
-
*
|
|
4
|
-
* Replaces the "show everything" approach with targeted subgraph queries:
|
|
5
|
-
* - Focus on a specific node and get its upstream/downstream chain
|
|
6
|
-
* - Search nodes by name across all types
|
|
7
|
-
* - Filter by type, domain, depth
|
|
8
|
-
*/
|
|
9
|
-
// ---- Node ID Resolution ----
|
|
10
|
-
/** Type prefixes to try when resolving a name to a node ID */
|
|
11
1
|
const NODE_PREFIXES = [
|
|
12
|
-
'block
|
|
13
|
-
'
|
|
14
|
-
'
|
|
15
|
-
'
|
|
16
|
-
'
|
|
17
|
-
'
|
|
18
|
-
'dimension
|
|
19
|
-
'
|
|
20
|
-
'
|
|
2
|
+
'block',
|
|
3
|
+
'dashboard',
|
|
4
|
+
'dbt_model',
|
|
5
|
+
'dbt_source',
|
|
6
|
+
'source_table',
|
|
7
|
+
'metric',
|
|
8
|
+
'dimension',
|
|
9
|
+
'domain',
|
|
10
|
+
'chart',
|
|
21
11
|
];
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return
|
|
12
|
+
export function queryLineage(graph, query) {
|
|
13
|
+
const matches = query.search ? searchLineage(graph, query.search) : [];
|
|
14
|
+
const focalNode = query.focus ? resolveFocusNode(graph, query.focus) : undefined;
|
|
15
|
+
let resultGraph = focalNode
|
|
16
|
+
? buildFocusedSubgraph(graph, focalNode.id, query.upstreamDepth, query.downstreamDepth)
|
|
17
|
+
: graph;
|
|
18
|
+
if (query.types?.length || query.domain) {
|
|
19
|
+
const allowedTypes = query.types ? new Set(query.types) : null;
|
|
20
|
+
resultGraph = resultGraph.subgraph((node) => {
|
|
21
|
+
if (allowedTypes && !allowedTypes.has(node.type))
|
|
22
|
+
return false;
|
|
23
|
+
if (query.domain && node.domain !== query.domain)
|
|
24
|
+
return false;
|
|
25
|
+
return true;
|
|
26
|
+
});
|
|
36
27
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
28
|
+
if (!query.focus && query.search) {
|
|
29
|
+
const matchIds = new Set(matches.map((match) => match.node.id));
|
|
30
|
+
resultGraph = graph.subgraph((node) => {
|
|
31
|
+
if (query.domain && node.domain !== query.domain)
|
|
32
|
+
return false;
|
|
33
|
+
if (query.types?.length && !query.types.includes(node.type))
|
|
34
|
+
return false;
|
|
35
|
+
return matchIds.has(node.id);
|
|
36
|
+
});
|
|
42
37
|
}
|
|
43
|
-
return
|
|
38
|
+
return {
|
|
39
|
+
graph: resultGraph.toJSON(),
|
|
40
|
+
focalNode,
|
|
41
|
+
matches,
|
|
42
|
+
};
|
|
44
43
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
return 100; // exact match
|
|
51
|
-
if (name.startsWith(q))
|
|
52
|
-
return 80; // starts with
|
|
53
|
-
if (name.endsWith(q))
|
|
54
|
-
return 60; // ends with
|
|
55
|
-
if (name.includes(q))
|
|
56
|
-
return 40; // substring
|
|
57
|
-
return 0; // no match
|
|
44
|
+
function buildFocusedSubgraph(graph, focusId, upstreamDepth, downstreamDepth) {
|
|
45
|
+
const includedIds = new Set([focusId]);
|
|
46
|
+
walkDirection(graph, focusId, 'upstream', normalizeDepth(upstreamDepth), includedIds);
|
|
47
|
+
walkDirection(graph, focusId, 'downstream', normalizeDepth(downstreamDepth), includedIds);
|
|
48
|
+
return graph.subgraph((node) => includedIds.has(node.id));
|
|
58
49
|
}
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* BFS traversal with depth limiting.
|
|
62
|
-
* direction='upstream' follows incoming edges, 'downstream' follows outgoing.
|
|
63
|
-
*/
|
|
64
|
-
function depthLimitedBFS(graph, startId, direction, maxDepth) {
|
|
65
|
-
const visited = new Set();
|
|
50
|
+
function walkDirection(graph, startId, direction, maxDepth, includedIds) {
|
|
66
51
|
const queue = [{ id: startId, depth: 0 }];
|
|
52
|
+
const seen = new Set([startId]);
|
|
67
53
|
while (queue.length > 0) {
|
|
68
|
-
const
|
|
69
|
-
if (depth
|
|
54
|
+
const current = queue.shift();
|
|
55
|
+
if (current.depth >= maxDepth)
|
|
70
56
|
continue;
|
|
71
57
|
const edges = direction === 'upstream'
|
|
72
|
-
? graph.getIncomingEdges(id)
|
|
73
|
-
: graph.getOutgoingEdges(id);
|
|
58
|
+
? graph.getIncomingEdges(current.id)
|
|
59
|
+
: graph.getOutgoingEdges(current.id);
|
|
74
60
|
for (const edge of edges) {
|
|
75
61
|
const nextId = direction === 'upstream' ? edge.source : edge.target;
|
|
76
|
-
if (
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
62
|
+
if (seen.has(nextId))
|
|
63
|
+
continue;
|
|
64
|
+
seen.add(nextId);
|
|
65
|
+
includedIds.add(nextId);
|
|
66
|
+
queue.push({ id: nextId, depth: current.depth + 1 });
|
|
80
67
|
}
|
|
81
68
|
}
|
|
82
|
-
return visited;
|
|
83
69
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
* Examples:
|
|
89
|
-
* queryLineage(graph, { focus: 'total_revenue' })
|
|
90
|
-
* → subgraph centered on metric:total_revenue with full upstream/downstream
|
|
91
|
-
*
|
|
92
|
-
* queryLineage(graph, { search: 'revenue' })
|
|
93
|
-
* → all nodes matching "revenue" with relevance scores
|
|
94
|
-
*
|
|
95
|
-
* queryLineage(graph, { focus: 'fct_orders', upstreamDepth: 2, downstreamDepth: 1 })
|
|
96
|
-
* → 2 hops upstream, 1 hop downstream from dbt_model:fct_orders
|
|
97
|
-
*
|
|
98
|
-
* queryLineage(graph, { domain: 'finance', types: ['metric', 'block'] })
|
|
99
|
-
* → all finance metrics and blocks with their immediate connections
|
|
100
|
-
*/
|
|
101
|
-
export function queryLineage(graph, query) {
|
|
102
|
-
// Search mode: return matching nodes with scores
|
|
103
|
-
if (query.search && !query.focus) {
|
|
104
|
-
return executeSearch(graph, query);
|
|
105
|
-
}
|
|
106
|
-
// Focus mode: return subgraph centered on a node
|
|
107
|
-
if (query.focus) {
|
|
108
|
-
return executeFocus(graph, query);
|
|
109
|
-
}
|
|
110
|
-
// Filter mode: domain and/or type filter
|
|
111
|
-
if (query.domain || query.types) {
|
|
112
|
-
return executeFilter(graph, query);
|
|
113
|
-
}
|
|
114
|
-
// Default: return the full graph
|
|
115
|
-
return { graph: graph.toJSON() };
|
|
70
|
+
function normalizeDepth(depth) {
|
|
71
|
+
return depth === undefined || !Number.isFinite(depth) || depth < 0
|
|
72
|
+
? Number.POSITIVE_INFINITY
|
|
73
|
+
: depth;
|
|
116
74
|
}
|
|
117
|
-
function
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
for (const
|
|
121
|
-
|
|
122
|
-
if (
|
|
123
|
-
|
|
124
|
-
// Apply domain filter
|
|
125
|
-
if (query.domain && node.domain !== query.domain)
|
|
126
|
-
continue;
|
|
127
|
-
const score = scoreMatch(node.name, searchTerm);
|
|
128
|
-
if (score > 0) {
|
|
129
|
-
matches.push({ node, score });
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
// Sort by score descending, then alphabetically
|
|
133
|
-
matches.sort((a, b) => b.score - a.score || a.node.name.localeCompare(b.node.name));
|
|
134
|
-
// Build a subgraph of matched nodes with their immediate edges
|
|
135
|
-
const matchedIds = new Set(matches.map((m) => m.node.id));
|
|
136
|
-
const sub = graph.subgraph((n) => matchedIds.has(n.id));
|
|
137
|
-
return {
|
|
138
|
-
graph: sub.toJSON(),
|
|
139
|
-
matches,
|
|
140
|
-
};
|
|
141
|
-
}
|
|
142
|
-
function executeFocus(graph, query) {
|
|
143
|
-
const focalNode = resolveNode(graph, query.focus);
|
|
144
|
-
if (!focalNode) {
|
|
145
|
-
return { graph: { nodes: [], edges: [] }, matches: [] };
|
|
75
|
+
function resolveFocusNode(graph, rawFocus) {
|
|
76
|
+
if (graph.getNode(rawFocus))
|
|
77
|
+
return graph.getNode(rawFocus);
|
|
78
|
+
for (const prefix of NODE_PREFIXES) {
|
|
79
|
+
const candidate = graph.getNode(`${prefix}:${rawFocus}`);
|
|
80
|
+
if (candidate)
|
|
81
|
+
return candidate;
|
|
146
82
|
}
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
// Upstream BFS
|
|
152
|
-
const upstreamIds = depthLimitedBFS(graph, focalNode.id, 'upstream', upstreamDepth);
|
|
153
|
-
for (const id of upstreamIds)
|
|
154
|
-
nodeIds.add(id);
|
|
155
|
-
// Downstream BFS
|
|
156
|
-
const downstreamIds = depthLimitedBFS(graph, focalNode.id, 'downstream', downstreamDepth);
|
|
157
|
-
for (const id of downstreamIds)
|
|
158
|
-
nodeIds.add(id);
|
|
159
|
-
// Apply type/domain filters if specified
|
|
160
|
-
const sub = graph.subgraph((n) => {
|
|
161
|
-
if (!nodeIds.has(n.id))
|
|
162
|
-
return false;
|
|
163
|
-
if (query.types && !query.types.includes(n.type) && n.id !== focalNode.id)
|
|
164
|
-
return false;
|
|
165
|
-
if (query.domain && n.domain !== query.domain && n.id !== focalNode.id)
|
|
166
|
-
return false;
|
|
167
|
-
return true;
|
|
168
|
-
});
|
|
169
|
-
return {
|
|
170
|
-
graph: sub.toJSON(),
|
|
171
|
-
focalNode,
|
|
172
|
-
};
|
|
83
|
+
const normalized = rawFocus.trim().toLowerCase();
|
|
84
|
+
return graph
|
|
85
|
+
.getAllNodes()
|
|
86
|
+
.find((node) => node.name.toLowerCase() === normalized);
|
|
173
87
|
}
|
|
174
|
-
function
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
for (const edge of graph.getIncomingEdges(node.id)) {
|
|
187
|
-
matchedIds.add(edge.source);
|
|
188
|
-
}
|
|
189
|
-
for (const edge of graph.getOutgoingEdges(node.id)) {
|
|
190
|
-
matchedIds.add(edge.target);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
const sub = graph.subgraph((n) => matchedIds.has(n.id));
|
|
194
|
-
return { graph: sub.toJSON() };
|
|
88
|
+
function searchLineage(graph, rawTerm) {
|
|
89
|
+
const term = rawTerm.trim().toLowerCase();
|
|
90
|
+
if (!term)
|
|
91
|
+
return [];
|
|
92
|
+
return graph
|
|
93
|
+
.getAllNodes()
|
|
94
|
+
.map((node) => ({
|
|
95
|
+
node,
|
|
96
|
+
score: scoreMatch(node, term),
|
|
97
|
+
}))
|
|
98
|
+
.filter((entry) => entry.score > 0)
|
|
99
|
+
.sort((a, b) => b.score - a.score || a.node.name.localeCompare(b.node.name));
|
|
195
100
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
return
|
|
101
|
+
function scoreMatch(node, term) {
|
|
102
|
+
const name = node.name.toLowerCase();
|
|
103
|
+
const id = node.id.toLowerCase();
|
|
104
|
+
if (name === term || id === term)
|
|
105
|
+
return 100;
|
|
106
|
+
if (name.startsWith(term))
|
|
107
|
+
return 75;
|
|
108
|
+
if (id.startsWith(term))
|
|
109
|
+
return 60;
|
|
110
|
+
if (name.includes(term))
|
|
111
|
+
return 40;
|
|
112
|
+
if (id.includes(term))
|
|
113
|
+
return 30;
|
|
114
|
+
return 0;
|
|
210
115
|
}
|
|
211
116
|
//# sourceMappingURL=query.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"query.js","sourceRoot":"","sources":["../../src/lineage/query.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"query.js","sourceRoot":"","sources":["../../src/lineage/query.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAsB;IACvC,OAAO;IACP,WAAW;IACX,WAAW;IACX,YAAY;IACZ,cAAc;IACd,QAAQ;IACR,WAAW;IACX,QAAQ;IACR,OAAO;CACR,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,KAAmB;IACnE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEjF,IAAI,WAAW,GAAG,SAAS;QACzB,CAAC,CAAC,oBAAoB,CAClB,KAAK,EACL,SAAS,CAAC,EAAE,EACZ,KAAK,CAAC,aAAa,EACnB,KAAK,CAAC,eAAe,CACtB;QACH,CAAC,CAAC,KAAK,CAAC;IAEV,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACxC,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/D,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE;YAC1C,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC/D,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;gBAAE,OAAO,KAAK,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAChE,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE;YACpC,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;gBAAE,OAAO,KAAK,CAAC;YAC/D,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC1E,OAAO,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE;QAC3B,SAAS;QACT,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAmB,EACnB,OAAe,EACf,aAAsB,EACtB,eAAwB;IAExB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IAE/C,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,CAAC,aAAa,CAAC,EAAE,WAAW,CAAC,CAAC;IACtF,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,cAAc,CAAC,eAAe,CAAC,EAAE,WAAW,CAAC,CAAC;IAE1F,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,aAAa,CACpB,KAAmB,EACnB,OAAe,EACf,SAAoC,EACpC,QAAgB,EAChB,WAAwB;IAExB,MAAM,KAAK,GAAyC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAChF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IAExC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAC/B,IAAI,OAAO,CAAC,KAAK,IAAI,QAAQ;YAAE,SAAS;QAExC,MAAM,KAAK,GAAG,SAAS,KAAK,UAAU;YACpC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAEvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACpE,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS;YAC/B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjB,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAyB;IAC/C,OAAO,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAChE,CAAC,CAAC,MAAM,CAAC,iBAAiB;QAC1B,CAAC,CAAC,KAAK,CAAC;AACZ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAmB,EAAE,QAAgB;IAC7D,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE5D,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,MAAM,IAAI,QAAQ,EAAE,CAAC,CAAC;QACzD,IAAI,SAAS;YAAE,OAAO,SAAS,CAAC;IAClC,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACjD,OAAO,KAAK;SACT,WAAW,EAAE;SACb,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,UAAU,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,aAAa,CAAC,KAAmB,EAAE,OAAe;IACzD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,OAAO,KAAK;SACT,WAAW,EAAE;SACb,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACd,IAAI;QACJ,KAAK,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC;KAC9B,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,UAAU,CAAC,IAAiB,EAAE,IAAY;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;IACjC,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAC7C,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../src/manifest/builder.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;
|
|
1
|
+
{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../src/manifest/builder.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAgBH,OAAO,KAAK,EACV,WAAW,EASZ,MAAM,YAAY,CAAC;AAIpB,MAAM,WAAW,oBAAoB;IACnC,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2CAA2C;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,sDAAsD;IACtD,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,WAAW,CAmDxE"}
|
package/dist/manifest/builder.js
CHANGED
|
@@ -39,8 +39,8 @@ export function buildManifest(options) {
|
|
|
39
39
|
if (options.dbtManifestPath) {
|
|
40
40
|
dbtImport = importDbtManifest(options.dbtManifestPath, sources);
|
|
41
41
|
}
|
|
42
|
-
// Build lineage
|
|
43
|
-
const lineage = buildManifestLineage(blocks, metrics, dimensions,
|
|
42
|
+
// Build lineage
|
|
43
|
+
const lineage = buildManifestLineage(blocks, metrics, dimensions, notebooks, dbtImport);
|
|
44
44
|
return {
|
|
45
45
|
manifestVersion: 2,
|
|
46
46
|
dqlVersion,
|
|
@@ -352,41 +352,9 @@ function importDbtManifest(manifestPath, sources) {
|
|
|
352
352
|
let modelsImported = 0;
|
|
353
353
|
let sourcesImported = 0;
|
|
354
354
|
const projectName = manifest.metadata?.project_name;
|
|
355
|
-
// Maps for resolving dbt uniqueId → table name (for depends_on edge resolution)
|
|
356
|
-
const uniqueIdToName = new Map();
|
|
357
355
|
const dbtDagModels = [];
|
|
358
356
|
const dbtDagEdges = [];
|
|
359
|
-
//
|
|
360
|
-
const dbtSources = manifest.sources ?? {};
|
|
361
|
-
for (const [uniqueId, src] of Object.entries(dbtSources)) {
|
|
362
|
-
const tableName = src.identifier ?? src.name;
|
|
363
|
-
if (!tableName)
|
|
364
|
-
continue;
|
|
365
|
-
uniqueIdToName.set(uniqueId, tableName);
|
|
366
|
-
if (!sources[tableName]) {
|
|
367
|
-
sources[tableName] = {
|
|
368
|
-
name: tableName,
|
|
369
|
-
origin: 'dbt',
|
|
370
|
-
referencedBy: [],
|
|
371
|
-
};
|
|
372
|
-
}
|
|
373
|
-
sources[tableName].dbtModel = {
|
|
374
|
-
uniqueId,
|
|
375
|
-
database: src.database,
|
|
376
|
-
schema: src.schema,
|
|
377
|
-
description: src.description,
|
|
378
|
-
};
|
|
379
|
-
// Mark as dbt_source type for lineage
|
|
380
|
-
sources[tableName].dbtNodeType = 'source';
|
|
381
|
-
sourcesImported++;
|
|
382
|
-
dbtDagModels.push({
|
|
383
|
-
uniqueId,
|
|
384
|
-
name: tableName,
|
|
385
|
-
type: 'source',
|
|
386
|
-
dependsOn: [],
|
|
387
|
-
});
|
|
388
|
-
}
|
|
389
|
-
// Pass 2: Import models with dependency tracking
|
|
357
|
+
// Import models as source tables
|
|
390
358
|
const nodes = manifest.nodes ?? {};
|
|
391
359
|
for (const [uniqueId, node] of Object.entries(nodes)) {
|
|
392
360
|
if (node.resource_type !== 'model')
|
|
@@ -394,16 +362,14 @@ function importDbtManifest(manifestPath, sources) {
|
|
|
394
362
|
const tableName = node.alias ?? node.name;
|
|
395
363
|
if (!tableName)
|
|
396
364
|
continue;
|
|
397
|
-
|
|
398
|
-
// Extract column metadata
|
|
365
|
+
// Create or update source entry
|
|
399
366
|
const columns = node.columns
|
|
400
|
-
? Object.
|
|
401
|
-
name:
|
|
402
|
-
|
|
403
|
-
|
|
367
|
+
? Object.values(node.columns).map((value) => ({
|
|
368
|
+
name: value.name,
|
|
369
|
+
description: value.description,
|
|
370
|
+
type: value.data_type,
|
|
404
371
|
}))
|
|
405
372
|
: undefined;
|
|
406
|
-
// Create or update source entry
|
|
407
373
|
if (!sources[tableName]) {
|
|
408
374
|
sources[tableName] = {
|
|
409
375
|
name: tableName,
|
|
@@ -418,39 +384,62 @@ function importDbtManifest(manifestPath, sources) {
|
|
|
418
384
|
schema: node.schema,
|
|
419
385
|
materializedAs: node.config?.materialized,
|
|
420
386
|
description: node.description,
|
|
421
|
-
columns: columns
|
|
422
|
-
? Object.fromEntries(columns.map((
|
|
387
|
+
columns: node.columns
|
|
388
|
+
? Object.fromEntries(Object.entries(node.columns).map(([k, v]) => [
|
|
389
|
+
k,
|
|
390
|
+
{ name: v.name, description: v.description, type: v.data_type },
|
|
391
|
+
]))
|
|
423
392
|
: undefined,
|
|
424
393
|
};
|
|
425
|
-
// Mark as dbt_model type for lineage
|
|
426
|
-
sources[tableName].dbtNodeType = 'model';
|
|
427
394
|
if (sources[tableName].origin === 'sql') {
|
|
428
395
|
sources[tableName].origin = 'dbt';
|
|
429
396
|
}
|
|
430
397
|
modelsImported++;
|
|
431
|
-
|
|
432
|
-
const dependsOnIds = node.depends_on?.nodes ?? [];
|
|
398
|
+
const dependsOn = Array.isArray(node.depends_on?.nodes) ? node.depends_on.nodes : [];
|
|
433
399
|
dbtDagModels.push({
|
|
434
400
|
uniqueId,
|
|
435
401
|
name: tableName,
|
|
436
402
|
type: 'model',
|
|
437
|
-
dependsOn
|
|
403
|
+
dependsOn,
|
|
438
404
|
columns,
|
|
405
|
+
schema: node.schema,
|
|
406
|
+
database: node.database,
|
|
407
|
+
materialized: node.config?.materialized,
|
|
408
|
+
description: node.description,
|
|
439
409
|
});
|
|
410
|
+
for (const dependency of dependsOn) {
|
|
411
|
+
dbtDagEdges.push({ source: dependency, target: uniqueId });
|
|
412
|
+
}
|
|
440
413
|
}
|
|
441
|
-
//
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
414
|
+
// Import dbt sources
|
|
415
|
+
const dbtSources = manifest.sources ?? {};
|
|
416
|
+
for (const [uniqueId, src] of Object.entries(dbtSources)) {
|
|
417
|
+
const tableName = src.identifier ?? src.name;
|
|
418
|
+
if (!tableName)
|
|
419
|
+
continue;
|
|
420
|
+
if (!sources[tableName]) {
|
|
421
|
+
sources[tableName] = {
|
|
422
|
+
name: tableName,
|
|
423
|
+
origin: 'dbt',
|
|
424
|
+
referencedBy: [],
|
|
425
|
+
};
|
|
452
426
|
}
|
|
453
|
-
|
|
427
|
+
sources[tableName].dbtModel = {
|
|
428
|
+
uniqueId,
|
|
429
|
+
database: src.database,
|
|
430
|
+
schema: src.schema,
|
|
431
|
+
description: src.description,
|
|
432
|
+
};
|
|
433
|
+
sourcesImported++;
|
|
434
|
+
dbtDagModels.push({
|
|
435
|
+
uniqueId,
|
|
436
|
+
name: tableName,
|
|
437
|
+
type: 'source',
|
|
438
|
+
dependsOn: [],
|
|
439
|
+
schema: src.schema,
|
|
440
|
+
database: src.database,
|
|
441
|
+
description: src.description,
|
|
442
|
+
});
|
|
454
443
|
}
|
|
455
444
|
return {
|
|
456
445
|
manifestPath,
|
|
@@ -465,7 +454,7 @@ function importDbtManifest(manifestPath, sources) {
|
|
|
465
454
|
};
|
|
466
455
|
}
|
|
467
456
|
// ---- Lineage Builder ----
|
|
468
|
-
function buildManifestLineage(blocks, metrics, dimensions,
|
|
457
|
+
function buildManifestLineage(blocks, metrics, dimensions, notebooks, dbtImport) {
|
|
469
458
|
// Convert to lineage builder input format
|
|
470
459
|
const lineageBlocks = Object.values(blocks).map((b) => ({
|
|
471
460
|
name: b.name,
|
|
@@ -487,26 +476,27 @@ function buildManifestLineage(blocks, metrics, dimensions, dbtImport, notebooks)
|
|
|
487
476
|
name: d.name,
|
|
488
477
|
table: d.table,
|
|
489
478
|
}));
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
:
|
|
479
|
+
const dashboards = Object.values(notebooks ?? {}).map((notebook) => ({
|
|
480
|
+
name: notebook.title,
|
|
481
|
+
blocks: notebook.cells
|
|
482
|
+
.map((cell) => cell.blockName)
|
|
483
|
+
.filter((name) => Boolean(name)),
|
|
484
|
+
charts: notebook.cells
|
|
485
|
+
.filter((cell) => cell.chartType && cell.blockName)
|
|
486
|
+
.map((cell) => cell.blockName)
|
|
487
|
+
.filter(Boolean),
|
|
488
|
+
}));
|
|
489
|
+
const dbtModels = (dbtImport?.dbtDag?.models ?? []).map((model) => ({
|
|
490
|
+
name: model.name,
|
|
491
|
+
uniqueId: model.uniqueId,
|
|
492
|
+
type: model.type,
|
|
493
|
+
dependsOn: model.dependsOn,
|
|
494
|
+
columns: model.columns,
|
|
495
|
+
schema: model.schema,
|
|
496
|
+
database: model.database,
|
|
497
|
+
materialized: model.materialized,
|
|
498
|
+
description: model.description,
|
|
499
|
+
}));
|
|
510
500
|
const graph = buildLineageGraph(lineageBlocks, lineageMetrics, lineageDimensions, {
|
|
511
501
|
dbtModels,
|
|
512
502
|
dashboards,
|