@dbt-tools/core 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +51 -0
- package/dist/analysis/analysis-snapshot.d.ts +145 -0
- package/dist/analysis/analysis-snapshot.js +615 -0
- package/dist/analysis/dependency-service.d.ts +56 -0
- package/dist/analysis/dependency-service.js +75 -0
- package/dist/analysis/execution-analyzer.d.ts +85 -0
- package/dist/analysis/execution-analyzer.js +245 -0
- package/dist/analysis/manifest-graph.d.ts +118 -0
- package/dist/analysis/manifest-graph.js +651 -0
- package/dist/analysis/run-results-search.d.ts +56 -0
- package/dist/analysis/run-results-search.js +127 -0
- package/dist/analysis/sql-analyzer.d.ts +30 -0
- package/dist/analysis/sql-analyzer.js +218 -0
- package/dist/browser.d.ts +11 -0
- package/dist/browser.js +17 -0
- package/dist/errors/error-handler.d.ts +26 -0
- package/dist/errors/error-handler.js +59 -0
- package/dist/formatting/field-filter.d.ts +29 -0
- package/dist/formatting/field-filter.js +112 -0
- package/dist/formatting/graph-export.d.ts +9 -0
- package/dist/formatting/graph-export.js +147 -0
- package/dist/formatting/output-formatter.d.ts +77 -0
- package/dist/formatting/output-formatter.js +160 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +38 -0
- package/dist/introspection/schema-generator.d.ts +29 -0
- package/dist/introspection/schema-generator.js +275 -0
- package/dist/io/artifact-loader.d.ts +27 -0
- package/dist/io/artifact-loader.js +142 -0
- package/dist/types.d.ts +43 -0
- package/dist/types.js +2 -0
- package/dist/validation/input-validator.d.ts +39 -0
- package/dist/validation/input-validator.js +167 -0
- package/dist/version.d.ts +28 -0
- package/dist/version.js +60 -0
- package/package.json +47 -0
|
@@ -0,0 +1,651 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ManifestGraph = void 0;
|
|
4
|
+
const graphology_1 = require("graphology");
|
|
5
|
+
const graphology_dag_1 = require("graphology-dag");
|
|
6
|
+
const version_1 = require("../version");
|
|
7
|
+
function getManifestMetrics(manifest) {
|
|
8
|
+
return manifest.metrics;
|
|
9
|
+
}
|
|
10
|
+
function getManifestSemanticModels(manifest) {
|
|
11
|
+
return manifest.semantic_models;
|
|
12
|
+
}
|
|
13
|
+
function getManifestUnitTests(manifest) {
|
|
14
|
+
return manifest.unit_tests;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* ManifestGraph builds and manages a directed graph from a dbt manifest.
|
|
18
|
+
*
|
|
19
|
+
* This class transforms dbt artifacts into a graphology graph, enabling
|
|
20
|
+
* efficient graph operations like cycle detection, path finding, and traversal.
|
|
21
|
+
*/
|
|
22
|
+
class ManifestGraph {
|
|
23
|
+
constructor(manifest) {
|
|
24
|
+
this.relationMap = new Map(); // relation_name -> unique_id
|
|
25
|
+
if (!(0, version_1.isSupportedVersion)(manifest)) {
|
|
26
|
+
const versionInfo = (0, version_1.getVersionInfo)(manifest);
|
|
27
|
+
throw new Error(`Unsupported dbt version. ` +
|
|
28
|
+
`Schema version: ${versionInfo.schema_version || "unknown"}, ` +
|
|
29
|
+
`dbt version: ${versionInfo.dbt_version || "unknown"}. ` +
|
|
30
|
+
`Requires dbt 1.10+ (manifest schema v${version_1.MIN_SUPPORTED_SCHEMA_VERSION}+)`);
|
|
31
|
+
}
|
|
32
|
+
this.graph = new graphology_1.DirectedGraph();
|
|
33
|
+
this.buildGraph(manifest);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Build the graph from manifest data
|
|
37
|
+
*/
|
|
38
|
+
buildGraph(manifest) {
|
|
39
|
+
// Add all nodes from manifest
|
|
40
|
+
this.addNodes(manifest);
|
|
41
|
+
// Add edges based on dependencies
|
|
42
|
+
this.addEdges(manifest);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Add nodes from manifest to the graph
|
|
46
|
+
*/
|
|
47
|
+
/** Map relation_name (lowercase) to unique_id when present on manifest entries. */
|
|
48
|
+
registerRelationName(uniqueId, relationName) {
|
|
49
|
+
if (!relationName)
|
|
50
|
+
return;
|
|
51
|
+
this.relationMap.set(relationName.toLowerCase(), uniqueId);
|
|
52
|
+
}
|
|
53
|
+
addNodes(manifest) {
|
|
54
|
+
this.addNodeEntries(manifest.nodes);
|
|
55
|
+
this.addSourceEntries(manifest.sources);
|
|
56
|
+
this.addMacroEntries(manifest.macros);
|
|
57
|
+
this.addExposureEntries(manifest.exposures);
|
|
58
|
+
this.addMetricEntries(getManifestMetrics(manifest));
|
|
59
|
+
this.addSemanticModelEntries(getManifestSemanticModels(manifest));
|
|
60
|
+
this.addUnitTestEntries(getManifestUnitTests(manifest));
|
|
61
|
+
}
|
|
62
|
+
addNodeEntries(nodes) {
|
|
63
|
+
if (!nodes)
|
|
64
|
+
return;
|
|
65
|
+
for (const [uniqueId, node] of Object.entries(nodes)) {
|
|
66
|
+
const nodeAny = node;
|
|
67
|
+
const resourceType = this.extractResourceType(nodeAny.resource_type || "model");
|
|
68
|
+
const config = nodeAny.config;
|
|
69
|
+
const materializedRaw = config === null || config === void 0 ? void 0 : config.materialized;
|
|
70
|
+
const materialized = typeof materializedRaw === "string" && materializedRaw.trim() !== ""
|
|
71
|
+
? materializedRaw
|
|
72
|
+
: undefined;
|
|
73
|
+
this.graph.addNode(uniqueId, Object.assign({ unique_id: uniqueId, resource_type: resourceType, name: nodeAny.name || uniqueId, package_name: nodeAny.package_name || "", path: nodeAny.path || undefined, original_file_path: nodeAny.original_file_path || undefined, patch_path: nodeAny.patch_path || undefined, database: nodeAny.database || undefined, schema: nodeAny.schema || undefined, tags: nodeAny.tags || undefined, description: nodeAny.description || undefined, compiled_code: nodeAny.compiled_code || undefined, raw_code: nodeAny.raw_code ||
|
|
74
|
+
nodeAny.raw_sql ||
|
|
75
|
+
undefined }, (materialized != null ? { materialized } : {})));
|
|
76
|
+
this.registerRelationName(uniqueId, nodeAny.relation_name);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
addSourceEntries(sources) {
|
|
80
|
+
if (!sources)
|
|
81
|
+
return;
|
|
82
|
+
for (const [uniqueId, source] of Object.entries(sources)) {
|
|
83
|
+
const sourceAny = source;
|
|
84
|
+
this.graph.addNode(uniqueId, {
|
|
85
|
+
unique_id: uniqueId,
|
|
86
|
+
resource_type: "source",
|
|
87
|
+
name: sourceAny.name || uniqueId,
|
|
88
|
+
package_name: sourceAny.package_name || "",
|
|
89
|
+
path: sourceAny.path || undefined,
|
|
90
|
+
original_file_path: sourceAny.original_file_path || undefined,
|
|
91
|
+
tags: sourceAny.tags || undefined,
|
|
92
|
+
description: sourceAny.description || undefined,
|
|
93
|
+
});
|
|
94
|
+
this.registerRelationName(uniqueId, sourceAny.relation_name);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
addMacroEntries(macros) {
|
|
98
|
+
if (!macros)
|
|
99
|
+
return;
|
|
100
|
+
for (const [uniqueId, macro] of Object.entries(macros)) {
|
|
101
|
+
const macroAny = macro;
|
|
102
|
+
this.graph.addNode(uniqueId, {
|
|
103
|
+
unique_id: uniqueId,
|
|
104
|
+
resource_type: "macro",
|
|
105
|
+
name: macroAny.name || uniqueId,
|
|
106
|
+
package_name: macroAny.package_name || "",
|
|
107
|
+
path: macroAny.path || undefined,
|
|
108
|
+
original_file_path: macroAny.original_file_path || undefined,
|
|
109
|
+
description: macroAny.description || undefined,
|
|
110
|
+
raw_code: macroAny.macro_sql || undefined,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
addExposureEntries(exposures) {
|
|
115
|
+
if (!exposures)
|
|
116
|
+
return;
|
|
117
|
+
for (const [uniqueId, exposure] of Object.entries(exposures)) {
|
|
118
|
+
const exposureAny = exposure;
|
|
119
|
+
this.graph.addNode(uniqueId, {
|
|
120
|
+
unique_id: uniqueId,
|
|
121
|
+
resource_type: "exposure",
|
|
122
|
+
name: exposureAny.name || uniqueId,
|
|
123
|
+
package_name: exposureAny.package_name || "",
|
|
124
|
+
tags: exposureAny.tags || undefined,
|
|
125
|
+
description: exposureAny.description || undefined,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
addMetricEntries(metrics) {
|
|
130
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
131
|
+
if (!metrics)
|
|
132
|
+
return;
|
|
133
|
+
for (const [uniqueId, metric] of Object.entries(metrics)) {
|
|
134
|
+
const metricAny = metric;
|
|
135
|
+
const tagsValue = metricAny.tags;
|
|
136
|
+
const tags = Array.isArray(tagsValue)
|
|
137
|
+
? tagsValue
|
|
138
|
+
: typeof tagsValue === "string"
|
|
139
|
+
? [tagsValue]
|
|
140
|
+
: undefined;
|
|
141
|
+
this.graph.addNode(uniqueId, {
|
|
142
|
+
unique_id: uniqueId,
|
|
143
|
+
resource_type: "metric",
|
|
144
|
+
name: metricAny.name || uniqueId,
|
|
145
|
+
package_name: metricAny.package_name || "",
|
|
146
|
+
path: metricAny.path || undefined,
|
|
147
|
+
original_file_path: metricAny.original_file_path || undefined,
|
|
148
|
+
description: metricAny.description || undefined,
|
|
149
|
+
label: metricAny.label || undefined,
|
|
150
|
+
metric_type: metricAny.type || undefined,
|
|
151
|
+
metric_expression: ((_a = metricAny.type_params) === null || _a === void 0 ? void 0 : _a.expr) || undefined,
|
|
152
|
+
metric_measure: ((_c = (_b = metricAny.type_params) === null || _b === void 0 ? void 0 : _b.measure) === null || _c === void 0 ? void 0 : _c.name) || undefined,
|
|
153
|
+
metric_input_measures: Array.isArray((_d = metricAny.type_params) === null || _d === void 0 ? void 0 : _d.input_measures)
|
|
154
|
+
? metricAny.type_params
|
|
155
|
+
.input_measures
|
|
156
|
+
.map((entry) => entry.name)
|
|
157
|
+
.filter((entry) => typeof entry === "string")
|
|
158
|
+
: undefined,
|
|
159
|
+
metric_input_metrics: Array.isArray((_e = metricAny.type_params) === null || _e === void 0 ? void 0 : _e.metrics)
|
|
160
|
+
? metricAny.type_params
|
|
161
|
+
.metrics
|
|
162
|
+
.map((entry) => entry.name)
|
|
163
|
+
.filter((entry) => typeof entry === "string")
|
|
164
|
+
: undefined,
|
|
165
|
+
metric_time_granularity: metricAny.time_granularity || undefined,
|
|
166
|
+
metric_filters: Array.isArray((_f = metricAny.filter) === null || _f === void 0 ? void 0 : _f.where_filters)
|
|
167
|
+
? metricAny.filter
|
|
168
|
+
.where_filters
|
|
169
|
+
.map((entry) => entry.where_sql_template)
|
|
170
|
+
.filter((entry) => typeof entry === "string")
|
|
171
|
+
: undefined,
|
|
172
|
+
metric_source_reference: ((_h = (_g = metricAny.depends_on) === null || _g === void 0 ? void 0 : _g.nodes) === null || _h === void 0 ? void 0 : _h[0]) || undefined,
|
|
173
|
+
tags,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
addSemanticModelEntries(semanticModels) {
|
|
178
|
+
var _a;
|
|
179
|
+
if (!semanticModels)
|
|
180
|
+
return;
|
|
181
|
+
for (const [uniqueId, semanticModel] of Object.entries(semanticModels)) {
|
|
182
|
+
const sm = semanticModel;
|
|
183
|
+
this.graph.addNode(uniqueId, {
|
|
184
|
+
unique_id: uniqueId,
|
|
185
|
+
resource_type: "semantic_model",
|
|
186
|
+
name: sm.name || uniqueId,
|
|
187
|
+
package_name: sm.package_name || "",
|
|
188
|
+
path: sm.path || undefined,
|
|
189
|
+
original_file_path: sm.original_file_path || undefined,
|
|
190
|
+
description: sm.description || undefined,
|
|
191
|
+
label: sm.label || undefined,
|
|
192
|
+
semantic_model_reference: sm.model || undefined,
|
|
193
|
+
semantic_model_default_time_dimension: ((_a = sm.defaults) === null || _a === void 0 ? void 0 : _a.agg_time_dimension) || undefined,
|
|
194
|
+
semantic_model_entities: Array.isArray(sm.entities)
|
|
195
|
+
? sm.entities
|
|
196
|
+
.map((entry) => entry.name)
|
|
197
|
+
.filter((entry) => typeof entry === "string")
|
|
198
|
+
: undefined,
|
|
199
|
+
semantic_model_measures: Array.isArray(sm.measures)
|
|
200
|
+
? sm.measures
|
|
201
|
+
.map((entry) => entry.name)
|
|
202
|
+
.filter((entry) => typeof entry === "string")
|
|
203
|
+
: undefined,
|
|
204
|
+
semantic_model_dimensions: Array.isArray(sm.dimensions)
|
|
205
|
+
? sm.dimensions
|
|
206
|
+
.map((entry) => entry.name)
|
|
207
|
+
.filter((entry) => typeof entry === "string")
|
|
208
|
+
: undefined,
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
addUnitTestEntries(unitTests) {
|
|
213
|
+
if (!unitTests)
|
|
214
|
+
return;
|
|
215
|
+
for (const [uniqueId, unitTest] of Object.entries(unitTests)) {
|
|
216
|
+
const ut = unitTest;
|
|
217
|
+
this.graph.addNode(uniqueId, {
|
|
218
|
+
unique_id: uniqueId,
|
|
219
|
+
resource_type: "unit_test",
|
|
220
|
+
name: ut.name || uniqueId,
|
|
221
|
+
package_name: ut.package_name || "",
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Add edges based on dependencies
|
|
227
|
+
*/
|
|
228
|
+
addEdges(manifest) {
|
|
229
|
+
if (manifest.parent_map) {
|
|
230
|
+
this.addEdgesFromParentMap(manifest.parent_map);
|
|
231
|
+
}
|
|
232
|
+
this.addEdgesFromNodeDependsOn(manifest.nodes);
|
|
233
|
+
this.addEdgesFromExposureDependsOn(manifest.exposures);
|
|
234
|
+
this.addEdgesFromMetricDependsOn(getManifestMetrics(manifest));
|
|
235
|
+
}
|
|
236
|
+
addEdgesFromParentMap(parentMap) {
|
|
237
|
+
for (const [childId, parentIds] of Object.entries(parentMap)) {
|
|
238
|
+
if (!this.graph.hasNode(childId))
|
|
239
|
+
continue;
|
|
240
|
+
for (const parentId of parentIds) {
|
|
241
|
+
if (this.graph.hasNode(parentId) &&
|
|
242
|
+
!this.graph.hasEdge(parentId, childId)) {
|
|
243
|
+
this.graph.addEdge(parentId, childId, {
|
|
244
|
+
dependency_type: this.inferDependencyType(parentId),
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
addEdgesFromDependsOn(childId, dependsOn) {
|
|
251
|
+
if (dependsOn.nodes) {
|
|
252
|
+
for (const depId of dependsOn.nodes) {
|
|
253
|
+
if (this.graph.hasNode(depId) && !this.graph.hasEdge(depId, childId)) {
|
|
254
|
+
this.graph.addEdge(depId, childId, { dependency_type: "node" });
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
if (dependsOn.macros) {
|
|
259
|
+
for (const macroId of dependsOn.macros) {
|
|
260
|
+
if (this.graph.hasNode(macroId) &&
|
|
261
|
+
!this.graph.hasEdge(macroId, childId)) {
|
|
262
|
+
this.graph.addEdge(macroId, childId, { dependency_type: "macro" });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
addEdgesFromNodeDependsOn(nodes) {
|
|
268
|
+
if (!nodes)
|
|
269
|
+
return;
|
|
270
|
+
for (const [uniqueId, node] of Object.entries(nodes)) {
|
|
271
|
+
const dep = node.depends_on;
|
|
272
|
+
if (dep)
|
|
273
|
+
this.addEdgesFromDependsOn(uniqueId, dep);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
addEdgesFromExposureDependsOn(exposures) {
|
|
277
|
+
if (!exposures)
|
|
278
|
+
return;
|
|
279
|
+
for (const [uniqueId, exposure] of Object.entries(exposures)) {
|
|
280
|
+
const dep = exposure.depends_on;
|
|
281
|
+
if (dep === null || dep === void 0 ? void 0 : dep.nodes)
|
|
282
|
+
this.addEdgesFromDependsOn(uniqueId, dep);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
addEdgesFromMetricDependsOn(metrics) {
|
|
286
|
+
if (!metrics)
|
|
287
|
+
return;
|
|
288
|
+
for (const [uniqueId, metric] of Object.entries(metrics)) {
|
|
289
|
+
const dep = metric.depends_on;
|
|
290
|
+
if (dep === null || dep === void 0 ? void 0 : dep.nodes)
|
|
291
|
+
this.addEdgesFromDependsOn(uniqueId, dep);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Extract resource type from manifest node
|
|
296
|
+
*/
|
|
297
|
+
extractResourceType(resourceType) {
|
|
298
|
+
const normalized = resourceType.toLowerCase();
|
|
299
|
+
if ([
|
|
300
|
+
"model",
|
|
301
|
+
"source",
|
|
302
|
+
"seed",
|
|
303
|
+
"snapshot",
|
|
304
|
+
"test",
|
|
305
|
+
"analysis",
|
|
306
|
+
"macro",
|
|
307
|
+
"exposure",
|
|
308
|
+
"metric",
|
|
309
|
+
"semantic_model",
|
|
310
|
+
"unit_test",
|
|
311
|
+
"function",
|
|
312
|
+
].includes(normalized)) {
|
|
313
|
+
return normalized;
|
|
314
|
+
}
|
|
315
|
+
return "model"; // Default fallback
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Infer dependency type from node ID
|
|
319
|
+
*/
|
|
320
|
+
inferDependencyType(nodeId) {
|
|
321
|
+
if (nodeId.startsWith("macro.")) {
|
|
322
|
+
return "macro";
|
|
323
|
+
}
|
|
324
|
+
if (nodeId.startsWith("source.")) {
|
|
325
|
+
return "source";
|
|
326
|
+
}
|
|
327
|
+
return "node";
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Get the underlying graphology graph
|
|
331
|
+
*/
|
|
332
|
+
getGraph() {
|
|
333
|
+
return this.graph;
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Get summary statistics about the graph
|
|
337
|
+
*/
|
|
338
|
+
getSummary() {
|
|
339
|
+
const nodesByType = {};
|
|
340
|
+
let totalNodes = 0;
|
|
341
|
+
// Count nodes by type
|
|
342
|
+
this.graph.forEachNode((nodeId, attributes) => {
|
|
343
|
+
totalNodes++;
|
|
344
|
+
const type = attributes.resource_type || "unknown";
|
|
345
|
+
nodesByType[type] = (nodesByType[type] || 0) + 1;
|
|
346
|
+
});
|
|
347
|
+
// Detect cycles using graphology-dag
|
|
348
|
+
const hasCycles = (0, graphology_dag_1.hasCycle)(this.graph);
|
|
349
|
+
return {
|
|
350
|
+
total_nodes: totalNodes,
|
|
351
|
+
nodes_by_type: nodesByType,
|
|
352
|
+
total_edges: this.graph.size,
|
|
353
|
+
has_cycles: hasCycles,
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Get all upstream dependencies of a node using BFS.
|
|
358
|
+
* @param nodeId - The node to find upstream dependencies for
|
|
359
|
+
* @param maxDepth - Optional limit; 1 = immediate neighbors only, undefined = all levels
|
|
360
|
+
* @returns Array of { nodeId, depth } where depth is the shortest distance from the node
|
|
361
|
+
*/
|
|
362
|
+
getUpstream(nodeId, maxDepth) {
|
|
363
|
+
if (!this.graph.hasNode(nodeId)) {
|
|
364
|
+
return [];
|
|
365
|
+
}
|
|
366
|
+
const result = [];
|
|
367
|
+
const visited = new Set();
|
|
368
|
+
const queue = [];
|
|
369
|
+
let head = 0;
|
|
370
|
+
for (const neighborId of this.graph.inboundNeighbors(nodeId)) {
|
|
371
|
+
if (!visited.has(neighborId)) {
|
|
372
|
+
visited.add(neighborId);
|
|
373
|
+
result.push({ nodeId: neighborId, depth: 1 });
|
|
374
|
+
if (maxDepth === undefined || 1 < maxDepth) {
|
|
375
|
+
queue.push({ id: neighborId, depth: 1 });
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
while (head < queue.length) {
|
|
380
|
+
const { id, depth } = queue[head++];
|
|
381
|
+
const nextDepth = depth + 1;
|
|
382
|
+
for (const neighborId of this.graph.inboundNeighbors(id)) {
|
|
383
|
+
if (!visited.has(neighborId)) {
|
|
384
|
+
visited.add(neighborId);
|
|
385
|
+
result.push({ nodeId: neighborId, depth: nextDepth });
|
|
386
|
+
if (maxDepth === undefined || nextDepth < maxDepth) {
|
|
387
|
+
queue.push({ id: neighborId, depth: nextDepth });
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
return result;
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* Get all downstream dependents of a node using BFS.
|
|
396
|
+
* @param nodeId - The node to find downstream dependents for
|
|
397
|
+
* @param maxDepth - Optional limit; 1 = immediate neighbors only, undefined = all levels
|
|
398
|
+
* @returns Array of { nodeId, depth } where depth is the shortest distance from the node
|
|
399
|
+
*/
|
|
400
|
+
getDownstream(nodeId, maxDepth) {
|
|
401
|
+
if (!this.graph.hasNode(nodeId)) {
|
|
402
|
+
return [];
|
|
403
|
+
}
|
|
404
|
+
const result = [];
|
|
405
|
+
const visited = new Set();
|
|
406
|
+
const queue = [];
|
|
407
|
+
let head = 0;
|
|
408
|
+
for (const neighborId of this.graph.outboundNeighbors(nodeId)) {
|
|
409
|
+
if (!visited.has(neighborId)) {
|
|
410
|
+
visited.add(neighborId);
|
|
411
|
+
result.push({ nodeId: neighborId, depth: 1 });
|
|
412
|
+
if (maxDepth === undefined || 1 < maxDepth) {
|
|
413
|
+
queue.push({ id: neighborId, depth: 1 });
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
while (head < queue.length) {
|
|
418
|
+
const { id, depth } = queue[head++];
|
|
419
|
+
const nextDepth = depth + 1;
|
|
420
|
+
for (const neighborId of this.graph.outboundNeighbors(id)) {
|
|
421
|
+
if (!visited.has(neighborId)) {
|
|
422
|
+
visited.add(neighborId);
|
|
423
|
+
result.push({ nodeId: neighborId, depth: nextDepth });
|
|
424
|
+
if (maxDepth === undefined || nextDepth < maxDepth) {
|
|
425
|
+
queue.push({ id: neighborId, depth: nextDepth });
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return result;
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Get upstream dependencies with parent info for tree construction.
|
|
434
|
+
* @returns Array of { nodeId, depth, parentId } where parentId is the BFS predecessor
|
|
435
|
+
*/
|
|
436
|
+
getUpstreamWithParents(nodeId, maxDepth) {
|
|
437
|
+
if (!this.graph.hasNode(nodeId)) {
|
|
438
|
+
return [];
|
|
439
|
+
}
|
|
440
|
+
const result = [];
|
|
441
|
+
const visited = new Set();
|
|
442
|
+
const queue = [];
|
|
443
|
+
for (const neighborId of this.graph.inboundNeighbors(nodeId)) {
|
|
444
|
+
if (!visited.has(neighborId)) {
|
|
445
|
+
visited.add(neighborId);
|
|
446
|
+
result.push({ nodeId: neighborId, depth: 1, parentId: nodeId });
|
|
447
|
+
if (maxDepth === undefined || 1 < maxDepth) {
|
|
448
|
+
queue.push({ id: neighborId, depth: 1, parentId: nodeId });
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
while (queue.length > 0) {
|
|
453
|
+
const { id, depth } = queue.shift();
|
|
454
|
+
const nextDepth = depth + 1;
|
|
455
|
+
for (const neighborId of this.graph.inboundNeighbors(id)) {
|
|
456
|
+
if (!visited.has(neighborId)) {
|
|
457
|
+
visited.add(neighborId);
|
|
458
|
+
result.push({ nodeId: neighborId, depth: nextDepth, parentId: id });
|
|
459
|
+
if (maxDepth === undefined || nextDepth < maxDepth) {
|
|
460
|
+
queue.push({ id: neighborId, depth: nextDepth, parentId: id });
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
return result;
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Get upstream dependencies in build order (topological sort).
|
|
469
|
+
* Sources and root models first, then models that depend on them.
|
|
470
|
+
* @param nodeId - The node to find upstream dependencies for
|
|
471
|
+
* @param maxDepth - Optional limit; 1 = immediate neighbors only, undefined = all levels
|
|
472
|
+
* @returns Array of { nodeId, depth } in build order
|
|
473
|
+
*/
|
|
474
|
+
getUpstreamBuildOrder(nodeId, maxDepth) {
|
|
475
|
+
const entries = this.getUpstream(nodeId, maxDepth);
|
|
476
|
+
if (entries.length === 0) {
|
|
477
|
+
return [];
|
|
478
|
+
}
|
|
479
|
+
const nodeIds = new Set(entries.map((e) => e.nodeId));
|
|
480
|
+
const depthByNode = new Map(entries.map((e) => [e.nodeId, e.depth]));
|
|
481
|
+
const subgraph = new graphology_1.DirectedGraph();
|
|
482
|
+
for (const nid of nodeIds) {
|
|
483
|
+
subgraph.addNode(nid, this.graph.getNodeAttributes(nid));
|
|
484
|
+
}
|
|
485
|
+
this.graph.forEachEdge((_edge, attr, source, target) => {
|
|
486
|
+
if (nodeIds.has(source) && nodeIds.has(target)) {
|
|
487
|
+
if (!subgraph.hasEdge(source, target)) {
|
|
488
|
+
subgraph.addEdge(source, target, attr);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
});
|
|
492
|
+
const orderedIds = (0, graphology_dag_1.topologicalSort)(subgraph);
|
|
493
|
+
return orderedIds.map((nid) => {
|
|
494
|
+
var _a;
|
|
495
|
+
return ({
|
|
496
|
+
nodeId: nid,
|
|
497
|
+
depth: (_a = depthByNode.get(nid)) !== null && _a !== void 0 ? _a : 0,
|
|
498
|
+
});
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Get downstream dependents with parent info for tree construction.
|
|
503
|
+
* @returns Array of { nodeId, depth, parentId } where parentId is the BFS predecessor
|
|
504
|
+
*/
|
|
505
|
+
getDownstreamWithParents(nodeId, maxDepth) {
|
|
506
|
+
if (!this.graph.hasNode(nodeId)) {
|
|
507
|
+
return [];
|
|
508
|
+
}
|
|
509
|
+
const result = [];
|
|
510
|
+
const visited = new Set();
|
|
511
|
+
const queue = [];
|
|
512
|
+
for (const neighborId of this.graph.outboundNeighbors(nodeId)) {
|
|
513
|
+
if (!visited.has(neighborId)) {
|
|
514
|
+
visited.add(neighborId);
|
|
515
|
+
result.push({ nodeId: neighborId, depth: 1, parentId: nodeId });
|
|
516
|
+
if (maxDepth === undefined || 1 < maxDepth) {
|
|
517
|
+
queue.push({ id: neighborId, depth: 1, parentId: nodeId });
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
while (queue.length > 0) {
|
|
522
|
+
const { id, depth } = queue.shift();
|
|
523
|
+
const nextDepth = depth + 1;
|
|
524
|
+
for (const neighborId of this.graph.outboundNeighbors(id)) {
|
|
525
|
+
if (!visited.has(neighborId)) {
|
|
526
|
+
visited.add(neighborId);
|
|
527
|
+
result.push({ nodeId: neighborId, depth: nextDepth, parentId: id });
|
|
528
|
+
if (maxDepth === undefined || nextDepth < maxDepth) {
|
|
529
|
+
queue.push({ id: neighborId, depth: nextDepth, parentId: id });
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
return result;
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
537
|
+
* Add field nodes from catalog metadata
|
|
538
|
+
*/
|
|
539
|
+
addFieldNodes(catalog) {
|
|
540
|
+
if (catalog.nodes) {
|
|
541
|
+
for (const [uniqueId, node] of Object.entries(catalog.nodes)) {
|
|
542
|
+
if (this.graph.hasNode(uniqueId)) {
|
|
543
|
+
const columns = node
|
|
544
|
+
.columns;
|
|
545
|
+
this.processCatalogColumns(uniqueId, columns);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
if (catalog.sources) {
|
|
550
|
+
for (const [uniqueId, source] of Object.entries(catalog.sources)) {
|
|
551
|
+
if (this.graph.hasNode(uniqueId)) {
|
|
552
|
+
const columns = source
|
|
553
|
+
.columns;
|
|
554
|
+
this.processCatalogColumns(uniqueId, columns);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
processCatalogColumns(parentUniqueId, columns) {
|
|
560
|
+
var _a;
|
|
561
|
+
if (!columns)
|
|
562
|
+
return;
|
|
563
|
+
const parentNode = this.graph.getNodeAttributes(parentUniqueId);
|
|
564
|
+
for (const [colName, colAttr] of Object.entries(columns)) {
|
|
565
|
+
const fieldUniqueId = `${parentUniqueId}#${colName}`;
|
|
566
|
+
const attr = colAttr;
|
|
567
|
+
const description = (_a = attr === null || attr === void 0 ? void 0 : attr.comment) !== null && _a !== void 0 ? _a : attr === null || attr === void 0 ? void 0 : attr.description;
|
|
568
|
+
// Add field node if it doesn't exist
|
|
569
|
+
if (!this.graph.hasNode(fieldUniqueId)) {
|
|
570
|
+
this.graph.addNode(fieldUniqueId, {
|
|
571
|
+
unique_id: fieldUniqueId,
|
|
572
|
+
resource_type: "field",
|
|
573
|
+
name: colName,
|
|
574
|
+
package_name: parentNode.package_name,
|
|
575
|
+
parent_id: parentUniqueId,
|
|
576
|
+
description,
|
|
577
|
+
});
|
|
578
|
+
// Add internal edge from parent to field
|
|
579
|
+
this.graph.addEdge(parentUniqueId, fieldUniqueId, {
|
|
580
|
+
dependency_type: "internal",
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
/**
|
|
586
|
+
* Add field-to-field edges based on SQL analysis
|
|
587
|
+
*/
|
|
588
|
+
addFieldEdges(childNodeId, dependencies) {
|
|
589
|
+
if (!this.graph.hasNode(childNodeId))
|
|
590
|
+
return;
|
|
591
|
+
for (const [targetCol, sourceCols] of Object.entries(dependencies)) {
|
|
592
|
+
const targetFieldId = `${childNodeId}#${targetCol}`;
|
|
593
|
+
// Ensure target field node exists
|
|
594
|
+
this.ensureFieldNode(childNodeId, targetCol);
|
|
595
|
+
for (const source of sourceCols) {
|
|
596
|
+
// Resolve source table name/relation name to unique ID
|
|
597
|
+
const sourceNodeId = this.resolveRelationToUniqueId(source.sourceTable);
|
|
598
|
+
if (!sourceNodeId)
|
|
599
|
+
continue;
|
|
600
|
+
const sourceFieldId = `${sourceNodeId}#${source.sourceColumn}`;
|
|
601
|
+
this.ensureFieldNode(sourceNodeId, source.sourceColumn);
|
|
602
|
+
// Add field edge from source field to target field
|
|
603
|
+
if (!this.graph.hasEdge(sourceFieldId, targetFieldId)) {
|
|
604
|
+
this.graph.addEdge(sourceFieldId, targetFieldId, {
|
|
605
|
+
dependency_type: "field",
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
ensureFieldNode(parentNodeId, colName) {
|
|
612
|
+
const fieldId = `${parentNodeId}#${colName}`;
|
|
613
|
+
if (!this.graph.hasNode(fieldId)) {
|
|
614
|
+
const parentAttr = this.graph.getNodeAttributes(parentNodeId);
|
|
615
|
+
this.graph.addNode(fieldId, {
|
|
616
|
+
unique_id: fieldId,
|
|
617
|
+
resource_type: "field",
|
|
618
|
+
name: colName,
|
|
619
|
+
package_name: parentAttr.package_name,
|
|
620
|
+
parent_id: parentNodeId,
|
|
621
|
+
});
|
|
622
|
+
// Add internal edge
|
|
623
|
+
this.graph.addEdge(parentNodeId, fieldId, {
|
|
624
|
+
dependency_type: "internal",
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
return fieldId;
|
|
628
|
+
}
|
|
629
|
+
resolveRelationToUniqueId(relationName) {
|
|
630
|
+
// Try exact match
|
|
631
|
+
const normalized = relationName.toLowerCase();
|
|
632
|
+
if (this.relationMap.has(normalized)) {
|
|
633
|
+
return this.relationMap.get(normalized);
|
|
634
|
+
}
|
|
635
|
+
// Try stripping quotes if any
|
|
636
|
+
const unquoted = normalized.replace(/["`]/g, "");
|
|
637
|
+
if (this.relationMap.has(unquoted)) {
|
|
638
|
+
return this.relationMap.get(unquoted);
|
|
639
|
+
}
|
|
640
|
+
// Try partial match if it's just the table name (alias)
|
|
641
|
+
// We also strip quotes from the mapped relations for comparison
|
|
642
|
+
for (const [rel, uid] of this.relationMap.entries()) {
|
|
643
|
+
const relUnquoted = rel.replace(/["`]/g, "");
|
|
644
|
+
if (relUnquoted.endsWith(`.${unquoted}`) || relUnquoted === unquoted) {
|
|
645
|
+
return uid;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
return undefined;
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
exports.ManifestGraph = ManifestGraph;
|