@duckcodeailabs/dql-core 1.6.20 → 1.6.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/dashboard-document.d.ts +41 -0
- package/dist/apps/dashboard-document.d.ts.map +1 -1
- package/dist/apps/dashboard-document.js +109 -0
- package/dist/apps/dashboard-document.js.map +1 -1
- package/dist/apps/index.d.ts +1 -1
- package/dist/apps/index.d.ts.map +1 -1
- package/dist/contracts/index.d.ts +1 -1
- package/dist/contracts/index.d.ts.map +1 -1
- package/dist/contracts/index.js.map +1 -1
- package/dist/contracts/registry.d.ts +25 -1
- package/dist/contracts/registry.d.ts.map +1 -1
- package/dist/contracts/registry.js +104 -12
- package/dist/contracts/registry.js.map +1 -1
- package/dist/contracts/types.d.ts +55 -0
- package/dist/contracts/types.d.ts.map +1 -1
- package/dist/contracts/types.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/lineage/builder.d.ts.map +1 -1
- package/dist/lineage/builder.js +12 -0
- package/dist/lineage/builder.js.map +1 -1
- package/dist/lineage/column-lineage.d.ts +27 -0
- package/dist/lineage/column-lineage.d.ts.map +1 -1
- package/dist/lineage/column-lineage.js +147 -0
- package/dist/lineage/column-lineage.js.map +1 -1
- package/dist/lineage/derivation.d.ts +111 -0
- package/dist/lineage/derivation.d.ts.map +1 -0
- package/dist/lineage/derivation.js +177 -0
- package/dist/lineage/derivation.js.map +1 -0
- package/dist/lineage/impact.d.ts +127 -0
- package/dist/lineage/impact.d.ts.map +1 -0
- package/dist/lineage/impact.js +305 -0
- package/dist/lineage/impact.js.map +1 -0
- package/dist/lineage/index.d.ts +3 -1
- package/dist/lineage/index.d.ts.map +1 -1
- package/dist/lineage/index.js +3 -1
- package/dist/lineage/index.js.map +1 -1
- package/dist/manifest/builder.d.ts +16 -0
- package/dist/manifest/builder.d.ts.map +1 -1
- package/dist/manifest/builder.js +141 -11
- package/dist/manifest/builder.js.map +1 -1
- package/dist/manifest/dbt-freshness.d.ts +65 -0
- package/dist/manifest/dbt-freshness.d.ts.map +1 -0
- package/dist/manifest/dbt-freshness.js +264 -0
- package/dist/manifest/dbt-freshness.js.map +1 -0
- package/dist/manifest/domain-writer.d.ts +66 -0
- package/dist/manifest/domain-writer.d.ts.map +1 -0
- package/dist/manifest/domain-writer.js +114 -0
- package/dist/manifest/domain-writer.js.map +1 -0
- package/dist/manifest/index.d.ts +4 -1
- package/dist/manifest/index.d.ts.map +1 -1
- package/dist/manifest/index.js +3 -0
- package/dist/manifest/index.js.map +1 -1
- package/dist/manifest/output-drift.d.ts +31 -0
- package/dist/manifest/output-drift.d.ts.map +1 -0
- package/dist/manifest/output-drift.js +97 -0
- package/dist/manifest/output-drift.js.map +1 -0
- package/dist/manifest/types.d.ts +156 -1
- package/dist/manifest/types.d.ts.map +1 -1
- package/dist/parser/parser.d.ts.map +1 -1
- package/dist/parser/parser.js +8 -0
- package/dist/parser/parser.js.map +1 -1
- package/dist/semantic/analyzer.d.ts +8 -0
- package/dist/semantic/analyzer.d.ts.map +1 -1
- package/dist/semantic/analyzer.js +8 -0
- package/dist/semantic/analyzer.js.map +1 -1
- package/dist/semantic/conflicts.d.ts +33 -0
- package/dist/semantic/conflicts.d.ts.map +1 -0
- package/dist/semantic/conflicts.js +270 -0
- package/dist/semantic/conflicts.js.map +1 -0
- package/dist/semantic/index.d.ts +1 -1
- package/dist/semantic/index.d.ts.map +1 -1
- package/dist/semantic/index.js +1 -1
- package/dist/semantic/index.js.map +1 -1
- package/dist/trust/index.d.ts +3 -0
- package/dist/trust/index.d.ts.map +1 -0
- package/dist/trust/index.js +2 -0
- package/dist/trust/index.js.map +1 -0
- package/dist/trust/labels.d.ts +122 -0
- package/dist/trust/labels.d.ts.map +1 -0
- package/dist/trust/labels.js +185 -0
- package/dist/trust/labels.js.map +1 -0
- package/package.json +5 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lineage/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,oBAAoB,GAC1B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/lineage/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,oBAAoB,EACpB,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,oBAAoB,GAC1B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,KAAK,YAAY,EACjB,KAAK,cAAc,GACpB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,GAChC,MAAM,0BAA0B,CAAC;AAElC,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,gBAAgB,GACtB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,iBAAiB,EACjB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAC1B,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,GAC3B,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,yBAAyB,EACzB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,GACzB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,eAAe,EACf,aAAa,EACb,iBAAiB,EACjB,sBAAsB,EACtB,KAAK,UAAU,EACf,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,UAAU,GAChB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,mBAAmB,EACnB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,wBAAwB,GAC9B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,aAAa,EACb,qBAAqB,EACrB,gBAAgB,EAChB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,UAAU,EACf,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC"}
|
package/dist/lineage/index.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
// Lineage analysis — SQL parsing, dependency resolution, graph building, and domain lineage
|
|
2
2
|
export { analyzeSqlReferences, extractTablesFromSql, } from './sql-parser.js';
|
|
3
|
-
export { extractColumnLineage, } from './column-lineage.js';
|
|
3
|
+
export { extractColumnLineage, extractRefColumnUsage, } from './column-lineage.js';
|
|
4
4
|
export { resolveDependencies, getUpstream, getDownstream, } from './dependency-resolver.js';
|
|
5
5
|
export { LineageGraph, getLayerForNodeType, } from './lineage-graph.js';
|
|
6
6
|
export { buildLineageGraph, } from './builder.js';
|
|
7
7
|
export { queryLineage, queryBusiness360, queryCompleteLineagePaths, } from './query.js';
|
|
8
8
|
export { buildTrustChain, analyzeImpact, detectDomainFlows, getDomainTrustOverview, } from './domain-lineage.js';
|
|
9
|
+
export { buildDerivationWalk, } from './derivation.js';
|
|
10
|
+
export { classifyBlockChange, changedBlocksFromDiff, computeImpact, computeImpactFromDiff, renderImpactText, } from './impact.js';
|
|
9
11
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/lineage/index.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAE5F,OAAO,EACL,oBAAoB,EACpB,oBAAoB,GAIrB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,oBAAoB,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/lineage/index.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAE5F,OAAO,EACL,oBAAoB,EACpB,oBAAoB,GAIrB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,oBAAoB,EACpB,qBAAqB,GAKtB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,mBAAmB,EACnB,WAAW,EACX,aAAa,GAGd,MAAM,0BAA0B,CAAC;AAElC,OAAO,EACL,YAAY,EACZ,mBAAmB,GAOpB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,iBAAiB,GASlB,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,YAAY,EACZ,gBAAgB,EAChB,yBAAyB,GAc1B,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,eAAe,EACf,aAAa,EACb,iBAAiB,EACjB,sBAAsB,GAMvB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,mBAAmB,GAMpB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,aAAa,EACb,qBAAqB,EACrB,gBAAgB,GASjB,MAAM,aAAa,CAAC"}
|
|
@@ -91,6 +91,22 @@ interface ProjectConfig {
|
|
|
91
91
|
/** Path to datalex-manifest.json, absolute or relative to projectRoot */
|
|
92
92
|
manifestPath?: string;
|
|
93
93
|
};
|
|
94
|
+
/**
|
|
95
|
+
* Optional `dql propose` conventions. Refines the convention-agnostic
|
|
96
|
+
* classifier + bounded selection. All fields are optional; defaults apply.
|
|
97
|
+
*/
|
|
98
|
+
propose?: {
|
|
99
|
+
/** Folders/tags that mean business (default ["marts","core","reporting"]). */
|
|
100
|
+
businessLayers?: string[];
|
|
101
|
+
/** Folders/tags that mean plumbing (default ["staging","intermediate","base"]). */
|
|
102
|
+
excludeLayers?: string[];
|
|
103
|
+
/** Max candidates generated per domain (default 8). */
|
|
104
|
+
maxPerDomain?: number;
|
|
105
|
+
/** Minimum demand score a candidate needs to be selected (default 0). */
|
|
106
|
+
minScore?: number;
|
|
107
|
+
/** Optional AI enrichment toggle (default "auto"). */
|
|
108
|
+
aiEnrichment?: 'auto' | 'on' | 'off';
|
|
109
|
+
};
|
|
94
110
|
}
|
|
95
111
|
/**
|
|
96
112
|
* Load `dql.config.json`. Exposed so CLI commands can read the same config
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../src/manifest/builder.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;
|
|
1
|
+
{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../src/manifest/builder.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAwBH,OAAO,KAAK,EACV,WAAW,EAeZ,MAAM,YAAY,CAAC;AAepB;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2CAA2C;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,sDAAsD;IACtD,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,EAAE,CAmDzE;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,WAAW,CAgJxE;AAID,UAAU,aAAa;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B;;;OAGG;IACH,GAAG,CAAC,EAAE;QACJ,0EAA0E;QAC1E,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,6FAA6F;QAC7F,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,iGAAiG;IACjG,OAAO,CAAC,EAAE;QACR,yEAAyE;QACzE,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,OAAO,CAAC,EAAE;QACR,8EAA8E;QAC9E,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B,mFAAmF;QACnF,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB,uDAAuD;QACvD,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,yEAAyE;QACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,sDAAsD;QACtD,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,KAAK,CAAC;KACtC,CAAC;CACH;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,aAAa,CAQpE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,MAAM,GAChB,MAAM,GAAG,IAAI,CAgBf;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CACxC,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,MAAM,EACjB,YAAY,CAAC,EAAE,aAAa,GAC3B,MAAM,GAAG,IAAI,CAef"}
|
package/dist/manifest/builder.js
CHANGED
|
@@ -8,14 +8,16 @@
|
|
|
8
8
|
import { readdirSync, readFileSync, existsSync } from 'node:fs';
|
|
9
9
|
import { join, extname, relative } from 'node:path';
|
|
10
10
|
import { Parser } from '../parser/index.js';
|
|
11
|
-
import { analyze } from '../semantic/index.js';
|
|
11
|
+
import { analyze, detectTrustConflicts } from '../semantic/index.js';
|
|
12
12
|
import { extractTablesFromSql } from '../lineage/sql-parser.js';
|
|
13
13
|
import { extractColumnLineage } from '../lineage/column-lineage.js';
|
|
14
|
+
import { detectOutputDrift } from './output-drift.js';
|
|
14
15
|
import { buildLineageGraph } from '../lineage/builder.js';
|
|
15
16
|
import { detectDomainFlows, getDomainTrustOverview } from '../lineage/domain-lineage.js';
|
|
16
17
|
import { loadSemanticLayerFromDir } from '../semantic/index.js';
|
|
17
18
|
import { DataLexContractRegistry } from '../contracts/index.js';
|
|
18
19
|
import { loadAppDocument, findAppDocuments, findDashboardsForApp, loadDashboardDocument, extractDashboardBlockRefs, appFolderRelPath, } from '../apps/index.js';
|
|
20
|
+
import { loadDbtRunState, applyBlockDataState } from './dbt-freshness.js';
|
|
19
21
|
/**
|
|
20
22
|
* Enumerate every file whose contents could change the manifest output.
|
|
21
23
|
*
|
|
@@ -121,6 +123,14 @@ export function buildManifest(options) {
|
|
|
121
123
|
validateDomains(domains, blocks, businessViews, terms, diagnostics);
|
|
122
124
|
validateBusinessViews(businessViews, blocks, diagnostics);
|
|
123
125
|
validateTermRefs(terms, blocks, businessViews, diagnostics);
|
|
126
|
+
// Cross-artifact trust-conflict detection: two certified terms / blocks that
|
|
127
|
+
// claim the same concept/grain but disagree. Additive `kind: 'conflict'`
|
|
128
|
+
// diagnostics; conservative heuristics avoid false positives.
|
|
129
|
+
diagnostics.push(...detectTrustConflicts(terms, blocks));
|
|
130
|
+
// Output-contract drift detection: a parent block `ref()`s a child and
|
|
131
|
+
// references a column the child no longer outputs. Additive `kind: 'drift'`
|
|
132
|
+
// warnings — never fails the build; freeform composition stays unrestricted.
|
|
133
|
+
diagnostics.push(...detectOutputDrift(blocks));
|
|
124
134
|
// Load semantic layer
|
|
125
135
|
const semanticDir = resolveSemanticPath(projectRoot, config);
|
|
126
136
|
const { metrics, dimensions } = loadSemanticDefinitions(projectRoot, semanticDir);
|
|
@@ -148,11 +158,19 @@ export function buildManifest(options) {
|
|
|
148
158
|
let dbtImport;
|
|
149
159
|
if (options.dbtManifestPath) {
|
|
150
160
|
const filters = options.dbtImportFilters ?? config.dbtImport;
|
|
161
|
+
// Freshness-aware trust — read dbt's run artifacts (run_results.json + source
|
|
162
|
+
// freshness) alongside the manifest, READ-ONLY. Empty/absent → every node is
|
|
163
|
+
// `unknown` and blocks are left untouched (additive, backward compatible).
|
|
164
|
+
const runState = loadDbtRunState(options.dbtManifestPath);
|
|
151
165
|
dbtImport = importDbtManifest(options.dbtManifestPath, sources, referencedTables, {
|
|
152
166
|
maxHops: options.maxDbtHops,
|
|
153
167
|
selectiveThreshold: options.selectiveDbtThreshold ?? 200,
|
|
154
168
|
filters,
|
|
169
|
+
runState,
|
|
155
170
|
});
|
|
171
|
+
// Roll up per-model run state into each certified block's effective
|
|
172
|
+
// `dataState` via its transitive dbt upstreams.
|
|
173
|
+
applyBlockDataState(blocks, sources, dbtImport.dbtDag, runState);
|
|
156
174
|
}
|
|
157
175
|
// Apps & dashboards (consumption layer). Scanned after blocks/notebooks
|
|
158
176
|
// because dashboard refs are resolved against the block path → name map.
|
|
@@ -335,6 +353,7 @@ function scanBlocks(projectRoot, dirs, diagnostics, datalexRegistry) {
|
|
|
335
353
|
const block = stmt;
|
|
336
354
|
if (block.kind !== 'BlockDecl')
|
|
337
355
|
continue;
|
|
356
|
+
const nextBlock = blockDeclToManifestBlock(block, relPath);
|
|
338
357
|
if (blocks[block.name]) {
|
|
339
358
|
diagnostics?.push({
|
|
340
359
|
kind: 'resolve',
|
|
@@ -343,7 +362,9 @@ function scanBlocks(projectRoot, dirs, diagnostics, datalexRegistry) {
|
|
|
343
362
|
message: `duplicate block "${block.name}" also declared in ${blocks[block.name].filePath}`,
|
|
344
363
|
});
|
|
345
364
|
}
|
|
346
|
-
blocks[block.name]
|
|
365
|
+
if (!blocks[block.name] || shouldReplaceDuplicateBlock(blocks[block.name], nextBlock)) {
|
|
366
|
+
blocks[block.name] = nextBlock;
|
|
367
|
+
}
|
|
347
368
|
}
|
|
348
369
|
}
|
|
349
370
|
catch (err) {
|
|
@@ -361,6 +382,26 @@ function scanBlocks(projectRoot, dirs, diagnostics, datalexRegistry) {
|
|
|
361
382
|
}
|
|
362
383
|
return blocks;
|
|
363
384
|
}
|
|
385
|
+
function shouldReplaceDuplicateBlock(existing, next) {
|
|
386
|
+
return duplicateBlockPriority(next) > duplicateBlockPriority(existing);
|
|
387
|
+
}
|
|
388
|
+
function duplicateBlockPriority(block) {
|
|
389
|
+
let score = 0;
|
|
390
|
+
if (block.status === 'certified')
|
|
391
|
+
score += 100;
|
|
392
|
+
else if (block.status === 'review')
|
|
393
|
+
score += 60;
|
|
394
|
+
else if (block.status === 'draft')
|
|
395
|
+
score += 20;
|
|
396
|
+
else
|
|
397
|
+
score += 10;
|
|
398
|
+
const path = block.filePath.replaceAll('\\', '/');
|
|
399
|
+
if (!path.includes('/_drafts/') && !path.startsWith('_drafts/') && !path.startsWith('blocks/_drafts/'))
|
|
400
|
+
score += 10;
|
|
401
|
+
if (path.startsWith('domains/') || path.startsWith('blocks/'))
|
|
402
|
+
score += 5;
|
|
403
|
+
return score;
|
|
404
|
+
}
|
|
364
405
|
// ---- Business View Scanning ----
|
|
365
406
|
function scanBusinessViews(projectRoot, dirs, diagnostics) {
|
|
366
407
|
const views = {};
|
|
@@ -1016,9 +1057,10 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1016
1057
|
const database = node.database;
|
|
1017
1058
|
const dependsOn = Array.isArray(node.depends_on?.nodes) ? node.depends_on.nodes : [];
|
|
1018
1059
|
const lookupKeys = buildLookupKeys(name, schema, database);
|
|
1060
|
+
const primaryLookupKeys = buildPrimaryLookupKeys(name, schema, database);
|
|
1019
1061
|
const tags = Array.isArray(node.tags) ? node.tags : [];
|
|
1020
1062
|
const path = (node.original_file_path ?? node.path);
|
|
1021
|
-
index.set(uniqueId, { uniqueId, name, type: 'model', schema, database, dependsOn, lookupKeys, tags, path, raw: node });
|
|
1063
|
+
index.set(uniqueId, { uniqueId, name, type: 'model', schema, database, dependsOn, lookupKeys, primaryLookupKeys, tags, path, raw: node });
|
|
1022
1064
|
}
|
|
1023
1065
|
const rawSources = manifest.sources ?? {};
|
|
1024
1066
|
for (const [uniqueId, src] of Object.entries(rawSources)) {
|
|
@@ -1028,9 +1070,10 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1028
1070
|
const schema = src.schema;
|
|
1029
1071
|
const database = src.database;
|
|
1030
1072
|
const lookupKeys = buildLookupKeys(name, schema, database);
|
|
1073
|
+
const primaryLookupKeys = buildPrimaryLookupKeys(name, schema, database);
|
|
1031
1074
|
const tags = Array.isArray(src.tags) ? src.tags : [];
|
|
1032
1075
|
const path = (src.original_file_path ?? src.path);
|
|
1033
|
-
index.set(uniqueId, { uniqueId, name, type: 'source', schema, database, dependsOn: [], lookupKeys, tags, path, raw: src });
|
|
1076
|
+
index.set(uniqueId, { uniqueId, name, type: 'source', schema, database, dependsOn: [], lookupKeys, primaryLookupKeys, tags, path, raw: src });
|
|
1034
1077
|
}
|
|
1035
1078
|
const totalDbtModels = [...index.values()].filter((e) => e.type === 'model').length;
|
|
1036
1079
|
const filters = opts.filters;
|
|
@@ -1050,21 +1093,33 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1050
1093
|
// Multiple dbt nodes may share a short alias, so short-name matches are
|
|
1051
1094
|
// used only when they resolve unambiguously.
|
|
1052
1095
|
const nameToIds = new Map();
|
|
1053
|
-
const
|
|
1096
|
+
const primaryNameToIds = new Map();
|
|
1097
|
+
const addTo = (map, key, uniqueId) => {
|
|
1054
1098
|
const normalized = key.toLowerCase();
|
|
1055
|
-
const existing =
|
|
1099
|
+
const existing = map.get(normalized);
|
|
1056
1100
|
if (existing)
|
|
1057
1101
|
existing.add(uniqueId);
|
|
1058
1102
|
else
|
|
1059
|
-
|
|
1103
|
+
map.set(normalized, new Set([uniqueId]));
|
|
1060
1104
|
};
|
|
1061
1105
|
for (const entry of index.values()) {
|
|
1062
|
-
for (const key of entry.lookupKeys)
|
|
1063
|
-
|
|
1064
|
-
|
|
1106
|
+
for (const key of entry.lookupKeys)
|
|
1107
|
+
addTo(nameToIds, key, entry.uniqueId);
|
|
1108
|
+
for (const key of entry.primaryLookupKeys)
|
|
1109
|
+
addTo(primaryNameToIds, key, entry.uniqueId);
|
|
1065
1110
|
}
|
|
1066
1111
|
const resolveDbtAnchor = (ref) => {
|
|
1067
|
-
|
|
1112
|
+
const refKeys = buildReferencedTableLookupKeys(ref);
|
|
1113
|
+
// Prefer an unambiguous match on EXACT model names before falling back to
|
|
1114
|
+
// dbt role-prefix-stripped aliases — otherwise `dev.customers` is ambiguous
|
|
1115
|
+
// between the `customers` mart and the stripped alias of `stg_customers`,
|
|
1116
|
+
// and the anchor (and its whole upstream subgraph) is silently skipped.
|
|
1117
|
+
for (const key of refKeys) {
|
|
1118
|
+
const ids = primaryNameToIds.get(key);
|
|
1119
|
+
if (ids?.size === 1)
|
|
1120
|
+
return [...ids][0];
|
|
1121
|
+
}
|
|
1122
|
+
for (const key of refKeys) {
|
|
1068
1123
|
const ids = nameToIds.get(key);
|
|
1069
1124
|
if (ids?.size === 1)
|
|
1070
1125
|
return [...ids][0];
|
|
@@ -1149,6 +1204,7 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1149
1204
|
type: v.data_type,
|
|
1150
1205
|
}))
|
|
1151
1206
|
: undefined;
|
|
1207
|
+
const modelRunState = opts.runState?.byUniqueId.get(uid);
|
|
1152
1208
|
if (!sources[tableName]) {
|
|
1153
1209
|
sources[tableName] = { name: tableName, origin: 'dbt', referencedBy: [] };
|
|
1154
1210
|
}
|
|
@@ -1164,6 +1220,7 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1164
1220
|
{ name: v.name, description: v.description, type: v.data_type },
|
|
1165
1221
|
]))
|
|
1166
1222
|
: undefined,
|
|
1223
|
+
runState: modelRunState,
|
|
1167
1224
|
};
|
|
1168
1225
|
if (sources[tableName].origin === 'sql')
|
|
1169
1226
|
sources[tableName].origin = 'dbt';
|
|
@@ -1179,6 +1236,7 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1179
1236
|
database: entry.database,
|
|
1180
1237
|
materialized: node.config?.materialized,
|
|
1181
1238
|
description: node.description,
|
|
1239
|
+
runState: modelRunState,
|
|
1182
1240
|
});
|
|
1183
1241
|
for (const dep of selectedDeps) {
|
|
1184
1242
|
dbtDagEdges.push({ source: dep, target: uid });
|
|
@@ -1192,11 +1250,13 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1192
1250
|
if (!sources[tableName]) {
|
|
1193
1251
|
sources[tableName] = { name: tableName, origin: 'dbt', referencedBy: [] };
|
|
1194
1252
|
}
|
|
1253
|
+
const sourceRunState = opts.runState?.byUniqueId.get(uid);
|
|
1195
1254
|
sources[tableName].dbtModel = {
|
|
1196
1255
|
uniqueId: uid,
|
|
1197
1256
|
database: entry.database,
|
|
1198
1257
|
schema: entry.schema,
|
|
1199
1258
|
description: src.description,
|
|
1259
|
+
runState: sourceRunState,
|
|
1200
1260
|
};
|
|
1201
1261
|
dbtDagModels.push({
|
|
1202
1262
|
uniqueId: uid,
|
|
@@ -1206,6 +1266,7 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1206
1266
|
schema: entry.schema,
|
|
1207
1267
|
database: entry.database,
|
|
1208
1268
|
description: src.description,
|
|
1269
|
+
runState: sourceRunState,
|
|
1209
1270
|
});
|
|
1210
1271
|
sourcesImported++;
|
|
1211
1272
|
}
|
|
@@ -1220,9 +1281,23 @@ function importDbtManifest(manifestPath, sources, referencedTables, opts = {}) {
|
|
|
1220
1281
|
maxHops: opts.maxHops,
|
|
1221
1282
|
importedAt: new Date().toISOString(),
|
|
1222
1283
|
dbtDag: { models: dbtDagModels, edges: dbtDagEdges },
|
|
1284
|
+
runResultsPath: opts.runState?.runResultsPath,
|
|
1223
1285
|
};
|
|
1224
1286
|
}
|
|
1225
1287
|
/** Build all normalised lookup keys for a dbt node name. */
|
|
1288
|
+
/**
|
|
1289
|
+
* Lookup keys from a dbt node's EXACT name (no role-prefix stripping), so a
|
|
1290
|
+
* mart `customers` and a staging `stg_customers` never collide on the same key.
|
|
1291
|
+
*/
|
|
1292
|
+
function buildPrimaryLookupKeys(name, schema, database) {
|
|
1293
|
+
const alias = name.toLowerCase();
|
|
1294
|
+
const keys = new Set([alias]);
|
|
1295
|
+
if (schema)
|
|
1296
|
+
keys.add(`${schema}.${alias}`.toLowerCase());
|
|
1297
|
+
if (schema && database)
|
|
1298
|
+
keys.add(`${database}.${schema}.${alias}`.toLowerCase());
|
|
1299
|
+
return [...keys];
|
|
1300
|
+
}
|
|
1226
1301
|
function buildLookupKeys(name, schema, database) {
|
|
1227
1302
|
const aliases = new Set([name.toLowerCase()]);
|
|
1228
1303
|
const stripped = stripDbtRolePrefix(name);
|
|
@@ -1793,8 +1868,63 @@ function blockDeclToManifestBlock(block, filePath) {
|
|
|
1793
1868
|
unresolved: c.unresolved,
|
|
1794
1869
|
}))
|
|
1795
1870
|
: undefined,
|
|
1871
|
+
outputContract: buildOutputContract(Array.isArray(block.outputs) ? block.outputs : undefined, columnLineage),
|
|
1796
1872
|
};
|
|
1797
1873
|
}
|
|
1874
|
+
/**
|
|
1875
|
+
* Build a block's typed `outputContract` — the columns a parent that `ref()`s
|
|
1876
|
+
* this block can rely on. Additive and distinct from `outputs` (column-lineage
|
|
1877
|
+
* shaped). Source of truth, in priority order:
|
|
1878
|
+
*
|
|
1879
|
+
* 1. `declaredOutputs` (reviewer-declared field names) when present. These
|
|
1880
|
+
* are the contract the block author committed to, so they win. Column
|
|
1881
|
+
* lineage is used only to enrich `role`/`type` for matching names.
|
|
1882
|
+
* 2. Otherwise, the resolved column-lineage output columns — but only when
|
|
1883
|
+
* the projection is fully resolved (no `*`/unresolved entries). A star or
|
|
1884
|
+
* unparsed projection means the schema is open, so we emit no contract and
|
|
1885
|
+
* drift detection stays silent for this block (conservative).
|
|
1886
|
+
*
|
|
1887
|
+
* Returns undefined when neither source yields a usable schema, keeping the
|
|
1888
|
+
* field optional and backward-compatible.
|
|
1889
|
+
*/
|
|
1890
|
+
function buildOutputContract(declaredOutputs, columnLineage) {
|
|
1891
|
+
const roleByName = new Map();
|
|
1892
|
+
if (columnLineage?.parsed) {
|
|
1893
|
+
for (const col of columnLineage.columns) {
|
|
1894
|
+
if (col.isAggregate)
|
|
1895
|
+
roleByName.set(col.name.toLowerCase(), 'metric');
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
if (Array.isArray(declaredOutputs) && declaredOutputs.length > 0) {
|
|
1899
|
+
const seen = new Set();
|
|
1900
|
+
const contract = [];
|
|
1901
|
+
for (const raw of declaredOutputs) {
|
|
1902
|
+
const name = typeof raw === 'string' ? raw.trim() : '';
|
|
1903
|
+
if (!name || seen.has(name.toLowerCase()))
|
|
1904
|
+
continue;
|
|
1905
|
+
seen.add(name.toLowerCase());
|
|
1906
|
+
contract.push({ name, role: roleByName.get(name.toLowerCase()) });
|
|
1907
|
+
}
|
|
1908
|
+
return contract.length > 0 ? contract : undefined;
|
|
1909
|
+
}
|
|
1910
|
+
if (!columnLineage?.parsed || columnLineage.columns.length === 0)
|
|
1911
|
+
return undefined;
|
|
1912
|
+
// Only emit a derived contract when every output column is resolved — a star
|
|
1913
|
+
// or unresolved entry means we cannot enumerate the real output schema.
|
|
1914
|
+
if (columnLineage.columns.some((c) => c.unresolved || c.name === '*' || c.name.endsWith('.*'))) {
|
|
1915
|
+
return undefined;
|
|
1916
|
+
}
|
|
1917
|
+
const seen = new Set();
|
|
1918
|
+
const contract = [];
|
|
1919
|
+
for (const col of columnLineage.columns) {
|
|
1920
|
+
const name = col.name?.trim();
|
|
1921
|
+
if (!name || seen.has(name.toLowerCase()))
|
|
1922
|
+
continue;
|
|
1923
|
+
seen.add(name.toLowerCase());
|
|
1924
|
+
contract.push({ name, role: col.isAggregate ? 'metric' : undefined });
|
|
1925
|
+
}
|
|
1926
|
+
return contract.length > 0 ? contract : undefined;
|
|
1927
|
+
}
|
|
1798
1928
|
function domainDeclToManifestDomain(domain, filePath) {
|
|
1799
1929
|
return {
|
|
1800
1930
|
name: domain.name,
|