@duckcodeailabs/dql-core 1.6.33 → 1.7.0

package/dist/manifest/builder.js

@@ -16,8 +16,12 @@ import { buildLineageGraph } from '../lineage/builder.js';
 import { detectDomainFlows, getDomainTrustOverview } from '../lineage/domain-lineage.js';
 import { loadSemanticLayerFromDir } from '../semantic/index.js';
 import { DataLexContractRegistry } from '../contracts/index.js';
-import { loadAppDocument, findAppDocuments, findDashboardsForApp, loadDashboardDocument, extractDashboardBlockRefs, appFolderRelPath, } from '../apps/index.js';
+import { loadAppDocument, findAppDocuments, findDashboardsForApp, loadDashboardDocument, extractDashboardBlockRefs, appFolderRelPath, normalizeProductDomainContext, } from '../apps/index.js';
 import { loadDbtRunState, applyBlockDataState } from './dbt-freshness.js';
+import { blockParameterDefinitions } from '../blocks/parameters.js';
+import { loadDbtFirstModeling, siblingDbtArtifact } from './dbt-first-modeling.js';
+import { domainFolderSlug } from './domain-writer.js';
+import { loadDomainPackageRegistry } from './domain-package-registry.js';
 /**
  * Enumerate every file whose contents could change the manifest output.
  *
@@ -53,7 +57,7 @@ export function collectInputFiles(options) {
         for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dql']))
             files.add(f);
     }
-    const notebookDirs = ['notebooks', 'blocks', 'dashboards', 'workbooks', ...(options.extraNotebookDirs ?? [])];
+    const notebookDirs = ['notebooks', 'domains', 'blocks', 'dashboards', 'workbooks', ...(options.extraNotebookDirs ?? [])];
     for (const dir of notebookDirs) {
         for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dqlnb']))
             files.add(f);
@@ -65,6 +69,16 @@ export function collectInputFiles(options) {
     }
     if (options.dbtManifestPath && existsSync(options.dbtManifestPath)) {
         files.add(options.dbtManifestPath);
+        const catalogPath = siblingDbtArtifact(options.dbtManifestPath, 'catalog.json');
+        const semanticManifestPath = siblingDbtArtifact(options.dbtManifestPath, 'semantic_manifest.json');
+        if (catalogPath)
+            files.add(catalogPath);
+        if (semanticManifestPath)
+            files.add(semanticManifestPath);
+    }
+    if (config.manifestVersion === 3 && config.modeling?.mode === 'dbt-first') {
+        for (const f of scanFilesRecursive(join(projectRoot, 'domains'), ['.yaml', '.yml']))
+            files.add(f);
     }
     // Apps & dashboards. Manifests live at apps/<id>/dql.app.json; dashboards
     // live under apps/<id>/dashboards/*.dqld.
@@ -82,6 +96,14 @@ export function buildManifest(options) {
     // Load project config
     const config = loadProjectConfig(projectRoot);
     const projectName = config.project ?? 'dql-project';
+    const dbtFirstV3 = config.manifestVersion === 3 && config.modeling?.mode === 'dbt-first';
+    if ((config.manifestVersion === 3 || config.modeling?.mode === 'dbt-first') && !dbtFirstV3) {
+        diagnostics.push({
+            kind: 'config',
+            severity: 'error',
+            message: 'manifest v3 requires both `manifestVersion: 3` and `modeling.mode: "dbt-first"`; compiling with manifest v2 compatibility defaults',
+        });
+    }
     const datalexManifestPath = resolveDataLexManifestPath(projectRoot, options.datalexManifestPath, config);
     const datalexRegistry = datalexManifestPath
         ? new DataLexContractRegistry({ manifestPath: datalexManifestPath })
@@ -96,7 +118,12 @@ export function buildManifest(options) {
     }
     // Scan first-class business domains
     const domainDirs = ['domains', 'blocks', 'terms', 'business-views', ...(options.extraBlockDirs ?? [])];
-    const domains = scanDomains(projectRoot, domainDirs, diagnostics);
+    const packageRegistry = dbtFirstV3 ? loadDomainPackageRegistry(projectRoot) : undefined;
+    const domains = packageRegistry
+        ? Object.fromEntries(packageRegistry.values().map((pkg) => [pkg.id, pkg.domain]))
+        : scanDomains(projectRoot, domainDirs, diagnostics);
+    if (!packageRegistry)
+        validateDomainHierarchy(domains, diagnostics);
     // Scan blocks
     const blockDirs = ['blocks', 'domains', 'dashboards', 'workbooks', ...(options.extraBlockDirs ?? [])];
     const blocks = scanBlocks(projectRoot, blockDirs, diagnostics, datalexRegistry);
@@ -172,15 +199,33 @@ export function buildManifest(options) {
         // `dataState` via its transitive dbt upstreams.
         applyBlockDataState(blocks, sources, dbtImport.dbtDag, runState);
     }
+    let dbtFirstModeling;
+    if (dbtFirstV3) {
+        if (!options.dbtManifestPath || !existsSync(options.dbtManifestPath)) {
+            diagnostics.push({
+                kind: 'config',
+                severity: 'error',
+                message: 'manifest v3 dbt-first modeling requires a readable dbt manifest.json',
+            });
+        }
+        else {
+            dbtFirstModeling = loadDbtFirstModeling(projectRoot, options.dbtManifestPath);
+            diagnostics.push(...dbtFirstModeling.diagnostics);
+        }
+    }
     // Apps & dashboards (consumption layer). Scanned after blocks/notebooks
     // because dashboard refs are resolved against the block path → name map.
     const { apps, dashboards } = scanAppsAndDashboards(projectRoot, blocks, diagnostics);
     // Build lineage
-    const lineage = buildManifestLineage(domains, blocks, metrics, dimensions, notebooks, dbtImport, apps, dashboards, businessViews, terms);
+    const lineage = buildManifestLineage(domains, blocks, metrics, dimensions, notebooks, dbtFirstV3 ? stripDbtImportDetails(dbtImport) : dbtImport, apps, dashboards, businessViews, terms);
+    if (dbtFirstModeling)
+        appendDbtFirstModelingLineage(lineage, dbtFirstModeling.modeling);
     return {
-        manifestVersion: 2,
+        manifestVersion: dbtFirstV3 ? 3 : 2,
         dqlVersion,
-        generatedAt: new Date().toISOString(),
+        // v3 is a reproducible policy artifact. Runtime build time lives in the
+        // local cache, not in the canonical manifest content.
+        generatedAt: dbtFirstV3 ? new Date(0).toISOString() : new Date().toISOString(),
         project: projectName,
         projectRoot,
         domains,
@@ -190,11 +235,13 @@ export function buildManifest(options) {
         notebooks,
         metrics,
         dimensions,
-        sources,
+        sources: dbtFirstV3 ? stripDbtOwnedSourceDetails(sources) : sources,
         apps: Object.keys(apps).length > 0 ? apps : undefined,
         dashboards: Object.keys(dashboards).length > 0 ? dashboards : undefined,
         lineage,
-        dbtImport,
+        dbtImport: dbtFirstV3 ? undefined : dbtImport,
+        dbtProvenance: dbtFirstModeling?.provenance,
+        modeling: dbtFirstModeling?.modeling,
         diagnostics,
     };
 }
@@ -264,6 +311,75 @@ export function resolveDataLexManifestPath(projectRoot, explicit, loadedConfig)
 function isAbsPath(p) {
     return p.startsWith('/') || /^[A-Za-z]:[\\/]/.test(p);
 }
+/** Remove physical dbt payloads before serializing a v3 lineage graph. */
+function stripDbtImportDetails(dbtImport) {
+    if (!dbtImport?.dbtDag)
+        return dbtImport;
+    return {
+        ...dbtImport,
+        dbtDag: {
+            edges: dbtImport.dbtDag.edges,
+            models: dbtImport.dbtDag.models.map(({ columns: _columns, description: _description, ...model }) => model),
+        },
+    };
+}
+/** Keep source names/usage but never serialize dbt-owned schema content in v3. */
+function stripDbtOwnedSourceDetails(sources) {
+    return Object.fromEntries(Object.entries(sources).map(([name, source]) => {
+        const { dbtModel: _dbtModel, ...sparseSource } = source;
+        return [name, sparseSource];
+    }));
+}
+/**
+ * Add DQL-owned analytical bindings and relationship proof to lineage without
+ * mutating or reclassifying dbt transformation edges. A relationship edge is
+ * intentionally distinct from dbt `depends_on`: it is a reviewed analytical
+ * join policy, not inferred DAG evidence.
+ */
+function appendDbtFirstModelingLineage(lineage, modeling) {
+    const existingNodeIds = new Set(lineage.nodes.map((node) => node.id));
+    const existingEdges = new Set(lineage.edges.map((edge) => `${edge.source}:${edge.target}:${edge.type}`));
+    const dbtNodeByUniqueId = new Map();
+    for (const node of lineage.nodes) {
+        const uniqueId = typeof node.metadata?.uniqueId === 'string' ? node.metadata.uniqueId : undefined;
+        if (uniqueId)
+            dbtNodeByUniqueId.set(uniqueId, node.id);
+    }
+    const addEdge = (source, target, type) => {
+        const key = `${source}:${target}:${type}`;
+        if (!existingEdges.has(key)) {
+            lineage.edges.push({ source, target, type });
+            existingEdges.add(key);
+        }
+    };
+    for (const entity of Object.values(modeling.entities)) {
+        const entityId = `dql_entity:${entity.id}`;
+        if (!existingNodeIds.has(entityId)) {
+            lineage.nodes.push({
+                id: entityId,
+                type: 'dql_entity',
+                name: entity.id,
+                layer: 'answer',
+                domain: entity.domain,
+                filePath: entity.sourcePath,
+                metadata: {
+                    dbtUniqueId: entity.dbtUniqueId,
+                    grain: entity.grain,
+                    keys: entity.keys,
+                    identityFingerprint: entity.identityFingerprint,
+                    provenance: 'dql sparse overlay',
+                },
+            });
+            existingNodeIds.add(entityId);
+        }
+        const dbtNode = dbtNodeByUniqueId.get(entity.dbtUniqueId);
+        if (dbtNode)
+            addEdge(dbtNode, entityId, 'binds_dbt_model');
+    }
+    for (const relationship of Object.values(modeling.relationships)) {
+        addEdge(`dql_entity:${relationship.from}`, `dql_entity:${relationship.to}`, 'governed_relationship');
+    }
+}
 function resolveSemanticPath(projectRoot, config) {
     const customPath = config.semanticLayer?.path;
     if (customPath)
@@ -337,6 +453,41 @@ function scanDomains(projectRoot, dirs, diagnostics) {
     }
     return domains;
 }
+/** Validate parent references without changing legacy flat-domain semantics. */
+function validateDomainHierarchy(domains, diagnostics) {
+    const keyFor = (value) => domainAliasKey(value);
+    const byKey = new Map(Object.values(domains).map((domain) => [keyFor(domain.name), domain]));
+    for (const domain of Object.values(domains)) {
+        if (!domain.parent)
+            continue;
+        const parent = byKey.get(keyFor(domain.parent));
+        if (!parent) {
+            diagnostics.push({
+                kind: 'resolve',
+                filePath: domain.filePath,
+                severity: 'error',
+                message: `domain "${domain.name}" references missing parent "${domain.parent}"`,
+            });
+            continue;
+        }
+        const visited = new Set([keyFor(domain.name)]);
+        let current = parent;
+        while (current?.parent) {
+            const currentKey = keyFor(current.name);
+            if (visited.has(currentKey)) {
+                diagnostics.push({
+                    kind: 'resolve',
+                    filePath: domain.filePath,
+                    severity: 'error',
+                    message: `domain hierarchy contains a cycle involving "${domain.name}"`,
+                });
+                break;
+            }
+            visited.add(currentKey);
+            current = byKey.get(keyFor(current.parent));
+        }
+    }
+}
 function scanBlocks(projectRoot, dirs, diagnostics, datalexRegistry) {
     const blocks = {};
     for (const dir of dirs) {
@@ -846,9 +997,11 @@ function scanNotebooks(projectRoot, dirs, diagnostics, datalexRegistry) {
                     });
                 }
                 if (cells.length > 0) {
+                    const productContext = normalizeProductDomainContext(doc.metadata ?? {}, stringOrUndefined(doc.metadata?.domain));
                     notebooks[relPath] = {
                         title: doc.metadata?.title ?? doc.title ?? relPath,
                         filePath: relPath,
+                        ...productContext,
                         cells,
                     };
                 }
@@ -866,6 +1019,15 @@ function scanNotebooks(projectRoot, dirs, diagnostics, datalexRegistry) {
     }
     return notebooks;
 }
+function stringOrUndefined(value) {
+    return typeof value === 'string' && value.trim() ? value.trim() : undefined;
+}
+function stringArrayValue(value) {
+    if (!Array.isArray(value))
+        return undefined;
+    const items = [...new Set(value.filter((item) => typeof item === 'string').map((item) => item.trim()).filter(Boolean))];
+    return items.length > 0 ? items : undefined;
+}
 /** Extract blocks declared inside notebook DQL cells into the blocks map. */
 function extractNotebookBlocks(notebooks) {
     const blocks = {};
@@ -1505,6 +1667,11 @@ function appDocumentToManifest(app, filePath, dashboardIds) {
         businessRules: app.businessRules,
         caveats: app.caveats,
         domain: app.domain,
+        ownerDomain: app.ownerDomain ?? app.domain,
+        usesDomains: app.usesDomains,
+        purpose: app.purpose,
+        requiredExports: app.requiredExports,
+        classification: app.classification,
         subdomain: app.subdomain,
         groups: app.groups ?? [],
         audience: app.audience,
@@ -1860,6 +2027,7 @@ function blockDeclToManifestBlock(block, filePath) {
                 policy: entry.policy,
             }))
             : undefined,
+        parameters: blockParameterDefinitions(block),
         filterBindings: Array.isArray(block.filterBindings)
             ? block.filterBindings.map((entry) => ({
                 filter: entry.filter,
@@ -1948,8 +2116,10 @@ function buildOutputContract(declaredOutputs, columnLineage) {
 }
 function domainDeclToManifestDomain(domain, filePath) {
     return {
+        id: typeof domain.id === 'string' ? domain.id : domainFolderSlug(domain.name),
         name: domain.name,
         filePath,
+        parent: typeof domain.parent === 'string' ? domain.parent : undefined,
         owner: typeof domain.owner === 'string' ? domain.owner : undefined,
         businessOwner: typeof domain.businessOwner === 'string' ? domain.businessOwner : undefined,
         boundedContext: typeof domain.boundedContext === 'string' ? domain.boundedContext : undefined,
@@ -1959,6 +2129,14 @@ function domainDeclToManifestDomain(domain, filePath) {
         tags: Array.isArray(domain.tags) ? domain.tags : undefined,
         businessOutcome: typeof domain.businessOutcome === 'string' ? domain.businessOutcome : undefined,
         description: typeof domain.description === 'string' ? domain.description : undefined,
+        inScope: Array.isArray(domain.inScope) ? domain.inScope : undefined,
+        outOfScope: Array.isArray(domain.outOfScope) ? domain.outOfScope : undefined,
+        dbtGroups: Array.isArray(domain.dbtGroups) ? domain.dbtGroups : undefined,
+        dbtPaths: Array.isArray(domain.dbtPaths) ? domain.dbtPaths : undefined,
+        dbtTags: Array.isArray(domain.dbtTags) ? domain.dbtTags : undefined,
+        semanticDomains: Array.isArray(domain.semanticDomains) ? domain.semanticDomains : undefined,
+        semanticTags: Array.isArray(domain.semanticTags) ? domain.semanticTags : undefined,
+        exports: Array.isArray(domain.exports) ? domain.exports : undefined,
     };
 }
 function termDeclToManifestTerm(term, filePath) {