@dataparade/cli 0.0.1

package/dist/src/classifier/postprocessing.js

@@ -0,0 +1,510 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mergeProperties = mergeProperties;
+exports.dedupeSourceLocations = dedupeSourceLocations;
+exports.compareSourceLocations = compareSourceLocations;
+exports.compareDetectedFromRefs = compareDetectedFromRefs;
+exports.getDatabaseCanonicalKey = getDatabaseCanonicalKey;
+exports.dedupeComponents = dedupeComponents;
+exports.mergeGlobalIdentityProviderThirdParties = mergeGlobalIdentityProviderThirdParties;
+exports.mergeDatabaseAssetsByType = mergeDatabaseAssetsByType;
+exports.compactAuthServiceComponents = compactAuthServiceComponents;
+const config_1 = require("./config");
+const naming_1 = require("./naming");
+const sectioning_1 = require("./sectioning");
+function aggregateComponentConfidence(components, strategy) {
+    if (components.length === 0)
+        return 0;
+    if (strategy === "average") {
+        const sum = components.reduce((acc, c) => acc + c.confidence, 0);
+        return Math.min(1, Math.max(0, sum / components.length));
+    }
+    const max = components.reduce((acc, c) => (c.confidence > acc ? c.confidence : acc), 0);
+    return Math.min(1, Math.max(0, max));
+}
+function mergeProperties(target, source) {
+    for (const [key, value] of Object.entries(source)) {
+        if (!(key in target)) {
+            target[key] = value;
+            continue;
+        }
+        const existing = target[key];
+        if (existing === value) {
+            continue;
+        }
+        const existingArray = Array.isArray(existing) ? existing : [existing];
+        const incomingArray = Array.isArray(value) ? value : [value];
+        const merged = [...existingArray];
+        for (const v of incomingArray) {
+            if (!merged.includes(v)) {
+                merged.push(v);
+            }
+        }
+        target[key] = merged;
+    }
+}
+function dedupeSourceLocations(locations) {
+    const seen = new Set();
+    const result = [];
+    for (const loc of locations) {
+        const key = `${loc.filePath}:${loc.startLine}:${loc.endLine}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        result.push(loc);
+    }
+    return result;
+}
+function compareSourceLocations(a, b) {
+    const fileCmp = a.filePath.localeCompare(b.filePath);
+    if (fileCmp !== 0)
+        return fileCmp;
+    if (a.startLine !== b.startLine)
+        return a.startLine - b.startLine;
+    if (a.endLine !== b.endLine)
+        return a.endLine - b.endLine;
+    return 0;
+}
+function compareDetectedFromRefs(a, b) {
+    const patternCmp = String(a.pattern).localeCompare(String(b.pattern));
+    if (patternCmp !== 0)
+        return patternCmp;
+    const al = a.sourceLocation;
+    const bl = b.sourceLocation;
+    if (!al && !bl)
+        return 0;
+    if (!al)
+        return 1;
+    if (!bl)
+        return -1;
+    return compareSourceLocations(al, bl);
+}
+function compareComponentsForDeterministicBase(a, b) {
+    if (a.confidence !== b.confidence)
+        return b.confidence - a.confidence;
+    const nameCmp = a.name.localeCompare(b.name);
+    if (nameCmp !== 0)
+        return nameCmp;
+    const typeCmp = a.type.localeCompare(b.type);
+    if (typeCmp !== 0)
+        return typeCmp;
+    return a.id.localeCompare(b.id);
+}
+function buildComponentDedupeKey(component, nameNormalization) {
+    const sectionId = (0, sectioning_1.getSectionIdFromProperties)(component.properties);
+    if (component.type === "asset") {
+        const tfAddr = component.properties?.terraform_address;
+        if (typeof tfAddr === "string" &&
+            tfAddr.trim() &&
+            !tfAddr.trim().startsWith("provider.")) {
+            return `asset:tf:${sectionId}:${tfAddr.trim().toLowerCase()}`;
+        }
+    }
+    if (component.type === "asset" && component.subType === "auth_service") {
+        return `asset:auth_service:${sectionId}`;
+    }
+    if (component.type === "asset" && component.subType === "database") {
+        return `asset:database:${sectionId}:${getDatabaseCanonicalKey(component)}`;
+    }
+    if (component.type === "third_party") {
+        const serviceName = component.properties.serviceName;
+        if (typeof serviceName === "string" && serviceName.trim()) {
+            const key = serviceName.trim().toLowerCase();
+            return `third_party:${sectionId}:${key}`;
+        }
+        const normalized = (0, naming_1.normalizeComponentName)(component.name, nameNormalization);
+        const keyName = normalized || component.name.trim().toLowerCase() || "<unknown>";
+        return `third_party:${sectionId}:${keyName}`;
+    }
+    const normalized = (0, naming_1.normalizeComponentName)(component.name, nameNormalization);
+    const keyName = normalized || component.name.trim().toLowerCase() || "<unknown>";
+    return `${component.type}:${sectionId}:${keyName}`;
+}
+/** Map client/database identifiers to a canonical key for merging same logical DB per project. */
+function getDatabaseCanonicalKey(component) {
+    const dbType = component.properties.databaseType;
+    if (typeof dbType === "string" && dbType.trim()) {
+        return dbType.trim().toLowerCase();
+    }
+    const client = component.properties.client;
+    if (typeof client === "string" && client.trim()) {
+        const c = client.trim().toLowerCase();
+        if (c === "supabase" || c === "pg" || c === "postgres")
+            return "postgres";
+        if (c === "redis")
+            return "redis";
+        if (c === "mongo" || c === "mongoose")
+            return "mongo";
+        if (c === "mysql" || c === "mysql2")
+            return "mysql";
+        return c;
+    }
+    const fallbackName = (component.name || "")
+        .trim()
+        .toLowerCase()
+        .replace(/[\s_-]+/g, " ");
+    return fallbackName || component.id;
+}
+function dedupeComponents(components) {
+    if (!components || components.length === 0) {
+        return [];
+    }
+    const config = (0, config_1.loadClassifierConfig)();
+    const groups = new Map();
+    for (const component of components) {
+        const key = buildComponentDedupeKey(component, config.nameNormalization);
+        const list = groups.get(key);
+        if (list) {
+            list.push(component);
+        }
+        else {
+            groups.set(key, [component]);
+        }
+    }
+    const deduped = [];
+    for (const [, group] of groups) {
+        const groupSorted = [...group].sort(compareComponentsForDeterministicBase);
+        const base = groupSorted[0];
+        let subType = base.subType;
+        if (!subType) {
+            subType = groupSorted.find((c) => c.subType)?.subType;
+        }
+        const confidence = aggregateComponentConfidence(groupSorted, config.resolution.confidenceAggregation);
+        const allLocations = [];
+        for (const c of groupSorted) {
+            allLocations.push(...c.sourceLocations);
+        }
+        const sourceLocations = dedupeSourceLocations(allLocations).sort(compareSourceLocations);
+        const detectedFromMap = new Map();
+        for (const c of groupSorted) {
+            for (const ref of c.detectedFrom) {
+                const loc = ref.sourceLocation;
+                const locKey = loc != null
+                    ? `${ref.pattern}:${loc.filePath}:${loc.startLine}:${loc.endLine}`
+                    : `${ref.pattern}:<no-location>`;
+                if (!detectedFromMap.has(locKey)) {
+                    detectedFromMap.set(locKey, ref);
+                }
+            }
+        }
+        const detectedFrom = Array.from(detectedFromMap.values()).sort(compareDetectedFromRefs);
+        const mergedProperties = {};
+        for (const c of groupSorted) {
+            mergeProperties(mergedProperties, c.properties);
+        }
+        deduped.push({
+            ...base,
+            subType,
+            confidence,
+            detectedFrom,
+            sourceLocations,
+            properties: mergedProperties,
+        });
+    }
+    deduped.sort((a, b) => {
+        const nameCompare = a.name.localeCompare(b.name);
+        if (nameCompare !== 0)
+            return nameCompare;
+        return a.type.localeCompare(b.type);
+    });
+    return deduped;
+}
+/**
+ * Identity providers are external SaaS: the same vendor (e.g. Auth0) detected from
+ * backend guards and frontend routes should be one third_party node so data-flow
+ * detection targets a single id. Per-section dedupe keys intentionally keep other
+ * third_party vendors separate.
+ */
+function getGlobalIdentityProviderMergeKey(component) {
+    if (component.type !== "third_party")
+        return undefined;
+    const svcRaw = component.properties?.serviceName;
+    const svc = typeof svcRaw === "string" ? svcRaw.trim().toLowerCase() : "";
+    const nm = (component.name || "").trim().toLowerCase();
+    const hay = `${svc} ${nm}`;
+    if (hay.includes("auth0"))
+        return "idp:auth0";
+    if (hay.includes("okta"))
+        return "idp:okta";
+    if (hay.includes("clerk"))
+        return "idp:clerk";
+    if (hay.includes("keycloak"))
+        return "idp:keycloak";
+    if (hay.includes("cognito"))
+        return "idp:cognito";
+    if (hay.includes("stytch"))
+        return "idp:stytch";
+    if (hay.includes("azure") && (hay.includes("ad") || hay.includes("b2c"))) {
+        return "idp:azure_ad";
+    }
+    return undefined;
+}
+/** Prefer merge base with repo paths under `backend/` so one IdP node stays section-scoped to API/auth code, not only the SPA. */
+function backendEvidencePathScore(component) {
+    const pathMatchesBackend = (filePath) => /(^|[\\/])backend([\\/]|$)/.test(filePath);
+    let n = 0;
+    for (const loc of component.sourceLocations) {
+        if (pathMatchesBackend(loc.filePath))
+            n += 1;
+    }
+    for (const ref of component.detectedFrom) {
+        const fp = ref.sourceLocation?.filePath;
+        if (typeof fp === "string" && pathMatchesBackend(fp))
+            n += 1;
+    }
+    return n;
+}
+function compareIdentityProviderMergeBase(a, b) {
+    const scoreDelta = backendEvidencePathScore(b) - backendEvidencePathScore(a);
+    if (scoreDelta !== 0)
+        return scoreDelta;
+    return compareComponentsForDeterministicBase(a, b);
+}
+function mergeGlobalIdentityProviderThirdParties(components) {
+    if (!components?.length)
+        return [];
+    const config = (0, config_1.loadClassifierConfig)();
+    const groups = new Map();
+    for (const c of components) {
+        const key = getGlobalIdentityProviderMergeKey(c);
+        if (!key)
+            continue;
+        const list = groups.get(key);
+        if (list)
+            list.push(c);
+        else
+            groups.set(key, [c]);
+    }
+    const removeIds = new Set();
+    const mergedByCanonicalId = new Map();
+    for (const [, group] of groups) {
+        if (group.length < 2)
+            continue;
+        const groupSorted = [...group].sort(compareIdentityProviderMergeBase);
+        const base = groupSorted[0];
+        let subType = base.subType;
+        if (!subType) {
+            subType = groupSorted.find((c) => c.subType)?.subType;
+        }
+        const confidence = aggregateComponentConfidence(groupSorted, config.resolution.confidenceAggregation);
+        const allLocations = [];
+        for (const c of groupSorted) {
+            allLocations.push(...c.sourceLocations);
+        }
+        const sourceLocations = dedupeSourceLocations(allLocations).sort(compareSourceLocations);
+        const detectedFromMap = new Map();
+        for (const c of groupSorted) {
+            for (const ref of c.detectedFrom) {
+                const loc = ref.sourceLocation;
+                const locKey = loc != null
+                    ? `${ref.pattern}:${loc.filePath}:${loc.startLine}:${loc.endLine}`
+                    : `${ref.pattern}:<no-location>`;
+                if (!detectedFromMap.has(locKey)) {
+                    detectedFromMap.set(locKey, ref);
+                }
+            }
+        }
+        const detectedFrom = Array.from(detectedFromMap.values()).sort(compareDetectedFromRefs);
+        const mergedProperties = {};
+        for (const c of groupSorted) {
+            mergeProperties(mergedProperties, c.properties);
+        }
+        // Keep a single section scope for tooling that expects string section_id.
+        mergedProperties.section_id = base.properties.section_id;
+        mergedProperties.section_label = base.properties.section_label;
+        mergedProperties.section_role = base.properties.section_role;
+        const merged = {
+            ...base,
+            subType,
+            confidence,
+            detectedFrom,
+            sourceLocations,
+            properties: mergedProperties,
+        };
+        mergedByCanonicalId.set(base.id, merged);
+        for (let i = 1; i < groupSorted.length; i++) {
+            removeIds.add(groupSorted[i].id);
+        }
+    }
+    const out = [];
+    for (const c of components) {
+        if (removeIds.has(c.id))
+            continue;
+        out.push(mergedByCanonicalId.get(c.id) ?? c);
+    }
+    out.sort((a, b) => {
+        const nameCompare = a.name.localeCompare(b.name);
+        if (nameCompare !== 0)
+            return nameCompare;
+        return a.type.localeCompare(b.type);
+    });
+    return out;
+}
+function mergeDatabaseAssetsByType(components) {
+    if (!components?.length)
+        return [];
+    const config = (0, config_1.loadClassifierConfig)();
+    const databaseComponents = [];
+    const other = [];
+    for (const c of components) {
+        if (c.type === "asset" && c.subType === "database") {
+            databaseComponents.push(c);
+        }
+        else {
+            other.push(c);
+        }
+    }
+    if (databaseComponents.length <= 1) {
+        return components;
+    }
+    const byKey = new Map();
+    for (const c of databaseComponents) {
+        const sectionId = (0, sectioning_1.getSectionIdFromProperties)(c.properties);
+        const key = `${sectionId}::${getDatabaseCanonicalKey(c)}`;
+        const list = byKey.get(key);
+        if (list)
+            list.push(c);
+        else
+            byKey.set(key, [c]);
+    }
+    const merged = [];
+    const byKeyEntries = Array.from(byKey.entries()).sort(([a], [b]) => a.localeCompare(b));
+    for (const [, group] of byKeyEntries) {
+        if (group.length === 1) {
+            merged.push(group[0]);
+            continue;
+        }
+        const groupSorted = [...group].sort(compareComponentsForDeterministicBase);
+        const base = groupSorted[0];
+        let subType = base.subType;
+        if (!subType) {
+            subType = groupSorted.find((c) => c.subType)?.subType;
+        }
+        const confidence = aggregateComponentConfidence(groupSorted, config.resolution.confidenceAggregation);
+        const allLocations = [];
+        for (const c of groupSorted)
+            allLocations.push(...c.sourceLocations);
+        const sourceLocations = dedupeSourceLocations(allLocations).sort(compareSourceLocations);
+        const detectedFromMap = new Map();
+        for (const c of groupSorted) {
+            for (const ref of c.detectedFrom) {
+                const loc = ref.sourceLocation;
+                const locKey = loc != null
+                    ? `${ref.pattern}:${loc.filePath}:${loc.startLine}:${loc.endLine}`
+                    : `${ref.pattern}:<no-location>`;
+                if (!detectedFromMap.has(locKey))
+                    detectedFromMap.set(locKey, ref);
+            }
+        }
+        const detectedFrom = Array.from(detectedFromMap.values()).sort(compareDetectedFromRefs);
+        const mergedProperties = {};
+        for (const c of groupSorted)
+            mergeProperties(mergedProperties, c.properties);
+        const preferredName = base.properties.databaseType ?? base.properties.client ?? base.name;
+        const displayName = typeof preferredName === "string"
+            ? preferredName.trim()
+            : base.name;
+        const name = displayName.length > 0
+            ? (0, naming_1.toDisplayName)(displayName.toLowerCase(), displayName)
+            : base.name;
+        merged.push({
+            ...base,
+            name,
+            subType,
+            confidence,
+            detectedFrom,
+            sourceLocations,
+            properties: mergedProperties,
+        });
+    }
+    merged.sort((a, b) => a.name.localeCompare(b.name));
+    return [...other, ...merged].sort((a, b) => {
+        const nameCompare = a.name.localeCompare(b.name);
+        if (nameCompare !== 0)
+            return nameCompare;
+        return a.type.localeCompare(b.type);
+    });
+}
+function isAuthProviderThirdParty(component) {
+    if (component.type !== "third_party")
+        return false;
+    const serviceName = component.properties?.serviceName;
+    const normalizedService = typeof serviceName === "string" ? serviceName.trim().toLowerCase() : "";
+    const normalizedName = (component.name || "").trim().toLowerCase();
+    return normalizedService.includes("auth0") || normalizedName.includes("auth0");
+}
+function mergeAuthServiceIntoTarget(authService, target) {
+    const mergedProperties = {};
+    mergeProperties(mergedProperties, target.properties);
+    mergeProperties(mergedProperties, authService.properties);
+    const confidence = Math.max(target.confidence, authService.confidence);
+    const sourceLocations = dedupeSourceLocations([
+        ...target.sourceLocations,
+        ...authService.sourceLocations,
+    ]).sort(compareSourceLocations);
+    const detectedFromMap = new Map();
+    for (const ref of [...target.detectedFrom, ...authService.detectedFrom]) {
+        const loc = ref.sourceLocation;
+        const locKey = loc != null
+            ? `${ref.pattern}:${loc.filePath}:${loc.startLine}:${loc.endLine}`
+            : `${ref.pattern}:<no-location>`;
+        if (!detectedFromMap.has(locKey))
+            detectedFromMap.set(locKey, ref);
+    }
+    return {
+        ...target,
+        confidence,
+        sourceLocations,
+        detectedFrom: Array.from(detectedFromMap.values()).sort(compareDetectedFromRefs),
+        properties: mergedProperties,
+    };
+}
+function isTerraformScannedResourceAsset(component) {
+    const addr = component.properties?.terraform_address;
+    return (typeof addr === "string" &&
+        Boolean(addr.trim()) &&
+        !addr.trim().startsWith("provider."));
+}
+function compactAuthServiceComponents(components) {
+    if (!components?.length)
+        return [];
+    const bySection = new Map();
+    for (const c of components) {
+        const sid = (0, sectioning_1.getSectionIdFromProperties)(c.properties);
+        const list = bySection.get(sid);
+        if (list)
+            list.push(c);
+        else
+            bySection.set(sid, [c]);
+    }
+    const replacementById = new Map();
+    const removeIds = new Set();
+    for (const [, sectionComponents] of bySection) {
+        const authServices = sectionComponents.filter((c) => c.type === "asset" &&
+            c.subType === "auth_service" &&
+            !isTerraformScannedResourceAsset(c));
+        if (authServices.length === 0)
+            continue;
+        const authThirdParty = sectionComponents.find(isAuthProviderThirdParty);
+        const mainApp = sectionComponents.find((c) => c.type === "asset" &&
+            (c.properties?.isMainApplication === true ||
+                c.properties?.isMainApplication === "true"));
+        const fallbackAsset = sectionComponents.find((c) => c.type === "asset");
+        const mergeTarget = authThirdParty ?? mainApp ?? fallbackAsset;
+        if (!mergeTarget)
+            continue;
+        let nextTarget = replacementById.get(mergeTarget.id) ?? mergeTarget;
+        for (const authService of authServices) {
+            nextTarget = mergeAuthServiceIntoTarget(authService, nextTarget);
+            removeIds.add(authService.id);
+        }
+        replacementById.set(mergeTarget.id, nextTarget);
+    }
+    const out = [];
+    for (const c of components) {
+        if (removeIds.has(c.id))
+            continue;
+        out.push(replacementById.get(c.id) ?? c);
+    }
+    return out;
+}

package/dist/src/classifier/sectioning.d.ts

@@ -0,0 +1,5 @@
+import type { DetectedComponent } from "../core/types/component";
+export declare function getSectionIdFromProperties(properties: Record<string, unknown> | undefined): string;
+export declare function pickDefaultSectionId(components: DetectedComponent[]): string;
+export declare function pickSectionRoleForSectionId(components: DetectedComponent[], sectionId: string): string;
+export declare function getSectionLabelFromProperties(properties: Record<string, unknown> | undefined): string | undefined;

package/dist/src/classifier/sectioning.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSectionIdFromProperties = getSectionIdFromProperties;
+exports.pickDefaultSectionId = pickDefaultSectionId;
+exports.pickSectionRoleForSectionId = pickSectionRoleForSectionId;
+exports.getSectionLabelFromProperties = getSectionLabelFromProperties;
+function getSectionIdFromProperties(properties) {
+    const raw = properties?.section_id;
+    if (typeof raw === "string" && raw.trim())
+        return raw.trim();
+    return "<unsectioned>";
+}
+function pickDefaultSectionId(components) {
+    const counts = new Map();
+    for (const c of components) {
+        const sid = getSectionIdFromProperties(c.properties);
+        counts.set(sid, (counts.get(sid) ?? 0) + 1);
+    }
+    if (counts.size === 0)
+        return "<unsectioned>";
+    // Prefer a concrete section over the synthetic "<unsectioned>" bucket.
+    const globalCount = counts.get("global") ?? 0;
+    let bestSectionId = "<unsectioned>";
+    let bestCount = -1;
+    for (const [sid, count] of counts.entries()) {
+        if (sid === "<unsectioned>")
+            continue;
+        // When we have a shared/global bucket, don't pick it for the injected
+        // Application/User nodes unless it's the only available section.
+        if (sid === "global")
+            continue;
+        if (count > bestCount) {
+            bestCount = count;
+            bestSectionId = sid;
+        }
+    }
+    if (bestCount >= 0)
+        return bestSectionId;
+    if (globalCount > 0)
+        return "global";
+    return "<unsectioned>";
+}
+function pickSectionRoleForSectionId(components, sectionId) {
+    // Prefer an existing role from components that already carry section metadata.
+    for (const c of components) {
+        if (getSectionIdFromProperties(c.properties) !== sectionId)
+            continue;
+        const rawRole = c.properties?.section_role;
+        if (typeof rawRole === "string" && rawRole.trim())
+            return rawRole.trim();
+    }
+    // Fall back to the convention used by the sectioning layer and tests.
+    return sectionId === "root" ? "root" : "service";
+}
+function getSectionLabelFromProperties(properties) {
+    const raw = properties?.section_label;
+    if (typeof raw === "string" && raw.trim())
+        return raw.trim();
+    return undefined;
+}

package/dist/src/classifier/terraform-provider-hub.d.ts

@@ -0,0 +1,6 @@
+import type { DetectedComponent } from "../core/types/component";
+/**
+ * Picks a single Terraform `provider.*` third_party node to act as the cloud
+ * control-plane hub (User → provider → resources) when there is no app asset.
+ */
+export declare function findTerraformPrimaryProviderHub(components: DetectedComponent[], sectionId?: string): DetectedComponent | undefined;

package/dist/src/classifier/terraform-provider-hub.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findTerraformPrimaryProviderHub = findTerraformPrimaryProviderHub;
+const PRIMARY_PROVIDER_LOCAL_NAMES = [
+    "aws",
+    "azurerm",
+    "google",
+    "google-beta",
+];
+/**
+ * Picks a single Terraform `provider.*` third_party node to act as the cloud
+ * control-plane hub (User → provider → resources) when there is no app asset.
+ */
+function findTerraformPrimaryProviderHub(components, sectionId) {
+    let pool = components.filter((c) => c.type === "third_party" &&
+        typeof c.properties?.terraform_address === "string" &&
+        c.properties.terraform_address.startsWith("provider."));
+    if (sectionId && sectionId.trim()) {
+        pool = pool.filter((c) => String(c.properties?.section_id ?? "") === sectionId);
+        if (pool.length === 0)
+            return undefined;
+    }
+    if (pool.length === 0)
+        return undefined;
+    const byLocal = new Map();
+    for (const p of pool) {
+        const addr = String(p.properties?.terraform_address ?? "");
+        const local = addr.replace(/^provider\./, "").trim();
+        if (local)
+            byLocal.set(local, p);
+    }
+    for (const name of PRIMARY_PROVIDER_LOCAL_NAMES) {
+        const hit = byLocal.get(name);
+        if (hit)
+            return hit;
+    }
+    return [...pool].sort((a, b) => String(a.properties?.terraform_address ?? "").localeCompare(String(b.properties?.terraform_address ?? "")))[0];
+}

package/dist/src/cli.d.ts

@@ -0,0 +1,2 @@
+import "./config/load-cli-env";
+export declare function run(argv?: string[]): Promise<void>;