@memberjunction/db-auto-doc 5.37.0 → 5.39.0

package/dist/discovery/ColumnClusterer.js

@@ -0,0 +1,205 @@
+/**
+ * ColumnClusterer — average-linkage agglomerative clustering on cosine distance.
+ *
+ * Input:  N columns each with a unit-normalized embedding vector.
+ * Output: clusters of ≥2 members spanning ≥2 distinct tables, sorted biggest-first.
+ *
+ * Algorithm:
+ *   1. Compute the full N×N upper-triangular pairwise cosine-distance matrix.
+ *   2. Repeatedly find the closest two clusters; if their distance ≤ mergeThreshold,
+ *      merge them and update the distance to every other cluster as the AVERAGE
+ *      pairwise distance between their members. Stop when the closest pair exceeds
+ *      the threshold.
+ *   3. Filter clusters by minClusterSize + minDistinctTables.
+ *
+ * Why average-linkage:
+ *   Complete-linkage requires EVERY pair across two clusters to be close before
+ *   merging — too tight in practice. Single-linkage chains unrelated points
+ *   through bridges. Average-linkage merges when clusters are similar on average,
+ *   which matches the actual "cluster = one concept" question.
+ */
+export class ColumnClusterer {
+    constructor(opts) {
+        this.opts = opts;
+        this.lastResolvedThreshold = 0;
+    }
+    cluster(columns) {
+        const n = columns.length;
+        if (n < this.opts.minClusterSize)
+            return [];
+        // Step 1 — pairwise cosine-distance matrix.
+        const distance = computePairwiseDistance(columns);
+        const resolvedThreshold = this.opts.mergeThreshold !== undefined
+            ? this.opts.mergeThreshold
+            : computeThresholdFromDistribution(distance, this.opts.mergeThresholdPercentile ?? 5);
+        this.lastResolvedThreshold = resolvedThreshold;
+        // Step 2 — agglomerative merge with average-linkage.
+        const clusters = new Map();
+        for (let i = 0; i < n; i++)
+            clusters.set(i, [i]);
+        const avgDist = new Map();
+        for (let i = 0; i < n; i++) {
+            const row = new Map();
+            for (let j = 0; j < n; j++) {
+                if (i !== j)
+                    row.set(j, distanceAt(distance, n, i, j));
+            }
+            avgDist.set(i, row);
+        }
+        while (clusters.size > 1) {
+            let bestA = -1;
+            let bestB = -1;
+            let bestD = Number.POSITIVE_INFINITY;
+            for (const [a, aRow] of avgDist) {
+                for (const [b, d] of aRow) {
+                    if (a < b && d < bestD) {
+                        bestD = d;
+                        bestA = a;
+                        bestB = b;
+                    }
+                }
+            }
+            if (bestD > resolvedThreshold || bestA < 0)
+                break;
+            const membersA = clusters.get(bestA);
+            const membersB = clusters.get(bestB);
+            const sizeA = membersA.length;
+            const sizeB = membersB.length;
+            const rowA = avgDist.get(bestA);
+            const rowB = avgDist.get(bestB);
+            for (const c of clusters.keys()) {
+                if (c === bestA || c === bestB)
+                    continue;
+                const dAC = rowA.get(c);
+                const dBC = rowB.get(c);
+                const newD = (sizeA * dAC + sizeB * dBC) / (sizeA + sizeB);
+                rowA.set(c, newD);
+                avgDist.get(c).set(bestA, newD);
+                avgDist.get(c).delete(bestB);
+            }
+            rowA.delete(bestB);
+            avgDist.delete(bestB);
+            clusters.set(bestA, membersA.concat(membersB));
+            clusters.delete(bestB);
+        }
+        // Step 3 — filter + emit.
+        const out = [];
+        for (const memberIdxs of clusters.values()) {
+            if (memberIdxs.length < this.opts.minClusterSize)
+                continue;
+            const tableSet = new Set(memberIdxs.map((i) => `${columns[i].schema}.${columns[i].table}`));
+            if (tableSet.size < this.opts.minDistinctTables)
+                continue;
+            const members = memberIdxs.map((i) => ({
+                schema: columns[i].schema,
+                table: columns[i].table,
+                column: columns[i].column,
+                participatesInFK: !!columns[i].participatesInFK,
+                fkTarget: columns[i].fkTarget ?? null,
+                isPrimaryKey: !!columns[i].isPrimaryKey,
+            }));
+            out.push({
+                memberIndexes: memberIdxs,
+                members,
+                maxIntraDistance: maxIntraDistance(memberIdxs, distance, n),
+            });
+        }
+        out.sort((a, b) => b.members.length - a.members.length);
+        return out;
+    }
+}
+// ─── Helpers ────────────────────────────────────────────────────────────────
+function computePairwiseDistance(columns) {
+    const n = columns.length;
+    const len = (n * (n - 1)) / 2;
+    const out = new Float32Array(len);
+    for (let i = 0; i < n; i++) {
+        const ei = columns[i].embedding;
+        for (let j = i + 1; j < n; j++) {
+            const ej = columns[j].embedding;
+            let s = 0;
+            const dim = Math.min(ei.length, ej.length);
+            for (let k = 0; k < dim; k++)
+                s += ei[k] * ej[k];
+            const d = 1 - s;
+            out[flatIndex(i, j, n)] = d > 0 ? d : 0;
+        }
+    }
+    return out;
+}
+function flatIndex(i, j, n) {
+    return i * n - (i * (i + 1)) / 2 + (j - i - 1);
+}
+/**
+ * Auto-calibrate the merge threshold from the actual distance distribution.
+ *
+ * Real embedding-distance distributions for "same vs different concept" data
+ * are usually BIMODAL: a dense cluster of small distances (same concept) and
+ * a dense cluster of large distances (different concept), with a clear gap
+ * between them. A flat percentile (p5) breaks in the degenerate cases where
+ * many descriptions land exactly identically — p5 becomes 0 and the merge
+ * loop refuses to merge anything that isn't byte-equal.
+ *
+ * Robust approach: find the largest gap in the BOTTOM half of the sorted
+ * distance distribution (where the same-concept / different-concept split
+ * lives), and place the threshold inside that gap so all same-concept pairs
+ * merge and no different-concept pair does.
+ *
+ * Fallback when no clear gap exists: use the pN percentile (legacy behavior).
+ * Floor the result so we still merge near-identical descriptions even when
+ * the geometry is degenerate.
+ */
+function computeThresholdFromDistribution(distances, percentile) {
+    if (distances.length === 0)
+        return 0;
+    const sorted = Array.from(distances).sort((a, b) => a - b);
+    const N = sorted.length;
+    // Search the entire distribution for the largest gap whose START is in the
+    // "same-concept territory" (≤0.40 cosine distance). Anything above 0.40 is
+    // already inter-concept, and the gaps up there are irrelevant noise.
+    //
+    // For Gemini clustering embeddings, same-concept pairs concentrate ≤0.15
+    // and different-concept pairs ≥0.30, so the boundary gap lives somewhere in
+    // [0.05, 0.35]. Searching that window finds it whether the intra-concept
+    // pairs are 10% or 60% of all pairs.
+    const GAP_START_MAX = 0.40;
+    const MIN_MEANINGFUL_GAP = 0.03;
+    const FLOOR = 0.05;
+    let bestGap = 0;
+    let bestGapStart = 0;
+    for (let i = 0; i < N - 1; i++) {
+        if (sorted[i] > GAP_START_MAX)
+            break;
+        const gap = sorted[i + 1] - sorted[i];
+        if (gap > bestGap) {
+            bestGap = gap;
+            bestGapStart = sorted[i];
+        }
+    }
+    if (bestGap >= MIN_MEANINGFUL_GAP) {
+        return Math.max(FLOOR, bestGapStart + bestGap * 0.5);
+    }
+    // No clear gap — use the pN percentile, floored to merge near-identical pairs.
+    const p = Math.max(0, Math.min(100, percentile));
+    const idx = Math.floor((N * p) / 100);
+    return Math.max(FLOOR, sorted[Math.min(idx, N - 1)]);
+}
+function distanceAt(matrix, n, i, j) {
+    if (i === j)
+        return 0;
+    if (i > j)
+        [i, j] = [j, i];
+    return matrix[flatIndex(i, j, n)];
+}
+function maxIntraDistance(memberIdxs, matrix, n) {
+    let max = 0;
+    for (let a = 0; a < memberIdxs.length; a++) {
+        for (let b = a + 1; b < memberIdxs.length; b++) {
+            const d = distanceAt(matrix, n, memberIdxs[a], memberIdxs[b]);
+            if (d > max)
+                max = d;
+        }
+    }
+    return max;
+}
+//# sourceMappingURL=ColumnClusterer.js.map

package/dist/discovery/ColumnClusterer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ColumnClusterer.js","sourceRoot":"","sources":["../../src/discovery/ColumnClusterer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAsCH,MAAM,OAAO,eAAe;IAGxB,YAA6B,IAAsB;QAAtB,SAAI,GAAJ,IAAI,CAAkB;QAF5C,0BAAqB,GAAG,CAAC,CAAC;IAEqB,CAAC;IAEhD,OAAO,CAAC,OAA+B;QAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QACzB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,EAAE,CAAC;QAE5C,4CAA4C;QAC5C,MAAM,QAAQ,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,KAAK,SAAS;YAC5D,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc;YAC1B,CAAC,CAAC,gCAAgC,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,CAAC,CAAC;QAC1F,IAAI,CAAC,qBAAqB,GAAG,iBAAiB,CAAC;QAE/C,qDAAqD;QACrD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+B,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzB,IAAI,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;YACf,IAAI,KAAK,GAAG,MAAM,CAAC,iBAAiB,CAAC;YACrC,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;oBACxB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;wBACrB,KAAK,GAAG,CAAC,CAAC;wBACV,KAAK,GAAG,CAAC,CAAC;wBACV,KAAK,GAAG,CAAC,CAAC;oBACd,CAAC;gBACL,CAAC;YACL,CAAC;YACD,IAAI,KAAK,GAAG,iBAAiB,IAAI,KAAK,GAAG,CAAC;gBAAE,MAAM;YAElD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC9B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACjC,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK;oBAAE,SAAS;gBACzC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;gBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,CAAC,KAAK,GAAG,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBAClB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEtB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/C,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAED,0BAA0B;QAC1B,MAAM,GAAG,GAAiB,EAAE,CAAC;QAC7B,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACzC,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,SAAS;YAC3D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC5F,IAAI,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YAC1D,MAAM,OAAO,GAA8B,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9D,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM;gBACzB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK;gBACvB,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM;gBACzB,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,gBAAgB;gBAC/C,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI;gBACrC,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY;aAC1C,CAAC,CAAC,CAAC;YACJ,GAAG,CAAC,IAAI,CAAC;gBACL,aAAa,EAAE,UAAU;gBACzB,OAAO;gBACP,gBAAgB,EAAE,gBAAgB,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;aAC9D,CAAC,CAAC;QACP,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACxD,OAAO,GAAG,CAAC;IACf,CAAC;CACJ;AAED,+EAA+E;AAE/E,SAAS,uBAAuB,CAAC,OAA+B;IAC5D,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;gBAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChB,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAC9C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AACnD,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,gCAAgC,CAAC,SAAuB,EAAE,UAAkB;IACjF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAExB,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,EAAE;IACF,yEAAyE;IACzE,4EAA4E;IAC5E,yEAAyE;IACzE,qCAAqC;IACrC,MAAM,aAAa,GAAG,IAAI,CAAC;IAC3B,MAAM,kBAAkB,GAAG,IAAI,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC;IAEnB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,aAAa;YAAE,MAAM;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;YAChB,OAAO,GAAG,GAAG,CAAC;YACd,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,IAAI,OAAO,IAAI,kBAAkB,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC;IACzD,CAAC;IAED,+EAA+E;IAC/E,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,UAAU,CAAC,MAAoB,EAAE,CAAS,EAAE,CAAS,EAAE,CAAS;IACrE,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,GAAG,CAAC;QAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,gBAAgB,CAAC,UAAoB,EAAE,MAAoB,EAAE,CAAS;IAC3E,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,GAAG,GAAG;gBAAE,GAAG,GAAG,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC"}

package/dist/discovery/ColumnNormalizer.d.ts

@@ -0,0 +1,106 @@
+/**
+ * TableNormalizer — one LLM call per TABLE (not per column), upstream of embedding.
+ *
+ * For each in-scope table, a single LLM call sees:
+ *   - The table's name + description + sibling columns
+ *   - Every column's identity + description + sample values + FK/PK status
+ *
+ * The call returns one normalized entry per column with:
+ *   - conceptName            : canonical snake_case (`email_address`, `customer_id`, ...)
+ *   - normalizationStrategy  : how values should be compared
+ *   - normalizedDescription  : business-concept-focused, system-agnostic sentence
+ *   - isUsefulOrganicKey     : false for audit/system/free-form (filtered out)
+ *   - confidence + reasoning
+ *
+ * Why per-table instead of per-column:
+ *   - Fewer calls (5K cols across 500 tables → 500 calls instead of 5K). At
+ *     Gemini Flash pricing, that's ~$0.04 instead of ~$0.20 for APTIFY-scale.
+ *   - System prompt amortizes across all columns in the table.
+ *   - The LLM sees siblings as context — knowing the table has FirstName + LastName
+ *     next to an Email column reveals it's a person email, not a server hostname.
+ *   - More token-efficient: one JSON array out instead of N independent objects.
+ *
+ * The single most important constraint: same-concept columns from DIFFERENT
+ * TABLES (across systems) must produce the same conceptName and a similar
+ * normalizedDescription so the embedding step naturally clusters them. The
+ * prompt enforces this via a canonical concept-name list.
+ */
+import { AIConfig } from '../types/config.js';
+import { OrganicKeyNormalizationStrategy } from '../types/organic-keys.js';
+/** One column's input to the normalizer. */
+export interface NormalizerInputColumn {
+    schema: string;
+    table: string;
+    column: string;
+    dataType: string;
+    /** Original LLM-generated description from DBAutoDoc's prior analysis pass. */
+    originalDescription: string;
+    /** Sample values from the column's actual data. */
+    sampleValues: string[];
+    /** Whether the column participates in any FK (declared or detected). */
+    participatesInFK: boolean;
+    fkTarget?: {
+        schema: string;
+        table: string;
+        column: string;
+    } | null;
+    isPrimaryKey: boolean;
+}
+/** All columns of one table, batched for a single LLM call. */
+export interface TableNormalizationInput {
+    schema: string;
+    schemaDescription?: string;
+    table: string;
+    tableDescription?: string;
+    columns: NormalizerInputColumn[];
+}
+/** One column's normalized output. */
+export interface NormalizedColumn extends NormalizerInputColumn {
+    conceptName: string;
+    normalizationStrategy: OrganicKeyNormalizationStrategy;
+    customNormalizationExpression?: string;
+    normalizedDescription: string;
+    isUsefulOrganicKey: boolean;
+    confidence: number;
+    reasoning: string;
+}
+/** Aggregate result of normalizing many tables. */
+export interface NormalizationBatchResult {
+    /** Useful organic-key columns surviving normalization (filtered to isUsefulOrganicKey=true). */
+    normalized: NormalizedColumn[];
+    /** Columns the normalizer marked isUsefulOrganicKey=false (audit, free-form, etc.). */
+    rejected: number;
+    /** Tables where the LLM call failed entirely. */
+    errors: number;
+    tokens: {
+        total: number;
+        input: number;
+        output: number;
+    };
+}
+export interface NormalizerOptions {
+    /** Concurrency for per-table LLM calls. Default 8. */
+    concurrency?: number;
+    /** Max retries per table on transient failures. Default 2. */
+    maxRetries?: number;
+    /** Progress callback (done count, total count). */
+    onProgress?: (done: number, total: number) => void;
+}
+export declare class TableNormalizer {
+    private readonly aiConfig;
+    private readonly llm;
+    constructor(aiConfig: AIConfig);
+    /** Normalize one table — one LLM call returning per-column entries. */
+    normalizeTable(input: TableNormalizationInput, maxRetries?: number): Promise<{
+        normalized: NormalizedColumn[];
+        tokens: {
+            total: number;
+            input: number;
+            output: number;
+        };
+        errorMessage?: string;
+    }>;
+    /** Batch normalize many tables with bounded concurrency. */
+    normalizeAll(tables: TableNormalizationInput[], opts?: NormalizerOptions): Promise<NormalizationBatchResult>;
+}
+//# sourceMappingURL=ColumnNormalizer.d.ts.map

package/dist/discovery/ColumnNormalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ColumnNormalizer.d.ts","sourceRoot":"","sources":["../../src/discovery/ColumnNormalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAIH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,+BAA+B,EAAE,MAAM,0BAA0B,CAAC;AAG3E,4CAA4C;AAC5C,MAAM,WAAW,qBAAqB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mDAAmD;IACnD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,wEAAwE;IACxE,gBAAgB,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACpE,YAAY,EAAE,OAAO,CAAC;CACzB;AAED,+DAA+D;AAC/D,MAAM,WAAW,uBAAuB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAiB,SAAQ,qBAAqB;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,+BAA+B,CAAC;IACvD,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,mDAAmD;AACnD,MAAM,WAAW,wBAAwB;IACrC,gGAAgG;IAChG,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAC/B,uFAAuF;IACvF,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5D;AAED,MAAM,WAAW,iBAAiB;IAC9B,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACtD;AAED,qBAAa,eAAe;IAGZ,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAFrC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAU;gBAED,QAAQ,EAAE,QAAQ;IAI/C,uEAAuE;IAC1D,cAAc,CACvB,KAAK,EAAE,uBAAuB,EAC9B,UAAU,SAAI,GACf,OAAO,CAAC;QACP,UAAU,EAAE,gBAAgB,EAAE,CAAC;QAC/B,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;QACzD,YAAY,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IA6EF,4DAA4D;IAC/C,YAAY,CACrB,MAAM,EAAE,uBAAuB,EAAE,EACjC,IAAI,GAAE,iBAAsB,GAC7B,OAAO,CAAC,wBAAwB,CAAC;CAqCvC"}