@memberjunction/db-auto-doc 5.36.0 → 5.38.0

package/dist/discovery/OrganicKeyTranslator.js

@@ -0,0 +1,166 @@
+/**
+ * OrganicKeyTranslator — PURE EMISSION.
+ *
+ * Takes a list of clusters that have ALREADY been gated by the Composer, plus
+ * the transitive spokes for those that survived, and fans them out into PR
+ * #2193's per-schema/per-table JSON.
+ *
+ * No filtering, no thresholds, no scoring logic in here. The Composer is the
+ * single emission gate; this file is its render-to-JSON helper.
+ *
+ * Same-concept consolidation: clusters that the LLM normalizer assigned the
+ * same canonical concept name (e.g. four separate geometric clusters all
+ * named `product_id`) are MERGED here into a single conceptual cluster before
+ * fan-out — deterministic equivalent of an LLM concept-merge pass.
+ */
+import { memberColumns, isCompoundMember, } from '../types/organic-keys.js';
+// ─── Entry point — pure fan-out ─────────────────────────────────────────────
+/**
+ * Render gated clusters into PR #2193 JSON. Same-concept consolidation +
+ * per-table fan-out + transitive spoke attachment. No filters, no thresholds.
+ *
+ * The caller (Composer) is responsible for filtering. Anything passed in
+ * gets emitted.
+ */
+export function translateClusters(clusters, transitiveSpokes = []) {
+    // Step 1 — group by normalized canonical concept name.
+    const byConcept = new Map();
+    for (const cluster of clusters) {
+        if (cluster.members.length < 2)
+            continue;
+        const key = normalizeConceptName(cluster.concept);
+        const bucket = byConcept.get(key);
+        if (bucket)
+            bucket.push(cluster);
+        else
+            byConcept.set(key, [cluster]);
+    }
+    const out = {};
+    for (const group of byConcept.values()) {
+        // Step 2 — union members across all clusters in the group, deduped by
+        // full column key (schema.table.col1,col2,...).
+        const memberMap = new Map();
+        for (const c of group) {
+            for (const m of c.members) {
+                const k = `${m.schema}.${m.table}.${memberColumns(m).join(',')}`;
+                if (!memberMap.has(k))
+                    memberMap.set(k, m);
+            }
+        }
+        const allMembers = Array.from(memberMap.values());
+        if (allMembers.length < 2)
+            continue;
+        const distinctTables = new Set(allMembers.map((m) => `${m.schema}.${m.table}`));
+        if (distinctTables.size < 2)
+            continue;
+        // Pick the anchor (highest confidence, ties by member count).
+        const anchor = group
+            .slice()
+            .sort((a, b) => (b.confidence - a.confidence) || (b.members.length - a.members.length))[0];
+        const name = prettyConceptName(anchor.concept);
+        // Step 3 — fan out: one entry per unique owner, spokes = all OTHER unique members.
+        for (const owner of allMembers) {
+            const tableEntry = upsertTable(out, owner.schema, owner.table);
+            const seenSpokes = new Set();
+            const spokes = [];
+            // Direct spokes — every other cluster member.
+            for (const target of allMembers) {
+                if (target === owner)
+                    continue;
+                const targetColumns = memberColumns(target);
+                const k = `${target.schema}.${target.table}.${targetColumns.join(',')}`;
+                if (seenSpokes.has(k))
+                    continue;
+                seenSpokes.add(k);
+                spokes.push({
+                    SchemaName: target.schema,
+                    TableName: target.table,
+                    RelatedFieldNames: targetColumns,
+                    DisplayName: `${target.schema}.${target.table}`,
+                });
+            }
+            // Transitive spokes — bridge views attached to matching hubs.
+            const ownerColumns = memberColumns(owner);
+            const matchingTransitive = transitiveSpokes.filter((t) => t.hubSchema === owner.schema &&
+                t.hubTable === owner.table &&
+                t.hubKeyFields.length === ownerColumns.length &&
+                t.hubKeyFields.every((f, idx) => f === ownerColumns[idx]));
+            for (const t of matchingTransitive) {
+                spokes.push({
+                    SchemaName: t.spokeSchema,
+                    TableName: t.spokeTable,
+                    TransitiveView: t.transitiveView,
+                    TransitiveMatchFieldNames: t.transitiveMatchFieldNames,
+                    TransitiveOutputFieldName: t.transitiveOutputFieldName,
+                    RelatedEntityJoinFieldName: t.relatedEntityJoinFieldName,
+                    DisplayName: `${t.spokeSchema}.${t.spokeTable} (via ${t.hubConcept ?? 'bridge'})`,
+                });
+            }
+            const compoundSuffix = isCompoundMember(owner) ? ` (compound: ${ownerColumns.join('+')})` : '';
+            // Per-column normalization: each emitted EntityOrganicKey row carries the
+            // transformation for ITS owner column. The runtime looks up each side's
+            // own expression at match time (see EntityInfo.BuildOrganicKeyViewParams),
+            // so different columns in the same cluster can carry different functions.
+            // Falls back to cluster-level normalization if a member didn't get its own.
+            const ownerNormalization = owner.normalizationStrategy ?? anchor.normalization;
+            const ownerCustomExpression = owner.customNormalizationExpression ?? anchor.customNormalizationExpression;
+            tableEntry.OrganicKeys.push({
+                Name: name + compoundSuffix,
+                Description: anchor.reasoning,
+                MatchFieldNames: ownerColumns,
+                NormalizationStrategy: ownerNormalization,
+                CustomNormalizationExpression: ownerCustomExpression,
+                AutoCreateRelatedViewOnForm: true,
+                RelatedEntities: spokes,
+            });
+        }
+    }
+    return out;
+}
+// ─── Helpers ────────────────────────────────────────────────────────────────
+function normalizeConceptName(name) {
+    return (name ?? '')
+        .toLowerCase()
+        .replace(/[\s\-]+/g, '_')
+        .replace(/_+/g, '_')
+        .replace(/^_|_$/g, '')
+        .trim();
+}
+function prettyConceptName(concept) {
+    const pretty = concept
+        .split('_')
+        .filter((p) => p.length > 0)
+        .map((p) => p[0].toUpperCase() + p.slice(1))
+        .join(' ');
+    return `${pretty} Match`;
+}
+function upsertTable(out, schema, table) {
+    let bySchema = out[schema];
+    if (!bySchema) {
+        bySchema = [];
+        out[schema] = bySchema;
+    }
+    let entry = bySchema.find((t) => t.TableName === table);
+    if (!entry) {
+        entry = { TableName: table, OrganicKeys: [] };
+        bySchema.push(entry);
+    }
+    return entry;
+}
+/** Tally the emit payload: number of schemas, tables, organic keys, and spokes it contains. */
+export function countOutputEntries(out) {
+    let tables = 0;
+    let keys = 0;
+    let spokes = 0;
+    const schemas = Object.keys(out).length;
+    for (const tableList of Object.values(out)) {
+        tables += tableList.length;
+        for (const t of tableList) {
+            keys += t.OrganicKeys.length;
+            for (const k of t.OrganicKeys)
+                spokes += k.RelatedEntities.length;
+        }
+    }
+    return { schemas, tables, keys, spokes };
+}
+//# sourceMappingURL=OrganicKeyTranslator.js.map

package/dist/discovery/OrganicKeyTranslator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OrganicKeyTranslator.js","sourceRoot":"","sources":["../../src/discovery/OrganicKeyTranslator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAIH,aAAa,EACb,gBAAgB,GACnB,MAAM,0BAA0B,CAAC;AA6DlC,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC7B,QAA6B,EAC7B,mBAA2C,EAAE;IAE7C,uDAAuD;IACvD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAC;IACzD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACzC,MAAM,GAAG,GAAG,oBAAoB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;;YAC5B,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,GAAG,GAA8B,EAAE,CAAC;IAE1C,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;QACrC,sEAAsE;QACtE,gDAAgD;QAChD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC7D,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACpB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC;QACD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACpC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,IAAI,cAAc,CAAC,IAAI,GAAG,CAAC;YAAE,SAAS;QAEtC,8DAA8D;QAC9D,MAAM,MAAM,GAAG,KAAK;aACf,KAAK,EAAE;aACP,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/F,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE/C,mFAAmF;QACnF,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YAE/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;YACrC,MAAM,MAAM,GAAoC,EAAE,CAAC;YAEnD,8CAA8C;YAC9C,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,MAAM,KAAK,KAAK;oBAAE,SAAS;gBAC/B,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;gBAC5C,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxE,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,SAAS;gBAChC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC;oBACR,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,SAAS,EAAE,MAAM,CAAC,KAAK;oBACvB,iBAAiB,EAAE,aAAa;oBAChC,WAAW,EAAE,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,EAAE;iBAClD,CAAC,CAAC;YACP,CAAC;YAED,8DAA8D;YAC9D,MAAM,YAAY,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YAC1C,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,MAAM,CAC9C,CAAC,CAAC,EAAE,EAAE,CACF,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,MAAM;gBAC5B,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK;gBAC1B,CAAC,CAAC,YAAY,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM;gBAC7C,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,CAChE,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,kBAAkB,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC;oBACR,UAAU,EAAE,CAAC,CAAC,WAAW;oBACzB,SAAS,EAAE,CAAC,CAAC,UAAU;oBACvB,cAAc,EAAE,CAAC,CAAC,cAAc;oBAChC,yBAAyB,EAAE,CAAC,CAAC,yBAAyB;oBACtD,yBAAyB,EAAE,CAAC,CAAC,yBAAyB;oBACtD,0BAA0B,EAAE,CAAC,CAAC,0BAA0B;oBACxD,WAAW,EAAE,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,SAAS,CAAC,CAAC,UAAU,IAAI,QAAQ,GAAG;iBACpF,CAAC,CAAC;YACP,CAAC;YAED,MAAM,cAAc,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/F,0EAA0E;YAC1E,wEAAwE;YACxE,2EAA2E;YAC3E,0EAA0E;YAC1E,4EAA4E;YAC5E,MAAM,kBAAkB,GAAG,KAAK,CAAC,qBAAqB,IAAI,MAAM,CAAC,aAAa,CAAC;YAC/E,MAAM,qBAAqB,GACvB,KAAK,CAAC,6BAA6B,IAAI,MAAM,CAAC,6BAA6B,CAAC;YAChF,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC;gBACxB,IAAI,EAAE,IAAI,GAAG,cAAc;gBAC3B,WAAW,EAAE,MAAM,CAAC,SAAS;gBAC7B,eAAe,EAAE,YAAY;gBAC7B,qBAAqB,EAAE,kBAAkB;gBACzC,6BAA6B,EAAE,qBAAqB;gBACpD,2BAA2B,EAAE,IAAI;gBACjC,eAAe,EAAE,MAAM;aAC1B,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAED,+EAA+E;AAE/E,SAAS,oBAAoB,CAAC,IAAY;IACtC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;SACd,WAAW,EAAE;SACb,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;SACrB,IAAI,EAAE,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe;IACtC,MAAM,MAAM,GAAG,OAAO;SACjB,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAC3C,IAAI,CAAC,GAAG,CAAC,CAAC;IACf,OAAO,GAAG,MAAM,QAAQ,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAChB,GAA8B,EAC9B,MAAc,EACd,KAAa;IAEb,IAAI,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,QAAQ,GAAG,EAAE,CAAC;QACd,GAAG,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;IAC3B,CAAC;IACD,IAAI,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC;IACxD,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,KAAK,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,kBAAkB,CAAC,GAA8B;IAM7D,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC;YAC7B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW;gBAAE,MAAM,IAAI,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC;QACtE,CAAC;IACL,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC7C,CAAC"}

package/dist/discovery/SemanticPhase.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Phase A — SEMANTIC.
+ *
+ * The defensible PR #2193 pipeline:
+ *
+ *   1. PREFILTER (deterministic)
+ *        Drop columns that cannot be organic keys regardless of semantics:
+ *        binary/blob types, audit-named columns, ultra-low-cardinality.
+ *
+ *   2. NORMALIZE TO BUSINESS SPACE (LLM, one call per table)
+ *        For each surviving column, produce a structured business-focused
+ *        description that encodes PR #2193's "if two rows share this value,
+ *        do they refer to the same real-world entity?" test, plus a canonical
+ *        snake_case conceptName, normalization strategy, and isOrganicKey gate.
+ *        Columns the normalizer rejects (audit, categorical, surrogate, free-
+ *        text, etc.) are dropped here.
+ *
+ *   3. EMBED the normalized descriptions
+ *        Same-concept descriptions geometrically converge regardless of
+ *        source-system, table, or column-name conventions. Sample values are
+ *        appended to distinguish e.g. emails from phones even when both
+ *        describe "identifying a person".
+ *
+ *   4. CLUSTER (agglomerative average-linkage, gap-detected threshold)
+ *        Each cluster is a candidate organic-key concept. The threshold is
+ *        auto-calibrated by finding the largest gap in the bottom of the
+ *        pairwise-distance distribution — robust across schemas/providers.
+ *
+ *   5. SPLIT-BY-CONCEPT
+ *        When the LLM has already assigned DIFFERENT conceptNames to columns
+ *        the embeddings put in the same cluster (e.g. product_id /
+ *        product_model_id / product_category_id all describe "an identifier
+ *        for a product-related entity" and embed close together), honor the
+ *        LLM's distinction by splitting the cluster along conceptName.
+ *        Catches Gemini's tight description-space compression without losing
+ *        cross-table merging when the LLM does converge on one name.
+ *
+ *   6. LABEL each cluster from its members' conceptName votes.
+ *
+ * No vocabulary discovery, no LLM cluster judge, no per-cluster refinement —
+ * the normalize step IS the PR #2193 judgment, the cluster + split steps ARE
+ * the cross-table identity proof.
+ */
+import { AIConfig, OrganicKeyDetectionConfig } from '../types/config.js';
+import { DatabaseDocumentation } from '../types/state.js';
+import { OrganicKeyCluster } from '../types/organic-keys.js';
+/** Result of the semantic (organic-key clustering) phase. */
+export interface SemanticPhaseResult {
+    clusters: OrganicKeyCluster[];
+    tokens: {
+        input: number;
+        output: number;
+        total: number;
+    };
+    summary: {
+        columnsInScope: number;
+        columnsNormalized: number;
+        columnsRejectedByNormalizer: number;
+        /** Clusters formed by the embedding clusterer, before the concept-name split. */
+        clustersBeforeSplit: number;
+        /** Final clusters after splitting by concept name and dropping sub-threshold sub-clusters. */
+        clustersFound: number;
+        /** Sub-clusters discarded during the split for falling below minClusterSize / minDistinctTables. */
+        clustersDropped: number;
+    };
+}
+/** Receives human-readable progress messages as the semantic phase advances. */
+export type ProgressCallback = (message: string) => void;
+export declare function runSemanticPhase(state: DatabaseDocumentation, config: OrganicKeyDetectionConfig, aiConfig: AIConfig, progress?: ProgressCallback): Promise<SemanticPhaseResult>;
+//# sourceMappingURL=SemanticPhase.d.ts.map

package/dist/discovery/SemanticPhase.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SemanticPhase.d.ts","sourceRoot":"","sources":["../../src/discovery/SemanticPhase.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,OAAO,EAAE,QAAQ,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAEH,iBAAiB,EAGpB,MAAM,0BAA0B,CAAC;AAmDlC,6DAA6D;AAC7D,MAAM,WAAW,mBAAmB;IAChC,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACzD,OAAO,EAAE;QACL,cAAc,EAAE,MAAM,CAAC;QACvB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,2BAA2B,EAAE,MAAM,CAAC;QACpC,iFAAiF;QACjF,mBAAmB,EAAE,MAAM,CAAC;QAC5B,8FAA8F;QAC9F,aAAa,EAAE,MAAM,CAAC;QACtB,oGAAoG;QACpG,eAAe,EAAE,MAAM,CAAC;KAC3B,CAAC;CACL;AAED,gFAAgF;AAChF,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;AAEzD,wBAAsB,gBAAgB,CAClC,KAAK,EAAE,qBAAqB,EAC5B,MAAM,EAAE,yBAAyB,EACjC,QAAQ,EAAE,QAAQ,EAClB,QAAQ,GAAE,gBAA2B,GACtC,OAAO,CAAC,mBAAmB,CAAC,CA2F9B"}

package/dist/discovery/SemanticPhase.js

@@ -0,0 +1,423 @@
+/**
+ * Phase A — SEMANTIC.
+ *
+ * The defensible PR #2193 pipeline:
+ *
+ *   1. PREFILTER (deterministic)
+ *        Drop columns that cannot be organic keys regardless of semantics:
+ *        binary/blob types, audit-named columns, ultra-low-cardinality.
+ *
+ *   2. NORMALIZE TO BUSINESS SPACE (LLM, one call per table)
+ *        For each surviving column, produce a structured business-focused
+ *        description that encodes PR #2193's "if two rows share this value,
+ *        do they refer to the same real-world entity?" test, plus a canonical
+ *        snake_case conceptName, normalization strategy, and isOrganicKey gate.
+ *        Columns the normalizer rejects (audit, categorical, surrogate, free-
+ *        text, etc.) are dropped here.
+ *
+ *   3. EMBED the normalized descriptions
+ *        Same-concept descriptions geometrically converge regardless of
+ *        source-system, table, or column-name conventions. Sample values are
+ *        appended to distinguish e.g. emails from phones even when both
+ *        describe "identifying a person".
+ *
+ *   4. CLUSTER (agglomerative average-linkage, gap-detected threshold)
+ *        Each cluster is a candidate organic-key concept. The threshold is
+ *        auto-calibrated by finding the largest gap in the bottom of the
+ *        pairwise-distance distribution — robust across schemas/providers.
+ *
+ *   5. SPLIT-BY-CONCEPT
+ *        When the LLM has already assigned DIFFERENT conceptNames to columns
+ *        the embeddings put in the same cluster (e.g. product_id /
+ *        product_model_id / product_category_id all describe "an identifier
+ *        for a product-related entity" and embed close together), honor the
+ *        LLM's distinction by splitting the cluster along conceptName.
+ *        Catches Gemini's tight description-space compression without losing
+ *        cross-table merging when the LLM does converge on one name.
+ *
+ *   6. LABEL each cluster from its members' conceptName votes.
+ *
+ * No vocabulary discovery, no LLM cluster judge, no per-cluster refinement —
+ * the normalize step IS the PR #2193 judgment, the cluster + split steps ARE
+ * the cross-table identity proof.
+ */
+import { DEFAULT_DETECTOR_CONFIG, } from '../types/organic-keys.js';
+import { TableNormalizer, } from './ColumnNormalizer.js';
+import { ColumnClusterer } from './ColumnClusterer.js';
+import { createEmbeddingProvider } from './EmbeddingProvider.js';
+const AUDIT_COLUMN_PATTERN = /^(modified|created|updated|inserted|changed)(date|at|time|by|on)?$|^rowguid$|^timestamp$|^row_?version$|^__mj_.*$/i;
+const NON_VALUEMATCHABLE_TYPES = /(binary|blob|image|varbinary|xml|geography|geometry|hierarchyid|sql_variant)/i;
+// ─── Prefilter cardinality cutoffs ───────────────────────────────────────────
+/**
+ * A column with fewer than this many distinct values AND a uniqueness ratio below
+ * {@link MIN_UNIQUENESS_RATIO} is treated as a low-cardinality categorical (boolean/enum),
+ * not an identity-bearing column, and is dropped before clustering.
+ */
+const MIN_DISTINCT_FOR_KEY = 10;
+const MIN_UNIQUENESS_RATIO = 0.001;
+// ─── Embedding-input sample caps ─────────────────────────────────────────────
+/** At most this many sample values are appended to a column's embedding text. */
+const EMBED_SAMPLE_COUNT = 4;
+/** Each appended sample value is truncated to this many characters. */
+const EMBED_SAMPLE_MAX_CHARS = 60;
+// ─── Cluster-confidence tightness penalty ────────────────────────────────────
+/**
+ * Cluster confidence = mean member confidence − (maxIntraDistance × {@link TIGHTNESS_PENALTY_SLOPE}),
+ * with the penalty capped at {@link TIGHTNESS_PENALTY_MAX} so a loose-but-real cluster isn't
+ * over-penalized into irrelevance.
+ */
+const TIGHTNESS_PENALTY_SLOPE = 0.5;
+const TIGHTNESS_PENALTY_MAX = 0.2;
+/**
+ * Clustering sensitivity → percentile of the pairwise cosine-distance distribution used as the
+ * auto-calibrated merge threshold. A lower percentile yields tighter (stricter) clusters.
+ */
+const SENSITIVITY_PERCENTILE = {
+    strict: 1,
+    balanced: 5,
+    permissive: 15,
+};
+export async function runSemanticPhase(state, config, aiConfig, progress = () => { }) {
+    // ─── 1. Prefilter ────────────────────────────────────────────────────────
+    const candidates = prefilter(state, config);
+    progress(`semantic: ${candidates.length} columns in scope`);
+    if (candidates.length < (config.minClusterSize ?? DEFAULT_DETECTOR_CONFIG.minClusterSize)) {
+        return emptyResult(candidates.length);
+    }
+    // ─── 2. Normalize to business space (LLM per-table) ──────────────────────
+    progress('semantic: normalizing column descriptions to business space');
+    const tableInputs = groupColumnsByTable(state, candidates);
+    const normResult = await new TableNormalizer(aiConfig).normalizeAll(tableInputs, {
+        concurrency: config.refinementConcurrency ?? DEFAULT_DETECTOR_CONFIG.refinementConcurrency,
+        maxRetries: config.maxRefinementRetries ?? 2,
+        onProgress: () => { },
+    });
+    progress(`semantic: ${normResult.normalized.length} columns kept (${normResult.rejected} rejected by PR-#2193 axes)`);
+    if (normResult.normalized.length < (config.minClusterSize ?? DEFAULT_DETECTOR_CONFIG.minClusterSize)) {
+        return {
+            clusters: [],
+            tokens: normResult.tokens,
+            summary: {
+                columnsInScope: candidates.length,
+                columnsNormalized: normResult.normalized.length,
+                columnsRejectedByNormalizer: normResult.rejected,
+                clustersBeforeSplit: 0,
+                clustersFound: 0,
+                clustersDropped: 0,
+            },
+        };
+    }
+    // ─── 3. Embed the normalized descriptions ────────────────────────────────
+    const embedProvider = resolveEmbeddingProvider(config, aiConfig);
+    progress(`semantic: embedding ${normResult.normalized.length} descriptions via ${embedProvider.name}`);
+    const texts = normResult.normalized.map((n) => buildEmbeddingText(n));
+    const embeddings = await embedProvider.embed(texts);
+    // ─── 4. Cluster columns by embedding distance ────────────────────────────
+    const sensitivity = config.clusteringSensitivity ?? 'balanced';
+    const percentile = sensitivityToPercentile(sensitivity);
+    const clusterer = new ColumnClusterer({
+        mergeThreshold: config.mergeThreshold,
+        mergeThresholdPercentile: percentile,
+        minClusterSize: config.minClusterSize ?? DEFAULT_DETECTOR_CONFIG.minClusterSize,
+        minDistinctTables: config.minDistinctTables ?? DEFAULT_DETECTOR_CONFIG.minDistinctTables,
+    });
+    const clustererInputs = normResult.normalized.map((n, i) => ({
+        schema: n.schema,
+        table: n.table,
+        column: n.column,
+        embedding: embeddings[i],
+        participatesInFK: n.participatesInFK,
+        fkTarget: n.fkTarget,
+        isPrimaryKey: n.isPrimaryKey,
+    }));
+    const rawClusters = clusterer.cluster(clustererInputs);
+    progress(`semantic: ${rawClusters.length} clusters formed (threshold=${clusterer.lastResolvedThreshold.toFixed(3)})`);
+    // ─── 5. Split clusters that contain multiple distinct conceptNames ──────
+    const minClusterSize = config.minClusterSize ?? DEFAULT_DETECTOR_CONFIG.minClusterSize;
+    const minDistinctTables = config.minDistinctTables ?? DEFAULT_DETECTOR_CONFIG.minDistinctTables;
+    const { clusters: split, dropped } = splitClustersByConceptName(rawClusters, normResult.normalized, minClusterSize, minDistinctTables);
+    if (split.length !== rawClusters.length) {
+        progress(`semantic: ${split.length} clusters after concept-name split`);
+    }
+    // ─── 6. Label clusters from member votes ─────────────────────────────────
+    const clusters = split.map((rc, idx) => {
+        const memberNormalized = rc.memberIndexes.map((i) => normResult.normalized[i]);
+        return labelCluster(memberNormalized, rc.members, rc.maxIntraDistance, idx);
+    });
+    return {
+        clusters,
+        tokens: normResult.tokens,
+        summary: {
+            columnsInScope: candidates.length,
+            columnsNormalized: normResult.normalized.length,
+            columnsRejectedByNormalizer: normResult.rejected,
+            clustersBeforeSplit: rawClusters.length,
+            clustersFound: clusters.length,
+            clustersDropped: dropped,
+        },
+    };
+}
+/**
+ * Split each raw embedding cluster into sub-clusters by distinct conceptName.
+ *
+ * Embedding clustering over-merges when distinct concepts produce similar descriptions
+ * (e.g. product_id vs product_model_id vs product_category_id all describe "identifier
+ * for a product-related entity"). The LLM normalize step already distinguished them by
+ * assigning different concept names — we honor that here by splitting any cluster whose
+ * members carry multiple distinct conceptName values.
+ *
+ * Sub-clusters that fall below `minClusterSize` / `minDistinctTables` are dropped (a concept
+ * with one column in one table isn't a cross-table organic key). Returns the surviving
+ * sub-clusters (sorted largest-first) and the count of dropped sub-clusters.
+ */
+function splitClustersByConceptName(rawClusters, normalized, minClusterSize, minDistinctTables) {
+    const split = [];
+    let dropped = 0;
+    for (const rc of rawClusters) {
+        const byConcept = groupMemberIndexesByConcept(rc.memberIndexes, normalized);
+        if (byConcept.size === 1) {
+            split.push(rc);
+            continue;
+        }
+        for (const [, idxs] of byConcept) {
+            const members = idxs.map((i) => toClusterMember(normalized[i]));
+            const tableSet = new Set(members.map((m) => `${m.schema}.${m.table}`));
+            if (members.length < minClusterSize || tableSet.size < minDistinctTables) {
+                dropped++;
+                continue;
+            }
+            split.push({ memberIndexes: idxs, members, maxIntraDistance: rc.maxIntraDistance });
+        }
+    }
+    split.sort((a, b) => b.members.length - a.members.length);
+    return { clusters: split, dropped };
+}
+/**
+ * Bucket member indexes by normalized conceptName. The grouping key is lowercased with all
+ * non-alphanumeric chars stripped, collapsing LLM stochastic variants (`sales_person_id` vs
+ * `salesPersonId`) into one bucket — they're the same identifier written differently.
+ */
+function groupMemberIndexesByConcept(memberIndexes, normalized) {
+    const byConcept = new Map();
+    for (const ci of memberIndexes) {
+        const cn = (normalized[ci].conceptName || '__unnamed__').toLowerCase().replace(/[^a-z0-9]/g, '');
+        const bucket = byConcept.get(cn);
+        if (bucket)
+            bucket.push(ci);
+        else
+            byConcept.set(cn, [ci]);
+    }
+    return byConcept;
+}
+/** Project a normalized column into the cluster-member descriptor (sans per-column normalization). */
+function toClusterMember(n) {
+    return {
+        schema: n.schema,
+        table: n.table,
+        column: n.column,
+        participatesInFK: n.participatesInFK,
+        fkTarget: n.fkTarget,
+        isPrimaryKey: n.isPrimaryKey,
+    };
+}
+// ─── Helpers ────────────────────────────────────────────────────────────────
+function prefilter(state, config) {
+    const sampleValueCount = config.sampleValueCount ?? DEFAULT_DETECTOR_CONFIG.sampleValueCount;
+    const out = [];
+    for (const schema of state.schemas) {
+        for (const table of schema.tables) {
+            for (const col of table.columns) {
+                if (NON_VALUEMATCHABLE_TYPES.test(col.dataType))
+                    continue;
+                if (AUDIT_COLUMN_PATTERN.test(col.name))
+                    continue;
+                if (col.statistics) {
+                    const distinct = col.statistics.distinctCount ?? 0;
+                    const ratio = col.statistics.uniquenessRatio ?? 0;
+                    // Drop ultra-low-cardinality / near-boolean columns.
+                    if (distinct < MIN_DISTINCT_FOR_KEY && ratio < MIN_UNIQUENESS_RATIO)
+                        continue;
+                }
+                out.push({
+                    schema: schema.name,
+                    table: table.name,
+                    column: col.name,
+                    dataType: col.dataType,
+                    originalDescription: col.userDescription ?? col.description ?? '',
+                    sampleValues: extractSampleValues(col, sampleValueCount),
+                    participatesInFK: !!col.isForeignKey,
+                    fkTarget: col.foreignKeyReferences
+                        ? {
+                            schema: col.foreignKeyReferences.schema,
+                            table: col.foreignKeyReferences.table,
+                            column: col.foreignKeyReferences.referencedColumn,
+                        }
+                        : null,
+                    isPrimaryKey: !!col.isPrimaryKey,
+                });
+            }
+        }
+    }
+    return out;
+}
+function groupColumnsByTable(state, columns) {
+    const schemaDesc = new Map();
+    const tableDesc = new Map();
+    for (const s of state.schemas) {
+        schemaDesc.set(s.name, s.description);
+        for (const t of s.tables)
+            tableDesc.set(`${s.name}.${t.name}`, t.description);
+    }
+    const buckets = new Map();
+    for (const c of columns) {
+        const k = `${c.schema}.${c.table}`;
+        const b = buckets.get(k);
+        if (b)
+            b.push(c);
+        else
+            buckets.set(k, [c]);
+    }
+    return Array.from(buckets.entries()).map(([k, cols]) => {
+        const [schema, table] = k.split('.');
+        return {
+            schema,
+            table,
+            schemaDescription: schemaDesc.get(schema),
+            tableDescription: tableDesc.get(k),
+            columns: cols,
+        };
+    });
+}
+/**
+ * Build the embedding input text from a normalized column.
+ *
+ * Gemini clustering embeddings compress "database column description" text into
+ * a tight neighborhood ([0, 0.15] cosine distance). To pull distinct concepts
+ * apart in that compressed geometry, we lead with the concept name repeated
+ * three times — embedding models give more weight to repeated tokens, so a
+ * column labeled "product_category_id" lands distinguishably away from one
+ * labeled "product_id" even though their template descriptions look similar.
+ *
+ * Sample values are appended last (when present) to add concrete value-format
+ * signal that distinguishes e.g. an email from a phone even when both describe
+ * "identifying a person".
+ */
+function buildEmbeddingText(n) {
+    const concept = (n.conceptName || 'unknown').replace(/_/g, ' ');
+    const parts = [];
+    parts.push(`${concept}. ${concept}. ${concept}.`);
+    parts.push(n.normalizedDescription);
+    if (n.sampleValues && n.sampleValues.length > 0) {
+        const samples = n.sampleValues.slice(0, EMBED_SAMPLE_COUNT).map((v) => String(v).slice(0, EMBED_SAMPLE_MAX_CHARS));
+        parts.push(`Sample values: ${samples.join(' | ')}.`);
+    }
+    return parts.join(' ');
+}
+function labelCluster(members, outMembers, maxIntraDistance, index) {
+    // Cluster concept = majority concept name (ties → highest confidence).
+    const conceptVotes = new Map();
+    for (const m of members) {
+        const key = m.conceptName || 'unknown';
+        const cur = conceptVotes.get(key) ?? { count: 0, sumConf: 0 };
+        cur.count += 1;
+        cur.sumConf += m.confidence;
+        conceptVotes.set(key, cur);
+    }
+    const concept = Array.from(conceptVotes.entries())
+        .sort((a, b) => b[1].count - a[1].count || b[1].sumConf - a[1].sumConf || a[0].localeCompare(b[0]))[0][0];
+    // Per-column normalization — each member keeps its own strategy + expression so
+    // the emitted EntityOrganicKey hub row for THAT column carries the right transformation.
+    // The runtime applies each side's own expression at match time (see
+    // BuildOrganicKeyViewParams in MJCore). The cluster-level `normalization` field is
+    // computed by majority vote as a summary / fallback for legacy consumers.
+    const stratCounts = new Map();
+    for (const m of members) {
+        stratCounts.set(m.normalizationStrategy, (stratCounts.get(m.normalizationStrategy) ?? 0) + 1);
+    }
+    const normalization = Array.from(stratCounts.entries())
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0][0];
+    // Highest-confidence member's customNormalizationExpression becomes the cluster-level
+    // fallback (only used if a member doesn't carry its own — shouldn't normally happen).
+    const customFallback = members
+        .slice()
+        .sort((a, b) => b.confidence - a.confidence)
+        .find((m) => m.customNormalizationExpression)?.customNormalizationExpression;
+    // Decorate outMembers with per-column normalization. Members are aligned by index
+    // with `members` (the NormalizedColumn array) via the caller's memberIndexes mapping.
+    const decoratedMembers = outMembers.map((om, i) => {
+        const nm = members[i];
+        return {
+            ...om,
+            normalizationStrategy: nm?.normalizationStrategy,
+            customNormalizationExpression: nm?.customNormalizationExpression,
+        };
+    });
+    // Cluster confidence = mean of member confidences, lightly penalized by tightness.
+    const meanConf = members.reduce((s, n) => s + n.confidence, 0) / members.length;
+    const tightnessPenalty = Math.min(TIGHTNESS_PENALTY_MAX, maxIntraDistance * TIGHTNESS_PENALTY_SLOPE);
+    const confidence = Math.max(0, Math.min(1, meanConf - tightnessPenalty));
+    // Reasoning from the highest-confidence member.
+    const repr = members.slice().sort((a, b) => b.confidence - a.confidence)[0];
+    return {
+        id: `cluster_${index}`,
+        concept,
+        normalization,
+        customNormalizationExpression: customFallback,
+        members: decoratedMembers,
+        confidence,
+        reasoning: repr.reasoning,
+        maxIntraDistance,
+    };
+}
+function extractSampleValues(col, max) {
+    const raw = col.statistics?.sampleValues ?? col.possibleValues ?? [];
+    if (!Array.isArray(raw))
+        return [];
+    const out = [];
+    for (const v of raw) {
+        if (v == null)
+            continue;
+        const s = String(v).trim();
+        if (s.length === 0)
+            continue;
+        out.push(s);
+        if (out.length >= max)
+            break;
+    }
+    return out;
+}
+function sensitivityToPercentile(s) {
+    return SENSITIVITY_PERCENTILE[s] ?? SENSITIVITY_PERCENTILE.balanced;
+}
+function resolveEmbeddingProvider(config, aiConfig) {
+    const cfg = config.embedding ?? {};
+    const provider = (cfg.provider ?? 'openai');
+    const apiKey = aiConfig.apiKey;
+    const impl = createEmbeddingProvider({
+        provider,
+        apiKey,
+        model: cfg.model,
+        dimensions: cfg.dimensions,
+        batchSize: cfg.batchSize,
+        endpoint: cfg.endpoint,
+    });
+    return {
+        name: `${provider}:${cfg.model ?? 'default'}`,
+        embed: (texts) => impl.embed(texts),
+    };
+}
+function emptyResult(columnsInScope) {
+    return {
+        clusters: [],
+        tokens: { input: 0, output: 0, total: 0 },
+        summary: {
+            columnsInScope,
+            columnsNormalized: 0,
+            columnsRejectedByNormalizer: 0,
+            clustersBeforeSplit: 0,
+            clustersFound: 0,
+            clustersDropped: 0,
+        },
+    };
+}
+//# sourceMappingURL=SemanticPhase.js.map