@useatlas/create 0.0.6 → 0.0.7

package/templates/nextjs-standalone/bin/atlas.ts

@@ -36,7 +36,7 @@
  * tables and views are profiled automatically.
  *
  * Requires ATLAS_DATASOURCE_URL in environment.
- * Supports PostgreSQL (postgresql://...) and MySQL (mysql://...).
+ * Supports PostgreSQL, MySQL, ClickHouse, Snowflake, DuckDB, and Salesforce.
  */
 
 import { Pool } from "pg";
@@ -46,6 +46,76 @@ import * as yaml from "js-yaml";
 import * as p from "@clack/prompts";
 import pc from "picocolors";
 import { type DBType } from "@atlas/api/lib/db/connection";
+import type { DuckDBConnection } from "@duckdb/node-api";
+import type { SObjectField } from "../../../plugins/salesforce/src/connection";
+import { checkEnvFile } from "../src/env-check";
+import { type ProfileProgressCallbacks, createProgressTracker, formatDuration } from "../src/progress";
+import {
+  type DatabaseObject,
+  type ColumnProfile,
+  type ForeignKey,
+  type TableProfile,
+  type ProfileError,
+  type ProfilingResult,
+  type ProfileLogger,
+  isFatalConnectionError,
+  checkFailureThreshold,
+  isView,
+  isMatView,
+  isViewLike,
+  mapSQLType,
+  analyzeTableProfiles,
+  generateEntityYAML,
+  generateCatalogYAML,
+  generateMetricYAML,
+  generateGlossaryYAML,
+  outputDirForDatasource,
+  listPostgresObjects,
+  listMySQLObjects,
+  profilePostgres,
+  profileMySQL,
+} from "@atlas/api/lib/profiler";
+
+/** Adapts the profiler's structured logger to CLI console output. */
+const cliProfileLogger: ProfileLogger = {
+  info(_obj, msg) { console.log(`  ${msg}`); },
+  warn(obj, msg) {
+    const ctx = [obj.table, obj.column].filter(Boolean).join(".");
+    console.warn(`  Warning: ${msg}${ctx ? ` (${ctx})` : ""}${obj.err ? `: ${obj.err}` : ""}`);
+  },
+  error(obj, msg) {
+    const ctx = [obj.table, obj.column].filter(Boolean).join(".");
+    console.error(`  ${msg}${ctx ? ` (${ctx})` : ""}${obj.err ? `: ${obj.err}` : ""}`);
+  },
+};
+
+// Re-export from shared profiler for test backward compatibility
+export {
+  type ColumnProfile,
+  type TableProfile,
+  type ProfileError,
+  type ProfilingResult,
+  FATAL_ERROR_PATTERN,
+  isFatalConnectionError,
+  checkFailureThreshold,
+  generateEntityYAML,
+  generateCatalogYAML,
+  generateMetricYAML,
+  generateGlossaryYAML,
+  mapSQLType,
+  isView,
+  isMatView,
+  isViewLike,
+  entityName,
+  outputDirForDatasource,
+  inferForeignKeys,
+  detectAbandonedTables,
+  detectEnumInconsistency,
+  detectDenormalizedTables,
+  analyzeTableProfiles,
+  pluralize,
+  singularize,
+} from "@atlas/api/lib/profiler";
 
 /** CLI-local DB type detection — supports all URL schemes (core + plugin databases). */
 function detectDBType(url: string): DBType {
@@ -70,67 +140,19 @@ async function loadDuckDB() {
 const SEMANTIC_DIR = path.resolve("semantic");
 const ENTITIES_DIR = path.join(SEMANTIC_DIR, "entities");
 
-// --- Interfaces ---
-
-export type ObjectType = "table" | "view" | "materialized_view";
-
-export interface DatabaseObject {
-  name: string;
-  type: ObjectType;
-}
-
-export interface ColumnProfile {
-  name: string;
-  type: string;
-  nullable: boolean;
-  unique_count: number | null;
-  null_count: number | null;
-  sample_values: string[];
-  is_primary_key: boolean;
-  is_foreign_key: boolean;
-  fk_target_table: string | null;
-  fk_target_column: string | null;
-  is_enum_like: boolean;
-  profiler_notes: string[];
-}
-
-export interface ForeignKey {
-  from_column: string;
-  to_table: string;
-  to_column: string;
-  source: "constraint" | "inferred";
-}
-
-export interface TableProfile {
-  table_name: string;
-  object_type: ObjectType;
-  row_count: number;
-  columns: ColumnProfile[];
-  primary_key_columns: string[];
-  foreign_keys: ForeignKey[];
-  inferred_foreign_keys: ForeignKey[];
-  profiler_notes: string[];
-  table_flags: {
-    possibly_abandoned: boolean;
-    possibly_denormalized: boolean;
-  };
-  matview_populated?: boolean;
-  partition_info?: { strategy: "range" | "list" | "hash"; key: string; children: string[] };
-}
-
-/** Check whether a profile represents a database view. */
-export function isView(profile: TableProfile): boolean {
-  return profile.object_type === "view";
-}
-
-/** Check whether a profile represents a materialized view. */
-export function isMatView(profile: TableProfile): boolean {
-  return profile.object_type === "materialized_view";
-}
-
-/** Check whether a profile is view-like (view or materialized view) — skip PK/FK/measures/patterns. */
-export function isViewLike(profile: TableProfile): boolean {
-  return profile.object_type === "view" || profile.object_type === "materialized_view";
+/** Log a warning summary for profiling errors (first 5 + overflow). CLI-specific: uses console.warn formatting rather than the profiler's structured logger. */
+export function logProfilingErrors(errors: ProfileError[], total: number): void {
+  const pct = Math.round((errors.length / total) * 100);
+  console.warn(
+    `\nWarning: ${errors.length}/${total} tables (${pct}%) failed to profile:`
+  );
+  const preview = errors.slice(0, 5);
+  for (const e of preview) {
+    console.warn(`  - ${e.table}: ${e.error}`);
+  }
+  if (errors.length > 5) {
+    console.warn(`  ... and ${errors.length - 5} more`);
+  }
 }
 
 // --- Shared helpers ---
@@ -163,638 +185,11 @@ function requireFlagIdentifier(args: string[], flag: string, label: string): str
   return value;
 }
 
-// --- PostgreSQL profiler ---
-
-/** Schema-qualified table reference for SQL queries. */
-function pgTableRef(tableName: string, schema: string): string {
-  const safeTable = tableName.replace(/"/g, '""');
-  const safeSchema = schema.replace(/"/g, '""');
-  return schema === "public" ? `"${safeTable}"` : `"${safeSchema}"."${safeTable}"`;
-}
-
-async function queryPrimaryKeys(
-  pool: Pool,
-  tableName: string,
-  schema: string = "public"
-): Promise<string[]> {
-  const result = await pool.query(
-    `
-    SELECT a.attname AS column_name
-    FROM pg_constraint c
-    JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY(c.conkey)
-    WHERE c.contype = 'p'
-      AND c.conrelid = $1::regclass
-    ORDER BY a.attnum
-    `,
-    [pgTableRef(tableName, schema)]
-  );
-  return result.rows.map((r: { column_name: string }) => r.column_name);
-}
-
-async function queryForeignKeys(
-  pool: Pool,
-  tableName: string,
-  schema: string = "public"
-): Promise<ForeignKey[]> {
-  const result = await pool.query(
-    `
-    SELECT
-      a_from.attname AS from_column,
-      cl_to.relname AS to_table,
-      a_to.attname AS to_column,
-      ns_to.nspname AS to_schema
-    FROM pg_constraint c
-    JOIN pg_attribute a_from
-      ON a_from.attrelid = c.conrelid AND a_from.attnum = ANY(c.conkey)
-    JOIN pg_class cl_to
-      ON cl_to.oid = c.confrelid
-    JOIN pg_namespace ns_to
-      ON ns_to.oid = cl_to.relnamespace
-    JOIN pg_attribute a_to
-      ON a_to.attrelid = c.confrelid AND a_to.attnum = ANY(c.confkey)
-    WHERE c.contype = 'f'
-      AND c.conrelid = $1::regclass
-    ORDER BY a_from.attname
-    `,
-    [pgTableRef(tableName, schema)]
-  );
-  return result.rows.map(
-    (r: { from_column: string; to_table: string; to_column: string; to_schema: string }) => ({
-      from_column: r.from_column,
-      // Qualify FK target with schema when it differs from the profiled schema
-      to_table: r.to_schema !== schema ? `${r.to_schema}.${r.to_table}` : r.to_table,
-      to_column: r.to_column,
-      source: "constraint" as const,
-    })
-  );
-}
-
-export async function listPostgresObjects(connectionString: string, schema: string = "public"): Promise<DatabaseObject[]> {
-  const pool = new Pool({ connectionString, max: 1, connectionTimeoutMillis: 5000 });
-  try {
-    const result = await pool.query(
-      `SELECT table_name, table_type FROM information_schema.tables
-       WHERE table_schema = $1 AND table_type IN ('BASE TABLE', 'VIEW')
-       ORDER BY table_name`,
-      [schema]
-    );
-    const objects: DatabaseObject[] = result.rows.map((r: { table_name: string; table_type: string }) => ({
-      name: r.table_name,
-      type: r.table_type === "VIEW" ? "view" as const : "table" as const,
-    }));
-
-    // Materialized views are not in information_schema.tables — query pg_class directly
-    try {
-      const matviewResult = await pool.query(
-        `SELECT c.relname AS table_name
-         FROM pg_class c
-         JOIN pg_namespace n ON n.oid = c.relnamespace
-         WHERE n.nspname = $1 AND c.relkind = 'm'
-         ORDER BY c.relname`,
-        [schema]
-      );
-      for (const r of matviewResult.rows as { table_name: string }[]) {
-        objects.push({ name: r.table_name, type: "materialized_view" });
-      }
-    } catch (mvErr) {
-      console.warn(`  Warning: Could not discover materialized views: ${mvErr instanceof Error ? mvErr.message : String(mvErr)}`);
-    }
-
-    return objects.sort((a, b) => a.name.localeCompare(b.name));
-  } finally {
-    await pool.end();
-  }
-}
-
-export async function listMySQLObjects(connectionString: string): Promise<DatabaseObject[]> {
-  // eslint-disable-next-line @typescript-eslint/no-require-imports
-  const mysql = require("mysql2/promise");
-  const pool = mysql.createPool({
-    uri: connectionString,
-    connectionLimit: 1,
-    connectTimeout: 5000,
-  });
-  try {
-    const [rows] = await pool.execute(
-      `SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES
-       WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE IN ('BASE TABLE', 'VIEW')
-       ORDER BY TABLE_NAME`
-    );
-    return (rows as { TABLE_NAME: string; TABLE_TYPE: string }[]).map((r) => ({
-      name: r.TABLE_NAME,
-      type: r.TABLE_TYPE === "VIEW" ? "view" as const : "table" as const,
-    }));
-  } finally {
-    await pool.end();
-  }
-}
-
-export async function profilePostgres(
-  connectionString: string,
-  filterTables?: string[],
-  prefetchedObjects?: DatabaseObject[],
-  schema: string = "public"
-): Promise<TableProfile[]> {
-  const pool = new Pool({ connectionString, max: 3 });
-  const profiles: TableProfile[] = [];
-  const errors: { table: string; error: string }[] = [];
-
-  let allObjects: DatabaseObject[];
-  if (prefetchedObjects) {
-    allObjects = prefetchedObjects;
-  } else {
-    const tablesResult = await pool.query(
-      `SELECT table_name, table_type FROM information_schema.tables
-       WHERE table_schema = $1 AND table_type IN ('BASE TABLE', 'VIEW')
-       ORDER BY table_name`,
-      [schema]
-    );
-    allObjects = tablesResult.rows.map((r: { table_name: string; table_type: string }) => ({
-      name: r.table_name,
-      type: r.table_type === "VIEW" ? "view" as const : "table" as const,
-    }));
-
-    // Materialized views are not in information_schema.tables — query pg_class directly
-    try {
-      const matviewResult = await pool.query(
-        `SELECT c.relname AS table_name
-         FROM pg_class c
-         JOIN pg_namespace n ON n.oid = c.relnamespace
-         WHERE n.nspname = $1 AND c.relkind = 'm'
-         ORDER BY c.relname`,
-        [schema]
-      );
-      for (const r of matviewResult.rows as { table_name: string }[]) {
-        allObjects.push({ name: r.table_name, type: "materialized_view" });
-      }
-    } catch (mvErr) {
-      console.warn(`  Warning: Could not discover materialized views: ${mvErr instanceof Error ? mvErr.message : String(mvErr)}`);
-    }
-    allObjects.sort((a, b) => a.name.localeCompare(b.name));
-  }
-
-  const objectsToProfile = filterTables
-    ? allObjects.filter((o) => filterTables.includes(o.name))
-    : allObjects;
-
-  for (const [i, obj] of objectsToProfile.entries()) {
-    const table_name = obj.name;
-    const objectType = obj.type;
-    const objectLabel = objectType === "view" ? " [view]" : objectType === "materialized_view" ? " [matview]" : "";
-    console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${table_name}${objectLabel}...`);
-
-    try {
-      // Check matview populated status BEFORE COUNT(*) — unpopulated matviews throw on scan
-      let matview_populated: boolean | undefined;
-      if (objectType === "materialized_view") {
-        try {
-          const mvResult = await pool.query(
-            `SELECT ispopulated FROM pg_matviews WHERE schemaname = $1 AND matviewname = $2`,
-            [schema, table_name]
-          );
-          if (mvResult.rows.length > 0) {
-            matview_populated = mvResult.rows[0].ispopulated;
-          }
-        } catch (mvErr) {
-          console.warn(`    Warning: Could not read matview status for ${table_name}: ${mvErr instanceof Error ? mvErr.message : String(mvErr)}`);
-        }
-      }
-
-      // Skip COUNT(*) for unpopulated matviews — they error on table scan
-      let rowCount: number;
-      if (matview_populated === false) {
-        rowCount = 0;
-        console.log(`    Materialized view ${table_name} is not populated — skipping data profiling`);
-      } else {
-        const countResult = await pool.query(
-          `SELECT COUNT(*) as c FROM ${pgTableRef(table_name, schema)}`
-        );
-        rowCount = parseInt(countResult.rows[0].c, 10);
-      }
-
-      // Get primary keys and foreign keys from system catalogs (skip for views and matviews)
-      let primaryKeyColumns: string[] = [];
-      let foreignKeys: ForeignKey[] = [];
-      if (objectType === "table") {
-        try {
-          primaryKeyColumns = await queryPrimaryKeys(pool, table_name, schema);
-        } catch (pkErr) {
-          console.warn(`    Warning: Could not read PK constraints for ${table_name}: ${pkErr instanceof Error ? pkErr.message : String(pkErr)}`);
-        }
-        try {
-          foreignKeys = await queryForeignKeys(pool, table_name, schema);
-        } catch (fkErr) {
-          console.warn(`    Warning: Could not read FK constraints for ${table_name}: ${fkErr instanceof Error ? fkErr.message : String(fkErr)}`);
-        }
-      }
-
-      const fkLookup = new Map(
-        foreignKeys.map((fk) => [fk.from_column, fk])
-      );
-
-      // information_schema.columns excludes materialized views in PostgreSQL,
-      // so use pg_attribute + pg_type for matviews (#255)
-      const colResult = objectType === "materialized_view"
-        ? await pool.query(
-            `
-            SELECT a.attname AS column_name,
-                   pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type,
-                   CASE WHEN a.attnotnull THEN 'NO' ELSE 'YES' END AS is_nullable
-            FROM pg_attribute a
-            JOIN pg_class c ON c.oid = a.attrelid
-            JOIN pg_namespace n ON n.oid = c.relnamespace
-            WHERE n.nspname = $2
-              AND c.relname = $1
-              AND a.attnum > 0
-              AND NOT a.attisdropped
-            ORDER BY a.attnum
-          `,
-            [table_name, schema]
-          )
-        : await pool.query(
-            `
-            SELECT column_name, data_type, is_nullable
-            FROM information_schema.columns
-            WHERE table_name = $1 AND table_schema = $2
-            ORDER BY ordinal_position
-          `,
-            [table_name, schema]
-          );
-
-      const columns: ColumnProfile[] = [];
-
-      for (const col of colResult.rows) {
-        let unique_count: number | null = null;
-        let null_count: number | null = null;
-        let sample_values: string[] = [];
-        let isEnumLike = false;
-
-        const isPK = primaryKeyColumns.includes(col.column_name);
-        const fkInfo = fkLookup.get(col.column_name);
-        const isFK = !!fkInfo;
-
-        // Skip data profiling for unpopulated matviews — no data to scan
-        if (matview_populated !== false) {
-          try {
-            const tableRef = pgTableRef(table_name, schema);
-            const uq = await pool.query(
-              `SELECT COUNT(DISTINCT "${col.column_name}") as c FROM ${tableRef}`
-            );
-            unique_count = parseInt(uq.rows[0].c, 10);
-
-            const nc = await pool.query(
-              `SELECT COUNT(*) as c FROM ${tableRef} WHERE "${col.column_name}" IS NULL`
-            );
-            null_count = parseInt(nc.rows[0].c, 10);
-
-            // For enum-like columns, get ALL distinct values; otherwise sample 10
-            const isTextType =
-              col.data_type === "text" ||
-              col.data_type === "character varying" ||
-              col.data_type === "character";
-            isEnumLike =
-              isTextType &&
-              unique_count !== null &&
-              unique_count < 20 &&
-              rowCount > 0 &&
-              unique_count / rowCount <= 0.05;
-
-            const sampleLimit = isEnumLike ? 100 : 10;
-            const sv = await pool.query(
-              `SELECT DISTINCT "${col.column_name}" as v FROM ${tableRef} WHERE "${col.column_name}" IS NOT NULL ORDER BY "${col.column_name}" LIMIT ${sampleLimit}`
-            );
-            sample_values = sv.rows.map((r: { v: unknown }) => String(r.v));
-          } catch (colErr) {
-            console.warn(`    Warning: Could not profile column ${table_name}.${col.column_name}: ${colErr instanceof Error ? colErr.message : String(colErr)}`);
-          }
-        }
-
-        columns.push({
-          name: col.column_name,
-          type: col.data_type,
-          nullable: col.is_nullable === "YES",
-          unique_count,
-          null_count,
-          sample_values,
-          is_primary_key: isPK,
-          is_foreign_key: isFK,
-          fk_target_table: fkInfo?.to_table ?? null,
-          fk_target_column: fkInfo?.to_column ?? null,
-          is_enum_like: isEnumLike,
-          profiler_notes: [],
-        });
-      }
-
-      profiles.push({
-        table_name,
-        object_type: objectType,
-        row_count: rowCount,
-        columns,
-        primary_key_columns: primaryKeyColumns,
-        foreign_keys: foreignKeys,
-        inferred_foreign_keys: [],
-        profiler_notes: [],
-        table_flags: { possibly_abandoned: false, possibly_denormalized: false },
-        ...(matview_populated !== undefined ? { matview_populated } : {}),
-      });
-    } catch (err) {
-      const msg = err instanceof Error ? err.message : String(err);
-      console.error(`  Warning: Failed to profile ${table_name}: ${msg}`);
-      errors.push({ table: table_name, error: msg });
-      continue;
-    }
-  }
-
-  // Batch-query partition metadata and attach to profiled tables
-  const partitionMap = new Map<string, { strategy: "range" | "list" | "hash"; key: string }>();
-  try {
-    const partResult = await pool.query(
-      `SELECT c.relname,
-              CASE pt.partstrat WHEN 'r' THEN 'range' WHEN 'l' THEN 'list' WHEN 'h' THEN 'hash' ELSE pt.partstrat END as strategy,
-              pg_get_partkeydef(c.oid) as partition_key
-       FROM pg_partitioned_table pt
-       JOIN pg_class c ON c.oid = pt.partrelid
-       JOIN pg_namespace n ON n.oid = c.relnamespace
-       WHERE n.nspname = $1`,
-      [schema]
-    );
-
-    for (const r of partResult.rows as { relname: string; strategy: string; partition_key: string }[]) {
-      if (r.strategy !== "range" && r.strategy !== "list" && r.strategy !== "hash") {
-        console.warn(`  Warning: Unrecognized partition strategy '${r.strategy}' for ${r.relname} — skipping`);
-        continue;
-      }
-      partitionMap.set(r.relname, { strategy: r.strategy, key: r.partition_key });
-    }
-  } catch (partErr) {
-    console.warn(`  Warning: Could not read partition metadata: ${partErr instanceof Error ? partErr.message : String(partErr)}`);
-  }
-
-  const childrenMap = new Map<string, string[]>();
-  try {
-    const childResult = await pool.query(
-      `SELECT p.relname as parent, c.relname as child
-       FROM pg_inherits i
-       JOIN pg_class c ON c.oid = i.inhrelid
-       JOIN pg_class p ON p.oid = i.inhparent
-       JOIN pg_namespace n ON n.oid = p.relnamespace
-       WHERE n.nspname = $1
-       ORDER BY p.relname, c.relname`,
-      [schema]
-    );
-    for (const r of childResult.rows as { parent: string; child: string }[]) {
-      const children = childrenMap.get(r.parent) ?? [];
-      children.push(r.child);
-      childrenMap.set(r.parent, children);
-    }
-  } catch (childErr) {
-    console.warn(`  Warning: Could not read partition children: ${childErr instanceof Error ? childErr.message : String(childErr)}`);
-  }
-
-  for (const profile of profiles) {
-    const partInfo = partitionMap.get(profile.table_name);
-    if (partInfo) {
-      profile.partition_info = {
-        strategy: partInfo.strategy,
-        key: partInfo.key,
-        children: childrenMap.get(profile.table_name) ?? [],
-      };
-    }
-  }
-
-  await pool.end();
-
-  if (errors.length > 0) {
-    console.log(`\nWarning: ${errors.length} table(s)/view(s) failed to profile:`);
-    for (const e of errors) {
-      console.log(`  - ${e.table}: ${e.error}`);
-    }
-  }
-
-  return profiles;
-}
-
-// --- MySQL profiler ---
-
-async function queryMySQLPrimaryKeys(
-  pool: import("mysql2/promise").Pool,
-  tableName: string
-): Promise<string[]> {
-  const [rows] = await pool.execute(
-    `SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE
-     WHERE TABLE_SCHEMA = DATABASE()
-       AND TABLE_NAME = ?
-       AND CONSTRAINT_NAME = 'PRIMARY'
-     ORDER BY ORDINAL_POSITION`,
-    [tableName]
-  );
-  return (rows as { COLUMN_NAME: string }[]).map((r) => r.COLUMN_NAME);
-}
-
-async function queryMySQLForeignKeys(
-  pool: import("mysql2/promise").Pool,
-  tableName: string
-): Promise<ForeignKey[]> {
-  const [rows] = await pool.execute(
-    `SELECT COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME
-     FROM information_schema.KEY_COLUMN_USAGE
-     WHERE TABLE_SCHEMA = DATABASE()
-       AND TABLE_NAME = ?
-       AND REFERENCED_TABLE_NAME IS NOT NULL
-     ORDER BY COLUMN_NAME`,
-    [tableName]
-  );
-  return (rows as { COLUMN_NAME: string; REFERENCED_TABLE_NAME: string; REFERENCED_COLUMN_NAME: string }[]).map((r) => ({
-    from_column: r.COLUMN_NAME,
-    to_table: r.REFERENCED_TABLE_NAME,
-    to_column: r.REFERENCED_COLUMN_NAME,
-    source: "constraint" as const,
-  }));
-}
-
-export async function profileMySQL(
-  connectionString: string,
-  filterTables?: string[],
-  prefetchedObjects?: DatabaseObject[]
-): Promise<TableProfile[]> {
-  // eslint-disable-next-line @typescript-eslint/no-require-imports
-  const mysql = require("mysql2/promise");
-  const pool = mysql.createPool({
-    uri: connectionString,
-    connectionLimit: 3,
-    supportBigNumbers: true,
-    bigNumberStrings: true,
-  });
-  const profiles: TableProfile[] = [];
-  const errors: { table: string; error: string }[] = [];
-
-  try {
-  let allObjects: DatabaseObject[];
-  if (prefetchedObjects) {
-    allObjects = prefetchedObjects;
-  } else {
-    const [tablesRows] = await pool.execute(
-      `SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES
-       WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE IN ('BASE TABLE', 'VIEW')
-       ORDER BY TABLE_NAME`
-    );
-    allObjects = (tablesRows as { TABLE_NAME: string; TABLE_TYPE: string }[]).map((r) => ({
-      name: r.TABLE_NAME,
-      type: r.TABLE_TYPE === "VIEW" ? "view" as const : "table" as const,
-    }));
-  }
-
-  const objectsToProfile = filterTables
-    ? allObjects.filter((o) => filterTables.includes(o.name))
-    : allObjects;
-
-  for (const [i, obj] of objectsToProfile.entries()) {
-    const table_name = obj.name;
-    const objectType = obj.type;
-    console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${table_name}${objectType === "view" ? " [view]" : ""}...`);
-
-    try {
-      const [countRows] = await pool.execute(
-        `SELECT COUNT(*) as c FROM \`${table_name}\``
-      );
-      const rowCount = parseInt(String((countRows as { c: number }[])[0].c), 10);
-
-      let primaryKeyColumns: string[] = [];
-      let foreignKeys: ForeignKey[] = [];
-      if (objectType === "table") {
-        try {
-          primaryKeyColumns = await queryMySQLPrimaryKeys(pool, table_name);
-        } catch (pkErr) {
-          console.warn(`    Warning: Could not read PK constraints for ${table_name}: ${pkErr instanceof Error ? pkErr.message : String(pkErr)}`);
-        }
-        try {
-          foreignKeys = await queryMySQLForeignKeys(pool, table_name);
-        } catch (fkErr) {
-          console.warn(`    Warning: Could not read FK constraints for ${table_name}: ${fkErr instanceof Error ? fkErr.message : String(fkErr)}`);
-        }
-      }
-
-      const fkLookup = new Map(
-        foreignKeys.map((fk) => [fk.from_column, fk])
-      );
-
-      const [colRows] = await pool.execute(
-        `SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_TYPE
-         FROM information_schema.COLUMNS
-         WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?
-         ORDER BY ORDINAL_POSITION`,
-        [table_name]
-      );
-
-      const columns: ColumnProfile[] = [];
-
-      for (const col of colRows as { COLUMN_NAME: string; DATA_TYPE: string; IS_NULLABLE: string; COLUMN_TYPE: string }[]) {
-        let unique_count: number | null = null;
-        let null_count: number | null = null;
-        let sample_values: string[] = [];
-        let isEnumLike = false;
-
-        const isPK = primaryKeyColumns.includes(col.COLUMN_NAME);
-        const fkInfo = fkLookup.get(col.COLUMN_NAME);
-        const isFK = !!fkInfo;
-
-        try {
-          const [uqRows] = await pool.execute(
-            `SELECT COUNT(DISTINCT \`${col.COLUMN_NAME}\`) as c FROM \`${table_name}\``
-          );
-          unique_count = parseInt(String((uqRows as { c: number }[])[0].c), 10);
-
-          const [ncRows] = await pool.execute(
-            `SELECT COUNT(*) as c FROM \`${table_name}\` WHERE \`${col.COLUMN_NAME}\` IS NULL`
-          );
-          null_count = parseInt(String((ncRows as { c: number }[])[0].c), 10);
-
-          // Enum-like detection: text/enum/set columns with <20 distinct values and <=5% cardinality
-          const dataType = col.DATA_TYPE.toLowerCase();
-          const isTextType =
-            dataType === "varchar" ||
-            dataType === "char" ||
-            dataType === "text" ||
-            dataType === "tinytext" ||
-            dataType === "mediumtext" ||
-            dataType === "longtext" ||
-            dataType === "enum" ||
-            dataType === "set";
-          isEnumLike =
-            isTextType &&
-            unique_count !== null &&
-            unique_count < 20 &&
-            rowCount > 0 &&
-            unique_count / rowCount <= 0.05;
-
-          const sampleLimit = isEnumLike ? 100 : 10;
-          const [svRows] = await pool.execute(
-            `SELECT DISTINCT \`${col.COLUMN_NAME}\` as v FROM \`${table_name}\` WHERE \`${col.COLUMN_NAME}\` IS NOT NULL ORDER BY \`${col.COLUMN_NAME}\` LIMIT ${sampleLimit}`
-          );
-          sample_values = (svRows as { v: unknown }[]).map((r) => String(r.v));
-        } catch (colErr) {
-          console.warn(`    Warning: Could not profile column ${table_name}.${col.COLUMN_NAME}: ${colErr instanceof Error ? colErr.message : String(colErr)}`);
-        }
-
-        columns.push({
-          name: col.COLUMN_NAME,
-          type: col.DATA_TYPE,
-          nullable: col.IS_NULLABLE === "YES",
-          unique_count,
-          null_count,
-          sample_values,
-          is_primary_key: isPK,
-          is_foreign_key: isFK,
-          fk_target_table: fkInfo?.to_table ?? null,
-          fk_target_column: fkInfo?.to_column ?? null,
-          is_enum_like: isEnumLike,
-          profiler_notes: [],
-        });
-      }
-
-      profiles.push({
-        table_name,
-        object_type: objectType,
-        row_count: rowCount,
-        columns,
-        primary_key_columns: primaryKeyColumns,
-        foreign_keys: foreignKeys,
-        inferred_foreign_keys: [],
-        profiler_notes: [],
-        table_flags: { possibly_abandoned: false, possibly_denormalized: false },
-      });
-    } catch (err) {
-      const msg = err instanceof Error ? err.message : String(err);
-      // Fail fast on connection-level errors that will affect all remaining tables
-      if (/PROTOCOL_CONNECTION_LOST|ER_SERVER_SHUTDOWN|ER_NET_READ_ERROR|ER_NET_WRITE_ERROR|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|ENOTFOUND|EPIPE|ETIMEDOUT/i.test(msg)) {
-        throw new Error(`Fatal database error while profiling ${table_name}: ${msg}`, { cause: err });
-      }
-      console.error(`  Warning: Failed to profile ${table_name}: ${msg}`);
-      errors.push({ table: table_name, error: msg });
-      continue;
-    }
-  }
-
-  } finally {
-    await pool.end().catch((err: unknown) => {
-      console.warn(`[atlas] MySQL pool cleanup warning: ${err instanceof Error ? err.message : String(err)}`);
-    });
-  }
-
-  if (errors.length > 0) {
-    console.log(`\nWarning: ${errors.length} table(s)/view(s) failed to profile:`);
-    for (const e of errors) {
-      console.log(`  - ${e.table}: ${e.error}`);
-    }
-  }
 
-  return profiles;
-}
 
 // --- ClickHouse profiler ---
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-type ClickHouseClient = { query: (opts: { query: string; format: string }) => Promise<{ json: () => Promise<any> }>; close: () => Promise<void> };
+type ClickHouseClient = { query: (opts: { query: string; format: string }) => Promise<{ json: () => Promise<{ data: Record<string, unknown>[] }> }>; close: () => Promise<void> };
 
 /** Run a single query against ClickHouse and return rows. */
 async function clickhouseQuery<T = Record<string, unknown>>(
@@ -816,7 +211,7 @@ function rewriteClickHouseUrl(url: string): string {
   return url.replace(/^clickhouses:\/\//, "https://").replace(/^clickhouse:\/\//, "http://");
 }
 
-export async function listClickHouseObjects(connectionString: string): Promise<DatabaseObject[]> {
+async function listClickHouseObjects(connectionString: string): Promise<DatabaseObject[]> {
   // eslint-disable-next-line @typescript-eslint/no-require-imports
   const { createClient } = require("@clickhouse/client");
   const client = createClient({ url: rewriteClickHouseUrl(connectionString) });
@@ -862,17 +257,18 @@ function mapClickHouseType(chType: string): string {
   return "string";
 }
 
-export async function profileClickHouse(
+async function profileClickHouse(
   connectionString: string,
   filterTables?: string[],
-  prefetchedObjects?: DatabaseObject[]
-): Promise<TableProfile[]> {
+  prefetchedObjects?: DatabaseObject[],
+  progress?: ProfileProgressCallbacks
+): Promise<ProfilingResult> {
   // eslint-disable-next-line @typescript-eslint/no-require-imports
   const { createClient } = require("@clickhouse/client");
   const client = createClient({ url: rewriteClickHouseUrl(connectionString) });
 
   const profiles: TableProfile[] = [];
-  const errors: { table: string; error: string }[] = [];
+  const errors: ProfileError[] = [];
 
   try {
     let allObjects: DatabaseObject[];
@@ -896,11 +292,18 @@ export async function profileClickHouse(
       ? allObjects.filter((o) => filterTables.includes(o.name))
       : allObjects;
 
+    progress?.onStart(objectsToProfile.length);
+
     for (const [i, obj] of objectsToProfile.entries()) {
       const table_name = obj.name;
       const objectType = obj.type;
       const safeTable = table_name.replace(/'/g, "''");
-      console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${table_name}${objectType === "view" ? " [view]" : ""}...`);
+      const objectLabel = objectType === "view" ? " [view]" : "";
+      if (progress) {
+        progress.onTableStart(table_name + objectLabel, i, objectsToProfile.length);
+      } else {
+        console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${table_name}${objectLabel}...`);
+      }
 
       try {
         const countRows = await clickhouseQuery<{ c: string }>(
@@ -916,6 +319,7 @@ export async function profileClickHouse(
           try {
             primaryKeyColumns = await queryClickHousePrimaryKeys(client, table_name);
           } catch (pkErr) {
+            if (isFatalConnectionError(pkErr)) throw pkErr;
             console.warn(`    Warning: Could not read PK columns for ${table_name}: ${pkErr instanceof Error ? pkErr.message : String(pkErr)}`);
           }
         }
@@ -974,6 +378,7 @@ export async function profileClickHouse(
             );
             sample_values = svRows.map((r) => String(r.v));
           } catch (colErr) {
+            if (isFatalConnectionError(colErr)) throw colErr;
             console.warn(`    Warning: Could not profile column ${table_name}.${col.name}: ${colErr instanceof Error ? colErr.message : String(colErr)}`);
           }
 
@@ -1004,12 +409,18 @@ export async function profileClickHouse(
           profiler_notes: [],
           table_flags: { possibly_abandoned: false, possibly_denormalized: false },
         });
+        progress?.onTableDone(table_name, i, objectsToProfile.length);
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
-        if (/ECONNRESET|ECONNREFUSED|EHOSTUNREACH|ENOTFOUND|EPIPE|ETIMEDOUT/i.test(msg)) {
+        // Fail fast on connection-level errors that will affect all remaining tables
+        if (isFatalConnectionError(err)) {
           throw new Error(`Fatal database error while profiling ${table_name}: ${msg}`, { cause: err });
         }
-        console.error(`  Warning: Failed to profile ${table_name}: ${msg}`);
+        if (progress) {
+          progress.onTableError(table_name, msg, i, objectsToProfile.length);
+        } else {
+          console.error(`  Warning: Failed to profile ${table_name}: ${msg}`);
+        }
         errors.push({ table: table_name, error: msg });
         continue;
       }
@@ -1020,14 +431,7 @@ export async function profileClickHouse(
     });
   }
 
-  if (errors.length > 0) {
-    console.log(`\nWarning: ${errors.length} table(s)/view(s) failed to profile:`);
-    for (const e of errors) {
-      console.log(`  - ${e.table}: ${e.error}`);
-    }
-  }
-
-  return profiles;
+  return { profiles, errors };
 }
 
 // --- Snowflake profiler ---
@@ -1087,7 +491,7 @@ async function createSnowflakePool(connectionString: string, max = 1) {
   return { pool, opts };
 }
 
-export async function listSnowflakeObjects(connectionString: string): Promise<DatabaseObject[]> {
+async function listSnowflakeObjects(connectionString: string): Promise<DatabaseObject[]> {
   const { pool } = await createSnowflakePool(connectionString, 1);
 
   try {
@@ -1167,16 +571,17 @@ function mapSnowflakeType(sfType: string): string {
   return "text";
 }
 
-export async function profileSnowflake(
+async function profileSnowflake(
   connectionString: string,
   filterTables?: string[],
   prefetchedObjects?: DatabaseObject[],
-): Promise<TableProfile[]> {
+  progress?: ProfileProgressCallbacks
+): Promise<ProfilingResult> {
   const { pool, opts } = await createSnowflakePool(connectionString, 3);
 
 
   const profiles: TableProfile[] = [];
-  const errors: { table: string; error: string }[] = [];
+  const errors: ProfileError[] = [];
   const escId = (name: string) => name.replace(/"/g, '""');
 
   try {
@@ -1200,10 +605,17 @@ export async function profileSnowflake(
       ? allObjects.filter((o) => filterTables.includes(o.name))
       : allObjects;
 
+    progress?.onStart(objectsToProfile.length);
+
     for (const [i, obj] of objectsToProfile.entries()) {
       const table_name = obj.name;
       const objectType = obj.type;
-      console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${table_name}${objectType === "view" ? " [view]" : ""}...`);
+      const objectLabel = objectType === "view" ? " [view]" : "";
+      if (progress) {
+        progress.onTableStart(table_name + objectLabel, i, objectsToProfile.length);
+      } else {
+        console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${table_name}${objectLabel}...`);
+      }
 
       try {
         let primaryKeyColumns: string[] = [];
@@ -1212,11 +624,13 @@ export async function profileSnowflake(
           try {
             primaryKeyColumns = await querySnowflakePrimaryKeys(pool, table_name, opts.database, opts.schema);
           } catch (pkErr) {
+            if (isFatalConnectionError(pkErr)) throw pkErr;
             console.warn(`    Warning: Could not read PK constraints for ${table_name}: ${pkErr instanceof Error ? pkErr.message : String(pkErr)}`);
           }
           try {
             foreignKeys = await querySnowflakeForeignKeys(pool, table_name, opts.database, opts.schema);
           } catch (fkErr) {
+            if (isFatalConnectionError(fkErr)) throw fkErr;
             console.warn(`    Warning: Could not read FK constraints for ${table_name}: ${fkErr instanceof Error ? fkErr.message : String(fkErr)}`);
           }
         }
@@ -1253,11 +667,13 @@ export async function profileSnowflake(
               });
             }
           } catch (bulkErr) {
+            if (isFatalConnectionError(bulkErr)) throw bulkErr;
             console.warn(`    Warning: Bulk stats query failed for ${table_name}, falling back to row count only: ${bulkErr instanceof Error ? bulkErr.message : String(bulkErr)}`);
             try {
               const countResult = await snowflakeQuery(pool, `SELECT COUNT(*) as "RC" FROM "${escId(table_name)}"`);
               rowCount = parseInt(String(countResult.rows[0]?.RC ?? "0"), 10);
             } catch (countErr) {
+              if (isFatalConnectionError(countErr)) throw countErr;
               console.warn(`    Warning: Row count query also failed for ${table_name}: ${countErr instanceof Error ? countErr.message : String(countErr)}`);
             }
           }
@@ -1266,6 +682,7 @@ export async function profileSnowflake(
             const countResult = await snowflakeQuery(pool, `SELECT COUNT(*) as "RC" FROM "${escId(table_name)}"`);
             rowCount = parseInt(String(countResult.rows[0]?.RC ?? "0"), 10);
           } catch (countErr) {
+            if (isFatalConnectionError(countErr)) throw countErr;
             console.warn(`    Warning: Row count query failed for ${table_name}: ${countErr instanceof Error ? countErr.message : String(countErr)}`);
           }
         }
@@ -1298,6 +715,7 @@ export async function profileSnowflake(
               samplesMap.get(cn)!.push(String(row.V));
             }
           } catch (sampleErr) {
+            if (isFatalConnectionError(sampleErr)) throw sampleErr;
             console.warn(`    Warning: Batched sample values query failed for ${table_name} (${colMeta.length} columns affected): ${sampleErr instanceof Error ? sampleErr.message : String(sampleErr)}`);
           }
         }
@@ -1335,12 +753,19 @@ export async function profileSnowflake(
           profiler_notes: [],
           table_flags: { possibly_abandoned: false, possibly_denormalized: false },
         });
+        progress?.onTableDone(table_name, i, objectsToProfile.length);
       } catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
-        if (/ECONNRESET|ECONNREFUSED|EHOSTUNREACH|ENOTFOUND|EPIPE|ETIMEDOUT|390100|390114|250001/i.test(msg)) {
+        // Fail fast on connection-level errors that will affect all remaining tables
+        // Snowflake-specific: 390100 = auth token expired, 390114 = auth token invalid, 250001 = connection failure
+        if (isFatalConnectionError(err) || /390100|390114|250001/.test(msg)) {
           throw new Error(`Fatal database error while profiling ${table_name}: ${msg}`, { cause: err });
         }
-        console.error(`  Warning: Failed to profile ${table_name}: ${msg}`);
+        if (progress) {
+          progress.onTableError(table_name, msg, i, objectsToProfile.length);
+        } else {
+          console.error(`  Warning: Failed to profile ${table_name}: ${msg}`);
+        }
         errors.push({ table: table_name, error: msg });
         continue;
       }
@@ -1354,58 +779,15 @@ export async function profileSnowflake(
     }
   }
 
-  if (errors.length > 0) {
-    console.log(`\nWarning: ${errors.length} table(s)/view(s) failed to profile:`);
-    for (const e of errors) {
-      console.log(`  - ${e.table}: ${e.error}`);
-    }
-  }
-
-  return profiles;
+  return { profiles, errors };
 }
 
 // --- Salesforce profiler ---
 
-/** Map Salesforce field types to semantic layer types. */
-function mapSalesforceFieldType(sfType: string): string {
-  const lower = sfType.toLowerCase();
-  switch (lower) {
-    case "int":
-    case "long":
-      return "integer";
-    case "double":
-    case "currency":
-    case "percent":
-      return "real";
-    case "boolean":
-      return "boolean";
-    case "date":
-    case "datetime":
-    case "time":
-      return "date";
-    case "string":
-    case "id":
-    case "reference":
-    case "textarea":
-    case "url":
-    case "email":
-    case "phone":
-    case "picklist":
-    case "multipicklist":
-    case "combobox":
-    case "encryptedstring":
-    case "base64":
-      return "text";
-    default:
-      return "text";
-  }
-}
-
-export async function listSalesforceObjects(connectionString: string): Promise<DatabaseObject[]> {
+async function listSalesforceObjects(connectionString: string): Promise<DatabaseObject[]> {
   const { parseSalesforceURL, createSalesforceConnection } = await import("../../../plugins/salesforce/src/connection");
   const config = parseSalesforceURL(connectionString);
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const source: any = createSalesforceConnection(config);
+  const source = createSalesforceConnection(config);
   try {
     const objects = await source.listObjects();
     return objects.map((obj: { name: string }) => ({
@@ -1417,18 +799,18 @@ export async function listSalesforceObjects(connectionString: string): Promise<D
   }
 }
 
-export async function profileSalesforce(
+async function profileSalesforce(
   connectionString: string,
   filterTables?: string[],
   prefetchedObjects?: DatabaseObject[],
-): Promise<TableProfile[]> {
+  progress?: ProfileProgressCallbacks
+): Promise<ProfilingResult> {
   const { parseSalesforceURL, createSalesforceConnection } = await import("../../../plugins/salesforce/src/connection");
   const config = parseSalesforceURL(connectionString);
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const source: any = createSalesforceConnection(config);
+  const source = createSalesforceConnection(config);
 
   const profiles: TableProfile[] = [];
-  const errors: { table: string; error: string }[] = [];
+  const errors: ProfileError[] = [];
 
   try {
     let allObjects: DatabaseObject[];
@@ -1446,9 +828,15 @@ export async function profileSalesforce(
       ? allObjects.filter((o) => filterTables.includes(o.name))
       : allObjects;
 
+    progress?.onStart(objectsToProfile.length);
+
     for (const [i, obj] of objectsToProfile.entries()) {
       const objectName = obj.name;
-      console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${objectName}...`);
+      if (progress) {
+        progress.onTableStart(objectName, i, objectsToProfile.length);
+      } else {
+        console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${objectName}...`);
+      }
 
       try {
         const desc = await source.describe(objectName);
@@ -1464,14 +852,14 @@ export async function profileSalesforce(
             rowCount = parseInt(String(countVal ?? "0"), 10);
           }
         } catch (countErr) {
+          if (isFatalConnectionError(countErr)) throw countErr;
           console.warn(`    Warning: Could not get row count for ${objectName}: ${countErr instanceof Error ? countErr.message : String(countErr)}`);
         }
 
         const foreignKeys: ForeignKey[] = [];
         const primaryKeyColumns: string[] = [];
 
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const columns: ColumnProfile[] = desc.fields.map((field: any) => {
+        const columns: ColumnProfile[] = desc.fields.map((field: SObjectField) => {
           const isPK = field.name === "Id";
           if (isPK) primaryKeyColumns.push(field.name);
 
@@ -1489,8 +877,7 @@ export async function profileSalesforce(
 
           // For picklist fields, extract active values as sample_values
           const sampleValues = isEnumLike
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            ? field.picklistValues.filter((pv: any) => pv.active).map((pv: any) => pv.value)
+            ? field.picklistValues.filter((pv) => pv.active).map((pv) => pv.value)
             : [];
 
           return {
@@ -1501,953 +888,57 @@ export async function profileSalesforce(
             null_count: null,
             sample_values: sampleValues,
             is_primary_key: isPK,
-            is_foreign_key: isFK,
-            fk_target_table: isFK ? field.referenceTo[0] : null,
-            fk_target_column: isFK ? "Id" : null,
-            is_enum_like: isEnumLike,
-            profiler_notes: [],
-          };
-        });
-
-        profiles.push({
-          table_name: objectName,
-          object_type: "table",
-          row_count: rowCount,
-          columns,
-          primary_key_columns: primaryKeyColumns,
-          foreign_keys: foreignKeys,
-          inferred_foreign_keys: [],
-          profiler_notes: [],
-          table_flags: { possibly_abandoned: false, possibly_denormalized: false },
-        });
-      } catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        if (/ECONNRESET|ECONNREFUSED|EHOSTUNREACH|ENOTFOUND|EPIPE|ETIMEDOUT/i.test(msg)) {
-          throw new Error(`Fatal Salesforce error while profiling ${objectName}: ${msg}`, { cause: err });
-        }
-        console.error(`  Warning: Failed to profile ${objectName}: ${msg}`);
-        errors.push({ table: objectName, error: msg });
-        continue;
-      }
-    }
-  } finally {
-    await source.close();
-  }
-
-  if (errors.length > 0) {
-    console.log(`\nWarning: ${errors.length} object(s) failed to profile:`);
-    for (const e of errors) {
-      console.log(`  - ${e.table}: ${e.error}`);
-    }
-  }
-
-  return profiles;
-}
-
-// --- Pluralization / singularization (shared by heuristics + YAML generation) ---
-
-/**
- * Plural → singular lookup for irregular English words.
- * Also used by YAML generation (singularize) and heuristics (inferForeignKeys).
- */
-const IRREGULAR_PLURALS: Record<string, string> = {
-  people: "person",
-  children: "child",
-  men: "man",
-  women: "woman",
-  mice: "mouse",
-  data: "datum",
-  criteria: "criterion",
-  analyses: "analysis",
-};
-
-/** Derived inverse: singular → plural, built once from IRREGULAR_PLURALS. */
-const IRREGULAR_SINGULARS_TO_PLURALS: Record<string, string> = Object.fromEntries(
-  Object.entries(IRREGULAR_PLURALS).map(([plural, singular]) => [singular, plural])
-);
-
-export function pluralize(word: string): string {
-  const lower = word.toLowerCase();
-  if (IRREGULAR_SINGULARS_TO_PLURALS[lower]) return IRREGULAR_SINGULARS_TO_PLURALS[lower];
-  if (lower.endsWith("y") && !/[aeiou]y$/i.test(lower))
-    return word.slice(0, -1) + "ies";
-  if (lower.endsWith("s") || lower.endsWith("x") || lower.endsWith("z") || lower.endsWith("sh") || lower.endsWith("ch"))
-    return word + "es";
-  return word + "s";
-}
-
-export function singularize(word: string): string {
-  const lower = word.toLowerCase();
-  if (IRREGULAR_PLURALS[lower]) return IRREGULAR_PLURALS[lower];
-  if (lower.endsWith("ies")) return word.slice(0, -3) + "y";
-  if (lower.endsWith("ses") || lower.endsWith("xes") || lower.endsWith("zes"))
-    return word.slice(0, -2);
-  if (lower.endsWith("s") && !lower.endsWith("ss") && !lower.endsWith("us") && !lower.endsWith("is")) return word.slice(0, -1);
-  return word;
-}
-
-// --- Profiler heuristics (pure functions on TableProfile[]) ---
-
-/**
- * For each `*_id` column without an existing FK constraint, try to match
- * the prefix to a table name (singular or plural). Only infer when the
- * target table has a PK column named `id`.
- */
-export function inferForeignKeys(profiles: TableProfile[]): void {
-  // Only tables (not views/matviews) can be FK targets — views have no PKs
-  const tableMap = new Map(
-    profiles.filter((p) => !isViewLike(p)).map((p) => [p.table_name, p])
-  );
-
-  for (const profile of profiles) {
-    if (isViewLike(profile)) continue;
-
-    const constrainedCols = new Set(profile.foreign_keys.map((fk) => fk.from_column));
-
-    for (const col of profile.columns) {
-      if (!col.name.endsWith("_id")) continue;
-      if (constrainedCols.has(col.name)) continue;
-      if (col.is_primary_key) continue;
-
-      const prefix = col.name.slice(0, -3); // strip "_id"
-      if (!prefix) continue;
-
-      // Try direct match, plural, singular
-      const candidates = [prefix, pluralize(prefix), singularize(prefix)];
-      let targetTable: TableProfile | undefined;
-      for (const candidate of candidates) {
-        targetTable = tableMap.get(candidate);
-        if (targetTable) break;
-      }
-
-      if (!targetTable) continue;
-
-      // Only infer when target has PK column named "id"
-      const hasPkId = targetTable.primary_key_columns.includes("id");
-      if (!hasPkId) continue;
-
-      const inferredFK: ForeignKey = {
-        from_column: col.name,
-        to_table: targetTable.table_name,
-        to_column: "id",
-        source: "inferred",
-      };
-
-      profile.inferred_foreign_keys.push(inferredFK);
-
-      col.profiler_notes.push(
-        `Likely FK to ${targetTable.table_name}.id (inferred from column name, no constraint exists)`
-      );
-    }
-  }
-}
-
-const ABANDONED_NAME_PATTERNS = [
-  /^old_/,
-  /^temp_/,
-  /^legacy_/,
-  /_legacy$/,
-  /_backup$/,
-  /_archive$/,
-  /_v\d+$/,
-];
-
-/**
- * Flag tables whose names match legacy/temp patterns AND have zero inbound FKs
- * (both constraint and inferred). Views and matviews are excluded — they cannot be abandoned.
- */
-export function detectAbandonedTables(profiles: TableProfile[]): void {
-  // Build set of tables referenced by any FK (constraint or inferred)
-  const referencedTables = new Set<string>();
-  for (const p of profiles) {
-    for (const fk of p.foreign_keys) referencedTables.add(fk.to_table);
-    for (const fk of p.inferred_foreign_keys) referencedTables.add(fk.to_table);
-  }
-
-  for (const profile of profiles) {
-    if (isViewLike(profile)) continue;
-
-    const nameMatches = ABANDONED_NAME_PATTERNS.some((pat) =>
-      pat.test(profile.table_name)
-    );
-    if (!nameMatches) continue;
-
-    const hasInboundFKs = referencedTables.has(profile.table_name);
-    if (hasInboundFKs) continue;
-
-    profile.table_flags.possibly_abandoned = true;
-    profile.profiler_notes.push(
-      `Possibly abandoned: name matches legacy/temp pattern and no other tables reference it`
-    );
-  }
-}
-
-/**
- * For enum-like columns, detect case-inconsistent values
- * (e.g., 'Technology', 'tech', 'TECHNOLOGY' all map to the same lowercase form).
- */
-export function detectEnumInconsistency(profiles: TableProfile[]): void {
-  for (const profile of profiles) {
-    for (const col of profile.columns) {
-      if (!col.is_enum_like) continue;
-      if (col.sample_values.length === 0) continue;
-
-      // Group by lowercase form
-      const groups = new Map<string, string[]>();
-      for (const val of col.sample_values) {
-        const lower = val.toLowerCase();
-        const existing = groups.get(lower) ?? [];
-        existing.push(val);
-        groups.set(lower, existing);
-      }
-
-      // Find groups with multiple original forms
-      const inconsistencies: string[] = [];
-      for (const [, originals] of groups) {
-        if (originals.length > 1) {
-          inconsistencies.push(originals.join(", "));
-        }
-      }
-
-      if (inconsistencies.length > 0) {
-        col.profiler_notes.push(
-          `Case-inconsistent enum values: [${inconsistencies.join("; ")}]. Consider using LOWER() for grouping`
-        );
-      }
-    }
-  }
-}
-
-const DENORMALIZED_NAME_PATTERNS = [
-  /_denormalized$/,
-  /_cache$/,
-  /_summary$/,
-  /_stats$/,
-  /_rollup$/,
-];
-
-/**
- * Flag tables whose names match denormalization patterns. Views and matviews are excluded.
- */
-export function detectDenormalizedTables(profiles: TableProfile[]): void {
-  for (const profile of profiles) {
-    if (isViewLike(profile)) continue;
-
-    const nameMatches = DENORMALIZED_NAME_PATTERNS.some((pat) =>
-      pat.test(profile.table_name)
-    );
-    if (!nameMatches) continue;
-
-    profile.table_flags.possibly_denormalized = true;
-    profile.profiler_notes.push(
-      `Possibly denormalized/materialized table: name matches reporting pattern. Data may duplicate other tables`
-    );
-  }
-}
-
-/**
- * Orchestrate all profiler heuristics. Initializes empty arrays/flags,
- * then runs all detectors in sequence.
- */
-export function analyzeTableProfiles(profiles: TableProfile[]): void {
-  // Reset containers on all profiles (clear any prior run)
-  for (const p of profiles) {
-    p.inferred_foreign_keys = [];
-    p.profiler_notes = [];
-    p.table_flags = { possibly_abandoned: false, possibly_denormalized: false };
-    for (const col of p.columns) {
-      col.profiler_notes = [];
-    }
-  }
-
-  inferForeignKeys(profiles);
-  detectAbandonedTables(profiles);
-  detectEnumInconsistency(profiles);
-  detectDenormalizedTables(profiles);
-}
-
-// --- Generate YAML from profile ---
-
-function entityName(tableName: string): string {
-  return tableName
-    .split("_")
-    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
-    .join("");
-}
-
-export function generateEntityYAML(
-  profile: TableProfile,
-  allProfiles: TableProfile[],
-  dbType: DBType,
-  schema: string = "public",
-  source?: string,
-): string {
-  const name = entityName(profile.table_name);
-  // DuckDB's default schema is "main" — don't qualify with it (same as Postgres "public")
-  const qualifiedTable = schema !== "public" && schema !== "main" ? `${schema}.${profile.table_name}` : profile.table_name;
-
-  // Build dimensions
-  const dimensions: Record<string, unknown>[] = profile.columns.map((col) => {
-    const dim: Record<string, unknown> = {
-      name: col.name,
-      sql: col.name,
-      type: dbType === "salesforce" ? mapSalesforceFieldType(col.type) : mapSQLType(col.type),
-    };
-
-    // Description
-    if (col.is_primary_key) {
-      dim.description = `Primary key`;
-      dim.primary_key = true;
-    } else if (col.is_foreign_key) {
-      dim.description = `Foreign key to ${col.fk_target_table}`;
-    }
-
-    if (col.unique_count !== null) dim.unique_count = col.unique_count;
-    if (col.null_count !== null && col.null_count > 0)
-      dim.null_count = col.null_count;
-    if (col.sample_values.length > 0) {
-      dim.sample_values = col.is_enum_like
-        ? col.sample_values
-        : col.sample_values.slice(0, 8);
-    }
-
-    return dim;
-  });
-
-  // Build virtual dimensions — dialect-aware CASE bucketing and date extractions
-  const virtualDims: Record<string, unknown>[] = [];
-  for (const col of profile.columns) {
-    if (col.is_primary_key || col.is_foreign_key) continue;
-    const mappedType = dbType === "salesforce" ? mapSalesforceFieldType(col.type) : mapSQLType(col.type);
-
-    if (mappedType === "number" && !col.name.endsWith("_id") && dbType !== "salesforce") {
-      const label = col.name.replace(/_/g, " ");
-      if (dbType === "mysql") {
-        // MySQL: simple fixed-boundary bucketing (no PERCENTILE_CONT)
-        virtualDims.push({
-          name: `${col.name}_bucket`,
-          sql: `CASE\n  WHEN ${col.name} IS NULL THEN 'Unknown'\n  WHEN ${col.name} < (SELECT AVG(${col.name}) * 0.5 FROM ${qualifiedTable}) THEN 'Low'\n  WHEN ${col.name} < (SELECT AVG(${col.name}) * 1.5 FROM ${qualifiedTable}) THEN 'Medium'\n  ELSE 'High'\nEND`,
-          type: "string",
-          description: `${label} bucketed into Low/Medium/High`,
-          virtual: true,
-          sample_values: ["Low", "Medium", "High"],
-        });
-      } else if (dbType === "clickhouse") {
-        // ClickHouse: quantile function for tercile bucketing
-        virtualDims.push({
-          name: `${col.name}_bucket`,
-          sql: `CASE\n  WHEN ${col.name} < (SELECT quantile(0.33)(${col.name}) FROM ${qualifiedTable}) THEN 'Low'\n  WHEN ${col.name} < (SELECT quantile(0.66)(${col.name}) FROM ${qualifiedTable}) THEN 'Medium'\n  ELSE 'High'\nEND`,
-          type: "string",
-          description: `${label} bucketed into Low/Medium/High terciles`,
-          virtual: true,
-          sample_values: ["Low", "Medium", "High"],
-        });
-      } else {
-        virtualDims.push({
-          name: `${col.name}_bucket`,
-          sql: `CASE\n  WHEN ${col.name} < (SELECT PERCENTILE_CONT(0.33) WITHIN GROUP (ORDER BY ${col.name}) FROM ${qualifiedTable}) THEN 'Low'\n  WHEN ${col.name} < (SELECT PERCENTILE_CONT(0.66) WITHIN GROUP (ORDER BY ${col.name}) FROM ${qualifiedTable}) THEN 'Medium'\n  ELSE 'High'\nEND`,
-          type: "string",
-          description: `${label} bucketed into Low/Medium/High terciles`,
-          virtual: true,
-          sample_values: ["Low", "Medium", "High"],
-        });
-      }
-    }
-
-    if (mappedType === "date") {
-      if (dbType === "mysql") {
-        virtualDims.push({
-          name: `${col.name}_year`,
-          sql: `YEAR(${col.name})`,
-          type: "number",
-          description: `Year extracted from ${col.name}`,
-          virtual: true,
-        });
-        virtualDims.push({
-          name: `${col.name}_month`,
-          sql: `DATE_FORMAT(${col.name}, '%Y-%m')`,
-          type: "string",
-          description: `Year-month extracted from ${col.name}`,
-          virtual: true,
-        });
-      } else if (dbType === "clickhouse") {
-        virtualDims.push({
-          name: `${col.name}_year`,
-          sql: `toYear(${col.name})`,
-          type: "number",
-          description: `Year extracted from ${col.name}`,
-          virtual: true,
-        });
-        virtualDims.push({
-          name: `${col.name}_month`,
-          sql: `formatDateTime(${col.name}, '%Y-%m')`,
-          type: "string",
-          description: `Year-month extracted from ${col.name}`,
-          virtual: true,
-        });
-      } else if (dbType === "salesforce") {
-        virtualDims.push({
-          name: `${col.name}_year`,
-          sql: `CALENDAR_YEAR(${col.name})`,
-          type: "number",
-          description: `Year extracted from ${col.name}`,
-          virtual: true,
-        });
-        virtualDims.push({
-          name: `${col.name}_month`,
-          sql: `CALENDAR_MONTH(${col.name})`,
-          type: "number",
-          description: `Month extracted from ${col.name}`,
-          virtual: true,
-        });
-      } else {
-        virtualDims.push({
-          name: `${col.name}_year`,
-          sql: `EXTRACT(YEAR FROM ${col.name})`,
-          type: "number",
-          description: `Year extracted from ${col.name}`,
-          virtual: true,
-        });
-        virtualDims.push({
-          name: `${col.name}_month`,
-          sql: `TO_CHAR(${col.name}, 'YYYY-MM')`,
-          type: "string",
-          description: `Year-month extracted from ${col.name}`,
-          virtual: true,
-        });
-      }
-    }
-  }
-
-  // Emit profiler_notes on dimensions
-  for (const dim of dimensions) {
-    const col = profile.columns.find((c) => c.name === dim.name);
-    if (col?.profiler_notes && col.profiler_notes.length > 0) {
-      dim.profiler_notes = col.profiler_notes;
-    }
-  }
-
-  // Build joins from constraint FKs
-  const joins: Record<string, unknown>[] = profile.foreign_keys.map((fk) => ({
-    target_entity: entityName(fk.to_table),
-    relationship: "many_to_one",
-    join_columns: {
-      from: fk.from_column,
-      to: fk.to_column,
-    },
-    description: `Each ${singularize(profile.table_name)} belongs to one ${singularize(fk.to_table)}`,
-  }));
-
-  // Add inferred joins
-  for (const fk of profile.inferred_foreign_keys) {
-    joins.push({
-      target_entity: entityName(fk.to_table),
-      relationship: "many_to_one",
-      join_columns: {
-        from: fk.from_column,
-        to: fk.to_column,
-      },
-      inferred: true,
-      note: `No FK constraint exists — inferred from column name ${fk.from_column}`,
-      description: `Each ${singularize(profile.table_name)} likely belongs to one ${singularize(fk.to_table)}`,
-    });
-  }
-
-  // Build measures (skip for views/matviews — they are pre-aggregated or derived; measures should reference source tables instead)
-  const measures: Record<string, unknown>[] = [];
-
-  if (!isViewLike(profile)) {
-    // count_distinct on PK
-    const pkCol = profile.columns.find((c) => c.is_primary_key);
-    if (pkCol) {
-      measures.push({
-        name: `${singularize(profile.table_name)}_count`,
-        sql: pkCol.name,
-        type: "count_distinct",
-      });
-    }
-
-    // sum/avg on numeric non-FK non-PK columns
-    for (const col of profile.columns) {
-      if (col.is_primary_key || col.is_foreign_key) continue;
-      if (col.name.endsWith("_id")) continue;
-      const mappedType = mapSQLType(col.type);
-      if (mappedType !== "number") continue;
-
-      measures.push({
-        name: `total_${col.name}`,
-        sql: col.name,
-        type: "sum",
-        description: `Sum of ${col.name.replace(/_/g, " ")}`,
-      });
-      measures.push({
-        name: `avg_${col.name}`,
-        sql: col.name,
-        type: "avg",
-        description: `Average ${col.name.replace(/_/g, " ")}`,
-      });
-    }
-  }
-
-  // Build use_cases
-  const useCases: string[] = [];
-
-  // Note for views
-  if (isView(profile)) {
-    useCases.push(`This is a database view — it may encapsulate complex joins or aggregations. Query it directly rather than recreating its logic`);
-  }
-
-  // Notes for materialized views
-  if (isMatView(profile)) {
-    useCases.push(`WARNING: This is a materialized view — data may be stale. Check with the user about refresh frequency before relying on real-time accuracy`);
-    if (profile.matview_populated === false) {
-      useCases.push(`WARNING: This materialized view has never been refreshed and contains no data`);
-    }
-  }
-
-  // Note for partitioned tables
-  if (profile.partition_info) {
-    useCases.push(`This table is partitioned by ${profile.partition_info.strategy} on (${profile.partition_info.key}). Always include ${profile.partition_info.key} in WHERE clauses for optimal query performance`);
-  }
-
-  // Prepend warnings for flagged tables
-  if (profile.table_flags.possibly_abandoned) {
-    useCases.push(`WARNING: This table appears to be abandoned/legacy. Verify with the user before querying`);
-  }
-  if (profile.table_flags.possibly_denormalized) {
-    useCases.push(`WARNING: This is a denormalized/materialized table. Data may be stale or duplicate other tables`);
-  }
-
-  const enumCols = profile.columns.filter((c) => c.is_enum_like);
-  const numericCols = profile.columns.filter(
-    (c) =>
-      mapSQLType(c.type) === "number" && !c.is_primary_key && !c.is_foreign_key && !c.name.endsWith("_id")
-  );
-  const dateCols = profile.columns.filter(
-    (c) => mapSQLType(c.type) === "date"
-  );
-
-  if (enumCols.length > 0)
-    useCases.push(
-      `Use for segmentation analysis by ${enumCols.map((c) => c.name).join(", ")}`
-    );
-  if (numericCols.length > 0)
-    useCases.push(
-      `Use for aggregation and trends on ${numericCols.map((c) => c.name).join(", ")}`
-    );
-  if (dateCols.length > 0)
-    useCases.push(`Use for time-series analysis using ${dateCols.map((c) => c.name).join(", ")}`);
-
-  // Combined FK list for use_cases
-  const allFKs = [...profile.foreign_keys, ...profile.inferred_foreign_keys];
-  if (joins.length > 0) {
-    const targets = allFKs.map((fk) => fk.to_table);
-    const uniqueTargets = [...new Set(targets)];
-    useCases.push(
-      `Join with ${uniqueTargets.join(", ")} for cross-entity analysis`
-    );
-  }
-  // Add "avoid" guidance for related tables (constraint + inferred)
-  const tablesPointingHere = allProfiles.filter((p) =>
-    [...p.foreign_keys, ...p.inferred_foreign_keys].some((fk) => fk.to_table === profile.table_name)
-  );
-  if (tablesPointingHere.length > 0) {
-    useCases.push(
-      `Avoid for row-level ${tablesPointingHere.map((p) => p.table_name).join("/")} queries — use those entities directly`
-    );
-  }
-  if (useCases.length === 0) {
-    useCases.push(`Use for querying ${profile.table_name} data`);
-  }
-
-  // Build query patterns (skip for views/matviews — the view IS the pattern)
-  const queryPatterns: Record<string, unknown>[] = [];
-
-  if (!isViewLike(profile)) {
-    // Pattern: count by enum column
-    for (const col of enumCols.slice(0, 2)) {
-      queryPatterns.push({
-        description: `${entityName(profile.table_name)} by ${col.name}`,
-        sql: `SELECT ${col.name}, COUNT(*) as count\nFROM ${qualifiedTable}\nGROUP BY ${col.name}\nORDER BY count DESC`,
-      });
-    }
-
-    // Pattern: aggregate numeric by enum
-    if (numericCols.length > 0 && enumCols.length > 0) {
-      const numCol = numericCols[0];
-      const enumCol = enumCols[0];
-      queryPatterns.push({
-        description: `Total ${numCol.name} by ${enumCol.name}`,
-        sql: `SELECT ${enumCol.name}, SUM(${numCol.name}) as total_${numCol.name}, COUNT(*) as count\nFROM ${qualifiedTable}\nGROUP BY ${enumCol.name}\nORDER BY total_${numCol.name} DESC`,
-      });
-    }
-  }
-
-  // Build description with optional suffix for flagged tables
-  const profileIsViewLike = isViewLike(profile);
-  const profileIsMatView = isMatView(profile);
-  let description: string;
-  if (profileIsMatView) {
-    description = `Materialized view: ${profile.table_name} (${profile.row_count.toLocaleString()} rows). Contains ${profile.columns.length} columns.`;
-  } else if (isView(profile)) {
-    description = `Database view: ${profile.table_name} (${profile.row_count.toLocaleString()} rows). Contains ${profile.columns.length} columns.`;
-  } else {
-    description = `Auto-profiled schema for ${profile.table_name} (${profile.row_count.toLocaleString()} rows). Contains ${profile.columns.length} columns${allFKs.length > 0 ? `, linked to ${[...new Set(allFKs.map((fk) => fk.to_table))].join(", ")}` : ""}.`;
-  }
-  if (profile.table_flags.possibly_abandoned) {
-    description += ` POSSIBLY ABANDONED — name matches legacy/temp pattern and no tables reference it.`;
-  }
-  if (profile.table_flags.possibly_denormalized) {
-    description += ` DENORMALIZED — data may duplicate other tables.`;
-  }
-
-  // Determine entity type
-  let entityType: string;
-  if (profileIsMatView) {
-    entityType = "materialized_view";
-  } else if (isView(profile)) {
-    entityType = "view";
-  } else {
-    entityType = "fact_table";
-  }
-
-  // Assemble entity
-  const entity: Record<string, unknown> = {
-    name,
-    type: entityType,
-    table: qualifiedTable,
-    ...(source ? { connection: source } : {}),
-    grain: profileIsMatView
-      ? `one row per result from ${profile.table_name} materialized view`
-      : profileIsViewLike
-        ? `one row per result from ${profile.table_name} view`
-        : `one row per ${singularize(profile.table_name).replace(/_/g, " ")} record`,
-    description,
-    dimensions: [...dimensions, ...virtualDims],
-  };
-
-  // Partition metadata
-  if (profile.partition_info) {
-    entity.partitioned = true;
-    entity.partition_strategy = profile.partition_info.strategy;
-    entity.partition_key = profile.partition_info.key;
-  }
-
-  if (measures.length > 0) entity.measures = measures;
-  if (joins.length > 0) entity.joins = joins;
-  entity.use_cases = useCases;
-  if (queryPatterns.length > 0) entity.query_patterns = queryPatterns;
-
-  // Emit table-level profiler notes
-  if (profile.profiler_notes.length > 0) {
-    entity.profiler_notes = profile.profiler_notes;
-  }
-
-  return yaml.dump(entity, { lineWidth: 120, noRefs: true });
-}
-
-export function generateCatalogYAML(profiles: TableProfile[]): string {
-  const catalog: Record<string, unknown> = {
-    version: "1.0",
-    entities: profiles.map((p) => {
-      const enumCols = p.columns.filter((c) => c.is_enum_like);
-      const numericCols = p.columns.filter(
-        (c) =>
-          mapSQLType(c.type) === "number" && !c.is_primary_key && !c.is_foreign_key && !c.name.endsWith("_id")
-      );
-
-      // Generate use_for from table characteristics
-      const useFor: string[] = [];
-      if (enumCols.length > 0) {
-        useFor.push(
-          `Segmentation by ${enumCols.map((c) => c.name).join(", ")}`
-        );
-      }
-      if (numericCols.length > 0) {
-        useFor.push(
-          `Aggregation on ${numericCols.map((c) => c.name).join(", ")}`
-        );
-      }
-      const allFKs = [...p.foreign_keys, ...p.inferred_foreign_keys];
-      if (allFKs.length > 0) {
-        useFor.push(
-          `Cross-entity analysis via ${[...new Set(allFKs.map((fk) => fk.to_table))].join(", ")}`
-        );
-      }
-      if (useFor.length === 0) {
-        useFor.push(`General queries on ${p.table_name}`);
-      }
-
-      // Generate common_questions from column types
-      const questions: string[] = [];
-      for (const col of enumCols.slice(0, 2)) {
-        questions.push(
-          `How many ${p.table_name} by ${col.name}?`
-        );
-      }
-      if (numericCols.length > 0) {
-        questions.push(
-          `What is the average ${numericCols[0].name} across ${p.table_name}?`
-        );
-      }
-      if (allFKs.length > 0) {
-        const fk = allFKs[0];
-        questions.push(
-          `How are ${p.table_name} distributed across ${fk.to_table}?`
-        );
-      }
-      if (questions.length === 0) {
-        questions.push(`What data is in ${p.table_name}?`);
-      }
-
-      const entryIsMatView = isMatView(p);
-      const entryIsViewLike = isViewLike(p);
-
-      let catalogDesc: string;
-      if (entryIsMatView) {
-        catalogDesc = `${p.table_name} [materialized view] (${p.row_count.toLocaleString()} rows, ${p.columns.length} columns)`;
-      } else if (isView(p)) {
-        catalogDesc = `${p.table_name} [view] (${p.row_count.toLocaleString()} rows, ${p.columns.length} columns)`;
-      } else {
-        catalogDesc = `${p.table_name} (${p.row_count.toLocaleString()} rows, ${p.columns.length} columns)`;
-      }
-      if (p.partition_info) {
-        catalogDesc += ` [partitioned by ${p.partition_info.strategy}]`;
-      }
-
-      return {
-        name: entityName(p.table_name),
-        file: `entities/${p.table_name}.yml`,
-        grain: entryIsMatView
-          ? `one row per result from ${p.table_name} materialized view`
-          : entryIsViewLike
-            ? `one row per result from ${p.table_name} view`
-            : `one row per ${singularize(p.table_name).replace(/_/g, " ")} record`,
-        description: catalogDesc,
-        use_for: useFor,
-        common_questions: questions,
-      };
-    }),
-    glossary: "glossary.yml",
-  };
-
-  // Add metrics section if we'll be generating metric files (exclude views/matviews)
-  const tablesWithNumericCols = profiles.filter((p) =>
-    !isViewLike(p) &&
-    p.columns.some(
-      (c) =>
-        mapSQLType(c.type) === "number" && !c.is_primary_key && !c.is_foreign_key && !c.name.endsWith("_id")
-    )
-  );
-  if (tablesWithNumericCols.length > 0) {
-    catalog.metrics = tablesWithNumericCols.map((p) => ({
-      file: `metrics/${p.table_name}.yml`,
-      description: `Auto-generated metrics for ${p.table_name}`,
-    }));
-  }
-
-  // Add tech_debt section for flagged tables
-  const flaggedTables: { table: string; issues: string[] }[] = [];
-  for (const p of profiles) {
-    const issues: string[] = [];
-    if (p.table_flags.possibly_abandoned) issues.push("possibly_abandoned");
-    if (p.table_flags.possibly_denormalized) issues.push("possibly_denormalized");
-    if (p.inferred_foreign_keys.length > 0) issues.push("missing_fk_constraints");
-    const hasEnumIssues = p.columns.some((c) =>
-      c.profiler_notes.some((n) => n.startsWith("Case-inconsistent"))
-    );
-    if (hasEnumIssues) issues.push("inconsistent_enums");
-    if (issues.length > 0) flaggedTables.push({ table: p.table_name, issues });
-  }
-  if (flaggedTables.length > 0) {
-    catalog.tech_debt = flaggedTables;
-  }
-
-  return yaml.dump(catalog, { lineWidth: 120, noRefs: true });
-}
-
-export function generateMetricYAML(profile: TableProfile, schema: string = "public"): string | null {
-  if (isViewLike(profile)) return null;
-
-  const numericCols = profile.columns.filter(
-    (c) =>
-      mapSQLType(c.type) === "number" &&
-      !c.is_primary_key &&
-      !c.is_foreign_key &&
-      !c.name.endsWith("_id")
-  );
-
-  if (numericCols.length === 0) return null;
-
-  const pkCol = profile.columns.find((c) => c.is_primary_key);
-  const enumCols = profile.columns.filter((c) => c.is_enum_like);
-  const qualifiedTable = schema !== "public" ? `${schema}.${profile.table_name}` : profile.table_name;
-
-  const metrics: Record<string, unknown>[] = [];
-
-  // Count metric
-  if (pkCol) {
-    metrics.push({
-      id: `${profile.table_name}_count`,
-      label: `Total ${entityName(profile.table_name)}`,
-      description: `Count of distinct ${profile.table_name} records.`,
-      type: "atomic",
-      sql: `SELECT COUNT(DISTINCT ${pkCol.name}) as count\nFROM ${qualifiedTable}`,
-      aggregation: "count_distinct",
-    });
-  }
-
-  // Sum and average for each numeric column
-  for (const col of numericCols) {
-    metrics.push({
-      id: `total_${col.name}`,
-      label: `Total ${col.name.replace(/_/g, " ")}`,
-      description: `Sum of ${col.name} across all ${profile.table_name}.`,
-      type: "atomic",
-      source: {
-        entity: entityName(profile.table_name),
-        measure: `total_${col.name}`,
-      },
-      sql: `SELECT SUM(${col.name}) as total_${col.name}\nFROM ${qualifiedTable}`,
-      aggregation: "sum",
-      objective: "maximize",
-    });
-
-    metrics.push({
-      id: `avg_${col.name}`,
-      label: `Average ${col.name.replace(/_/g, " ")}`,
-      description: `Average ${col.name} per ${singularize(profile.table_name)}.`,
-      type: "atomic",
-      sql: `SELECT AVG(${col.name}) as avg_${col.name}\nFROM ${qualifiedTable}`,
-      aggregation: "avg",
-    });
-
-    // Breakdown by first enum column if available
-    if (enumCols.length > 0) {
-      const enumCol = enumCols[0];
-      metrics.push({
-        id: `${col.name}_by_${enumCol.name}`,
-        label: `${col.name.replace(/_/g, " ")} by ${enumCol.name}`,
-        description: `${col.name} broken down by ${enumCol.name}.`,
-        type: "atomic",
-        sql: `SELECT ${enumCol.name}, SUM(${col.name}) as total_${col.name}, AVG(${col.name}) as avg_${col.name}, COUNT(*) as count\nFROM ${qualifiedTable}\nGROUP BY ${enumCol.name}\nORDER BY total_${col.name} DESC`,
-      });
-    }
-  }
-
-  return yaml.dump({ metrics }, { lineWidth: 120, noRefs: true });
-}
-
-export function generateGlossaryYAML(profiles: TableProfile[]): string {
-  const terms: Record<string, unknown> = {};
-
-  // Find columns that appear in multiple tables (ambiguous terms)
-  const columnToTables = new Map<string, string[]>();
-  for (const p of profiles) {
-    for (const col of p.columns) {
-      if (col.is_primary_key || col.is_foreign_key) continue;
-      const existing = columnToTables.get(col.name) ?? [];
-      existing.push(p.table_name);
-      columnToTables.set(col.name, existing);
-    }
-  }
-
-  for (const [colName, tables] of columnToTables) {
-    if (tables.length > 1) {
-      terms[colName] = {
-        status: "ambiguous",
-        note: `"${colName}" appears in multiple tables: ${tables.join(", ")}. ASK the user which table they mean.`,
-        possible_mappings: tables.map((t) => `${t}.${colName}`),
-      };
-    }
-  }
-
-  // Add FK relationship terms
-  for (const p of profiles) {
-    for (const fk of p.foreign_keys) {
-      const termName = fk.from_column.replace(/_id$/, "");
-      if (!terms[termName]) {
-        terms[termName] = {
-          status: "defined",
-          definition: `Refers to the ${fk.to_table} entity. Linked via ${p.table_name}.${fk.from_column} → ${fk.to_table}.${fk.to_column}.`,
-        };
-      }
-    }
-  }
+            is_foreign_key: isFK,
+            fk_target_table: isFK ? field.referenceTo[0] : null,
+            fk_target_column: isFK ? "Id" : null,
+            is_enum_like: isEnumLike,
+            profiler_notes: [],
+          };
+        });
 
-  // Add enum-like column terms
-  for (const p of profiles) {
-    for (const col of p.columns) {
-      if (col.is_enum_like && !terms[col.name]) {
-        terms[col.name] = {
-          status: "defined",
-          definition: `Categorical field on ${p.table_name}. Possible values: ${col.sample_values.join(", ")}.`,
-        };
+        profiles.push({
+          table_name: objectName,
+          object_type: "table",
+          row_count: rowCount,
+          columns,
+          primary_key_columns: primaryKeyColumns,
+          foreign_keys: foreignKeys,
+          inferred_foreign_keys: [],
+          profiler_notes: [],
+          table_flags: { possibly_abandoned: false, possibly_denormalized: false },
+        });
+        progress?.onTableDone(objectName, i, objectsToProfile.length);
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        // Fail fast on connection-level errors that will affect all remaining objects
+        if (isFatalConnectionError(err)) {
+          throw new Error(`Fatal Salesforce error while profiling ${objectName}: ${msg}`, { cause: err });
+        }
+        if (progress) {
+          progress.onTableError(objectName, msg, i, objectsToProfile.length);
+        } else {
+          console.error(`  Warning: Failed to profile ${objectName}: ${msg}`);
+        }
+        errors.push({ table: objectName, error: msg });
+        continue;
       }
     }
+  } finally {
+    await source.close();
   }
 
-  // Add ambiguous terms for columns with case-inconsistent enums
-  for (const p of profiles) {
-    for (const col of p.columns) {
-      if (!col.is_enum_like) continue;
-      const inconsistencyNote = col.profiler_notes.find((n) =>
-        n.startsWith("Case-inconsistent")
-      );
-      if (!inconsistencyNote) continue;
-
-      const termKey = `${p.table_name}.${col.name}`;
-      terms[termKey] = {
-        status: "ambiguous",
-        note: `${col.name} on ${p.table_name} has case-inconsistent values. Use LOWER(${col.name}) when grouping or filtering.`,
-        guidance: `Always wrap in LOWER() for reliable aggregation: GROUP BY LOWER(${col.name})`,
-      };
-    }
-  }
-
-  if (Object.keys(terms).length === 0) {
-    terms["example_term"] = {
-      status: "defined",
-      definition: "Replace this with your own business terms",
-    };
-  }
-
-  return yaml.dump({ terms }, { lineWidth: 120, noRefs: true });
+  return { profiles, errors };
 }
 
-export function mapSQLType(sqlType: string): string {
-  // Strip ClickHouse wrappers (Nullable, LowCardinality) before matching
-  const unwrapped = sqlType.replace(/Nullable\((.+)\)/g, "$1").replace(/LowCardinality\((.+)\)/g, "$1");
-  const t = unwrapped.toLowerCase();
-  // interval and money look like they contain "int" — handle before the numeric check
-  if (t.includes("interval") || t.includes("money")) return "string";
-  if (
-    t.includes("int") ||
-    t.includes("float") ||
-    t.includes("real") ||
-    t.includes("numeric") ||
-    t.includes("decimal") ||
-    t.includes("double") ||
-    t === "currency" ||
-    t === "percent" ||
-    t === "long"
-  )
-    return "number";
-  if (t.startsWith("bool")) return "boolean";
-  if (t.includes("date") || t.includes("time") || t.includes("timestamp"))
-    return "date";
-  return "string";
-}
 
 // --- DuckDB profiler (CSV/Parquet files) ---
 
 /** Helper to run a DuckDB query and return typed rows. */
 async function duckdbQuery<T = Record<string, unknown>>(
-  conn: unknown,
+  conn: DuckDBConnection,
   sql: string,
 ): Promise<T[]> {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const reader = await (conn as any).runAndReadAll(sql);
+  const reader = await conn.runAndReadAll(sql);
   return reader.getRowObjects() as T[];
 }
 
@@ -2488,18 +979,15 @@ export async function ingestIntoDuckDB(
         ? `read_csv_auto('${absPath.replace(/'/g, "''")}')`
         : `read_parquet('${absPath.replace(/'/g, "''")}')`
 
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      await (conn as any).run(`CREATE TABLE "${stem}" AS SELECT * FROM ${readFn}`);
+      await conn.run(`CREATE TABLE "${stem}" AS SELECT * FROM ${readFn}`);
       tableNames.push(stem);
       console.log(`  Loaded ${file.format.toUpperCase()} → table "${stem}" from ${file.path}`);
     }
     return tableNames;
   } finally {
     // DuckDB Neo API uses synchronous cleanup methods
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (conn as any).disconnectSync();
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (instance as any).closeSync();
+    conn.disconnectSync();
+    instance.closeSync();
   }
 }
 
@@ -2523,10 +1011,8 @@ export async function listDuckDBObjects(dbPath: string): Promise<DatabaseObject[
     }));
   } finally {
     // DuckDB Neo API uses synchronous cleanup methods
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (conn as any).disconnectSync();
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (instance as any).closeSync();
+    conn.disconnectSync();
+    instance.closeSync();
   }
 }
 
@@ -2548,11 +1034,13 @@ export async function profileDuckDB(
   dbPath: string,
   filterTables?: string[],
   prefetchedObjects?: DatabaseObject[],
-): Promise<TableProfile[]> {
+  progress?: ProfileProgressCallbacks
+): Promise<ProfilingResult> {
   const DuckDBInstance = await loadDuckDB();
   const instance = await DuckDBInstance.create(dbPath, { access_mode: "READ_ONLY" });
   const conn = await instance.connect();
   const profiles: TableProfile[] = [];
+  const errors: ProfileError[] = [];
 
   try {
     let allObjects: DatabaseObject[];
@@ -2566,10 +1054,17 @@ export async function profileDuckDB(
       ? allObjects.filter((o) => filterTables.includes(o.name))
       : allObjects;
 
+    progress?.onStart(objectsToProfile.length);
+
     for (const [i, obj] of objectsToProfile.entries()) {
       const tableName = obj.name;
       const objectType = obj.type;
-      console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${tableName}${objectType === "view" ? " [view]" : ""}...`);
+      const objectLabel = objectType === "view" ? " [view]" : "";
+      if (progress) {
+        progress.onTableStart(tableName + objectLabel, i, objectsToProfile.length);
+      } else {
+        console.log(`  [${i + 1}/${objectsToProfile.length}] Profiling ${tableName}${objectLabel}...`);
+      }
 
       try {
         const countRows = await duckdbQuery<{ c: number | bigint }>(conn, `SELECT COUNT(*) as c FROM "${tableName}"`);
@@ -2622,6 +1117,7 @@ export async function profileDuckDB(
               sampleValues = sampleRows.map((r) => String(r.v));
             }
           } catch (colErr) {
+            if (isFatalConnectionError(colErr)) throw colErr;
             console.warn(
               `    Warning: Could not profile column ${tableName}.${col.column_name}: ${colErr instanceof Error ? colErr.message : String(colErr)}`
             );
@@ -2657,19 +1153,28 @@ export async function profileDuckDB(
             possibly_denormalized: false,
           },
         });
+        progress?.onTableDone(tableName, i, objectsToProfile.length);
       } catch (err) {
-        console.warn(`  Warning: Failed to profile ${tableName}: ${err instanceof Error ? err.message : String(err)}`);
+        const msg = err instanceof Error ? err.message : String(err);
+        // Fail fast on connection-level errors that will affect all remaining tables
+        if (isFatalConnectionError(err)) {
+          throw new Error(`Fatal database error while profiling ${tableName}: ${msg}`, { cause: err });
+        }
+        if (progress) {
+          progress.onTableError(tableName, msg, i, objectsToProfile.length);
+        } else {
+          console.error(`  Warning: Failed to profile ${tableName}: ${msg}`);
+        }
+        errors.push({ table: tableName, error: msg });
       }
     }
   } finally {
     // DuckDB Neo API uses synchronous cleanup methods
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (conn as any).disconnectSync();
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (instance as any).closeSync();
+    conn.disconnectSync();
+    instance.closeSync();
   }
 
-  return profiles;
+  return { profiles, errors };
 }
 
 // --- Demo datasets ---
@@ -2916,7 +1421,13 @@ export function computeDiff(
 
     // Metadata differences
     const metadataChanges: string[] = [];
-    if (db.objectType && yml.objectType && db.objectType !== yml.objectType) {
+    // Only flag type changes that indicate real schema drift (e.g. table↔view).
+    // Semantic classifications like dimension_table vs fact_table are enrichment
+    // metadata — the profiler always assigns "fact_table" to non-views, so comparing
+    // it against enriched YAML produces false positives.
+    const semanticTypes = new Set(["fact_table", "dimension_table"]);
+    if (db.objectType && yml.objectType && db.objectType !== yml.objectType
+      && !(semanticTypes.has(db.objectType) && semanticTypes.has(yml.objectType))) {
       metadataChanges.push(`type changed: ${yml.objectType} → ${db.objectType}`);
     }
     if (db.partitionStrategy !== yml.partitionStrategy) {
@@ -3147,7 +1658,10 @@ export async function handleActionApproval(
       signal: AbortSignal.timeout(30_000),
     });
     if (!res.ok) {
-      const body = (await res.json().catch(() => ({}))) as Record<string, unknown>;
+      const body = (await res.json().catch(() => {
+        // intentionally ignored: error response may not be JSON; fall back to status code
+        return {};
+      })) as Record<string, unknown>;
       return { ok: false, error: (body.message as string) ?? `HTTP ${res.status}` };
     }
     const body = (await res.json()) as Record<string, unknown>;
@@ -3834,6 +2348,183 @@ async function handleQuery(args: string[]): Promise<void> {
   }
 }
 
+// --- Index CLI handler ---
+
+async function handleIndex(args: string[]): Promise<void> {
+  const statsOnly = args.includes("--stats");
+
+  if (!fs.existsSync(SEMANTIC_DIR)) {
+    console.error(pc.red("No semantic/ directory found. Run 'atlas init' first."));
+    process.exit(1);
+  }
+
+  try {
+    const { getSemanticIndexStats, buildSemanticIndex } = await import("@atlas/api/lib/semantic/search");
+
+    // Use stats-based validation — works for both default and per-source layouts
+    const stats = getSemanticIndexStats(SEMANTIC_DIR);
+
+    if (stats.entities === 0) {
+      console.error(pc.red("No valid entity YAML files found in semantic/. Run 'atlas init' first."));
+      process.exit(1);
+    }
+
+    if (statsOnly) {
+      console.log(
+        `${pc.bold("Semantic index stats:")} ` +
+        `${stats.entities} entities, ${stats.dimensions} dimensions, ` +
+        `${stats.measures} measures, ${stats.metrics} metrics, ` +
+        `${stats.glossaryTerms} glossary terms (${stats.keywords} keywords)`
+      );
+      return;
+    }
+
+    // Full rebuild — buildSemanticIndex does its own loading; stats above are for validation + display
+    const start = Date.now();
+    buildSemanticIndex(SEMANTIC_DIR);
+    const elapsed = Date.now() - start;
+
+    console.log(
+      pc.green("✓") + ` Indexed ${stats.entities} entities, ` +
+      `${stats.dimensions} dimensions, ${stats.measures} measures ` +
+      `(${stats.keywords} keywords) in ${elapsed}ms`
+    );
+  } catch (err) {
+    console.error(pc.red("Failed to build semantic index."));
+    console.error(`  ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+  }
+}
+
+// --- Learn CLI handler ---
+
+async function handleLearn(args: string[]): Promise<void> {
+  const applyMode = args.includes("--apply");
+  const runSuggestions = args.includes("--suggestions");
+  const limitArg = getFlag(args, "--limit");
+  const sinceArg = getFlag(args, "--since");
+  const sourceArg = requireFlagIdentifier(args, "--source", "source name");
+
+  // Resolve semantic directories
+  const semanticRoot = sourceArg
+    ? path.join(SEMANTIC_DIR, sourceArg)
+    : SEMANTIC_DIR;
+  const entitiesDir = sourceArg
+    ? path.join(semanticRoot, "entities")
+    : ENTITIES_DIR;
+
+  // Validate semantic layer exists
+  if (!fs.existsSync(entitiesDir)) {
+    console.error(pc.red(`No entities found at ${entitiesDir}. Run 'atlas init' first.`));
+    process.exit(1);
+  }
+
+  // Validate internal DB is configured
+  if (!process.env.DATABASE_URL) {
+    console.error(pc.red("DATABASE_URL is required for atlas learn."));
+    console.error("  The audit log is stored in the internal database.");
+    console.error("  Set DATABASE_URL=postgresql://... to enable audit log analysis.");
+    process.exit(1);
+  }
+
+  // Validate --limit
+  const limit = limitArg ? parseInt(limitArg, 10) : 1000;
+  if (Number.isNaN(limit) || limit <= 0) {
+    console.error(pc.red(`Invalid value for --limit: "${limitArg}". Expected a positive integer.`));
+    process.exit(1);
+  }
+
+  // Validate --since
+  if (sinceArg) {
+    const sinceDate = new Date(sinceArg);
+    if (Number.isNaN(sinceDate.getTime())) {
+      console.error(pc.red(`Invalid value for --since: "${sinceArg}". Expected ISO 8601 format (e.g., 2026-03-01).`));
+      process.exit(1);
+    }
+  }
+
+  console.log(`\nAtlas Learn — analyzing audit log for YAML improvements...\n`);
+
+  const { getInternalDB, closeInternalDB } = await import("@atlas/api/lib/db/internal");
+  try {
+    const { fetchAuditLog, analyzeQueries } = await import("../lib/learn/analyze");
+    const { loadEntities, loadGlossary, generateProposals, applyProposals } = await import("../lib/learn/propose");
+    const { formatDiff, formatSummary } = await import("../lib/learn/diff");
+
+    // 1. Fetch audit log
+    const pool = getInternalDB();
+    const rows = await fetchAuditLog(pool, { limit, since: sinceArg });
+
+    if (rows.length === 0) {
+      console.log(pc.yellow("No successful queries found in the audit log."));
+      console.log("  Run some queries first, then try again.");
+      return;
+    }
+
+    console.log(`  Analyzed ${pc.bold(String(rows.length))} successful queries`);
+
+    // 2. Analyze patterns
+    const analysis = analyzeQueries(rows);
+    console.log(`  Found ${pc.bold(String(analysis.patterns.length))} recurring patterns, ` +
+      `${pc.bold(String(analysis.joins.size))} join pairs, ` +
+      `${pc.bold(String(analysis.aliases.length))} column aliases`);
+
+    // 3. Load existing YAML
+    const entities = loadEntities(entitiesDir);
+    const glossaryData = loadGlossary(semanticRoot);
+
+    if (entities.size === 0) {
+      console.error(pc.red(`No valid entity YAML files found in ${entitiesDir}.`));
+      process.exit(1);
+    }
+
+    console.log(`  Comparing against ${pc.bold(String(entities.size))} entities\n`);
+
+    // 4. Generate proposals
+    const proposalSet = generateProposals(analysis, entities, glossaryData);
+
+    // 5. Output results
+    console.log(formatSummary(proposalSet));
+
+    if (proposalSet.proposals.length > 0) {
+      console.log(formatDiff(proposalSet));
+
+      if (applyMode) {
+        const { written, failed } = applyProposals(proposalSet);
+        if (written.length > 0) {
+          console.log(pc.green(`\n✓ Applied changes to ${written.length} file(s):`));
+          for (const f of written) {
+            console.log(`  ${f.replace(process.cwd() + "/", "")}`);
+          }
+        }
+        if (failed.length > 0) {
+          console.error(pc.red(`\n✗ Failed to write ${failed.length} file(s):`));
+          for (const f of failed) {
+            console.error(`  ${f.path.replace(process.cwd() + "/", "")}: ${f.error}`);
+          }
+          process.exit(1);
+        }
+      } else {
+        console.log(pc.dim("\nDry run — no files modified. Use --apply to write changes."));
+      }
+    }
+
+    if (runSuggestions) {
+      console.log("\n📊 Generating query suggestions from audit log...");
+      const { generateSuggestions } = await import("@atlas/api/lib/learn/suggestions");
+      const result = await generateSuggestions(null); // CLI runs in single-org mode
+      console.log(`  Created: ${pc.bold(String(result.created))} suggestions`);
+      console.log(`  Updated: ${pc.bold(String(result.updated))} suggestions`);
+    }
+  } catch (err) {
+    console.error(pc.red("Failed to analyze audit log."));
+    console.error(`  ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+  } finally {
+    await closeInternalDB();
+  }
+}
+
 // --- Diff CLI handler ---
 
 async function handleDiff(args: string[]): Promise<void> {
@@ -3927,8 +2618,7 @@ async function handleDiff(args: string[]): Promise<void> {
   } else if (dbType === "salesforce") {
     const { parseSalesforceURL, createSalesforceConnection } = await import("../../../plugins/salesforce/src/connection");
     const config = parseSalesforceURL(connStr);
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const source: any = createSalesforceConnection(config);
+    const source = createSalesforceConnection(config);
     try {
       const objects = await source.listObjects();
       console.log(`Connected: Salesforce (${objects.length} queryable objects)`);
@@ -3971,32 +2661,39 @@ async function handleDiff(args: string[]): Promise<void> {
   console.log(`\nProfiling ${dbType} database...\n`);
   let profiles: TableProfile[];
   try {
+    let result: ProfilingResult;
     switch (dbType) {
       case "mysql":
-        profiles = await profileMySQL(connStr, filterTables);
+        result = await profileMySQL(connStr, filterTables, undefined, undefined, cliProfileLogger);
         break;
       case "postgres":
-        profiles = await profilePostgres(connStr, filterTables, undefined, schemaArg);
+        result = await profilePostgres(connStr, filterTables, undefined, schemaArg, undefined, cliProfileLogger);
         break;
       case "clickhouse":
-        profiles = await profileClickHouse(connStr, filterTables);
+        result = await profileClickHouse(connStr, filterTables);
         break;
       case "snowflake":
-        profiles = await profileSnowflake(connStr, filterTables);
+        result = await profileSnowflake(connStr, filterTables);
         break;
       case "duckdb": {
         const { parseDuckDBUrl } = await import("../../../plugins/duckdb/src/connection");
         const duckConfig = parseDuckDBUrl(connStr);
-        profiles = await profileDuckDB(duckConfig.path, filterTables);
+        result = await profileDuckDB(duckConfig.path, filterTables);
         break;
       }
       case "salesforce":
-        profiles = await profileSalesforce(connStr, filterTables);
+        result = await profileSalesforce(connStr, filterTables);
         break;
       default: {
         throw new Error(`Unknown database type: ${dbType}`);
       }
     }
+    profiles = result.profiles;
+    if (result.errors.length > 0) {
+      const total = result.profiles.length + result.errors.length;
+      logProfilingErrors(result.errors, total);
+      console.warn(`Continuing diff with ${profiles.length} successfully profiled tables.\n`);
+    }
   } catch (err) {
     console.error(`\nError: Failed to profile database.`);
     console.error(err instanceof Error ? err.message : String(err));
@@ -4009,7 +2706,7 @@ async function handleDiff(args: string[]): Promise<void> {
   }
 
   // Run FK inference so inferred FKs are comparable
-  analyzeTableProfiles(profiles);
+  profiles = analyzeTableProfiles(profiles);
 
   // Build DB snapshots
   const dbSnapshots = new Map<string, EntitySnapshot>();
@@ -4058,7 +2755,7 @@ async function handleDiff(args: string[]): Promise<void> {
 
 // --- Profile a single datasource ---
 
-export interface ProfileDatasourceOpts {
+interface ProfileDatasourceOpts {
   id: string;              // "default", "warehouse", etc.
   url: string;
   dbType: DBType;
@@ -4067,25 +2764,18 @@ export interface ProfileDatasourceOpts {
   shouldEnrich: boolean;
   explicitEnrich: boolean;
   demoDataset: DemoDataset | null;  // null for multi-source runs (--demo is single-datasource only)
+  force: boolean;          // skip failure threshold check
+  orgId?: string;          // org-scoped mode: write to semantic/.orgs/{orgId}/
 }
 
-/**
- * Compute the output base directory for a datasource.
- * "default" → `semantic/`, anything else → `semantic/{id}/`.
- * Returns an absolute path resolved from the process working directory.
- */
-export function outputDirForDatasource(id: string): string {
-  return id === "default" ? SEMANTIC_DIR : path.join(SEMANTIC_DIR, id);
-}
-
-export interface DatasourceEntry {
+interface DatasourceEntry {
   id: string;
   url: string;
   schema: string;
 }
 
 async function profileDatasource(opts: ProfileDatasourceOpts): Promise<void> {
-  const { id, url: connStr, dbType, filterTables, shouldEnrich, explicitEnrich, demoDataset } = opts;
+  const { id, url: connStr, dbType, filterTables, shouldEnrich, explicitEnrich, demoDataset, force, orgId } = opts;
   let { schema: schemaArg } = opts;
 
   validateSchemaName(schemaArg);
@@ -4176,14 +2866,11 @@ async function profileDatasource(opts: ProfileDatasourceOpts): Promise<void> {
       const DuckDBInstance = await loadDuckDB();
       const testInstance = await DuckDBInstance.create(duckConfig.path, { access_mode: "READ_ONLY" });
       const testConn = await testInstance.connect();
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      const reader = await (testConn as any).runAndReadAll("SELECT version() as v");
+      const reader = await testConn.runAndReadAll("SELECT version() as v");
       const version = reader.getRowObjects()[0]?.v ?? "unknown";
       console.log(`Connected: DuckDB ${version}`);
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      (testConn as any).disconnectSync();
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      (testInstance as any).closeSync();
+      testConn.disconnectSync();
+      testInstance.closeSync();
     } catch (err) {
       console.error(`\nError: Cannot open DuckDB database.`);
       console.error(err instanceof Error ? err.message : String(err));
@@ -4193,8 +2880,7 @@ async function profileDatasource(opts: ProfileDatasourceOpts): Promise<void> {
   } else if (dbType === "salesforce") {
     const { parseSalesforceURL, createSalesforceConnection } = await import("../../../plugins/salesforce/src/connection");
     const config = parseSalesforceURL(connStr);
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const source: any = createSalesforceConnection(config);
+    const source = createSalesforceConnection(config);
     try {
       const objects = await source.listObjects();
       console.log(`Connected: Salesforce (${objects.length} queryable objects)`);
@@ -4232,10 +2918,10 @@ async function profileDatasource(opts: ProfileDatasourceOpts): Promise<void> {
     try {
       switch (dbType) {
         case "mysql":
-          allObjects = await listMySQLObjects(connStr);
+          allObjects = await listMySQLObjects(connStr, cliProfileLogger);
           break;
         case "postgres":
-          allObjects = await listPostgresObjects(connStr, schemaArg);
+          allObjects = await listPostgresObjects(connStr, schemaArg, cliProfileLogger);
           break;
         case "clickhouse":
           allObjects = await listClickHouseObjects(connStr);
@@ -4298,40 +2984,66 @@ async function profileDatasource(opts: ProfileDatasourceOpts): Promise<void> {
 
   console.log(`\nAtlas Init — profiling ${dbType} database...\n`);
 
-  let profiles: TableProfile[];
+  const progress = createProgressTracker();
+  const profilingStart = Date.now();
+
+  let result: ProfilingResult;
   switch (dbType) {
     case "mysql":
-      profiles = await profileMySQL(connStr, selectedTables, prefetchedObjects);
+      result = await profileMySQL(connStr, selectedTables, prefetchedObjects, progress, cliProfileLogger);
       break;
     case "postgres":
-      profiles = await profilePostgres(connStr, selectedTables, prefetchedObjects, schemaArg);
+      result = await profilePostgres(connStr, selectedTables, prefetchedObjects, schemaArg, progress, cliProfileLogger);
       break;
     case "clickhouse":
-      profiles = await profileClickHouse(connStr, selectedTables, prefetchedObjects);
+      result = await profileClickHouse(connStr, selectedTables, prefetchedObjects, progress);
       break;
     case "snowflake":
-      profiles = await profileSnowflake(connStr, selectedTables, prefetchedObjects);
+      result = await profileSnowflake(connStr, selectedTables, prefetchedObjects, progress);
       break;
     case "duckdb": {
       const { parseDuckDBUrl } = await import("../../../plugins/duckdb/src/connection");
       const duckConfig = parseDuckDBUrl(connStr);
-      profiles = await profileDuckDB(duckConfig.path, selectedTables, prefetchedObjects);
+      result = await profileDuckDB(duckConfig.path, selectedTables, prefetchedObjects, progress);
       break;
     }
     case "salesforce":
-      profiles = await profileSalesforce(connStr, selectedTables, prefetchedObjects);
+      result = await profileSalesforce(connStr, selectedTables, prefetchedObjects, progress);
       break;
     default: {
       throw new Error(`Unknown database type: ${dbType}`);
     }
   }
 
+  let { profiles } = result;
+  const { errors: profilingErrors } = result;
+  const profilingElapsed = Date.now() - profilingStart;
+  progress.onComplete(profiles.length, profilingElapsed);
+
   if (profiles.length === 0) {
     throw new Error("No tables or views were successfully profiled. Check the warnings above and verify your database permissions.");
   }
 
+  // Always warn about profiling errors
+  if (profilingErrors.length > 0) {
+    const totalAttempted = profiles.length + profilingErrors.length;
+    logProfilingErrors(profilingErrors, totalAttempted);
+
+    const { shouldAbort } = checkFailureThreshold(result, force);
+    if (shouldAbort) {
+      console.error(`\nThis usually indicates a connection or permission issue.`);
+      console.error(`Run \`atlas doctor\` to diagnose. Use \`--force\` to continue anyway.`);
+      throw new Error(
+        `Profiling failed for ${profilingErrors.length}/${totalAttempted} tables ` +
+        `(${Math.round((profilingErrors.length / totalAttempted) * 100)}%). ` +
+        `Use --force to continue anyway.`
+      );
+    }
+    console.warn(`Continuing with ${profiles.length} successfully profiled tables.\n`);
+  }
+
   // Run profiler heuristics
-  analyzeTableProfiles(profiles);
+  profiles = analyzeTableProfiles(profiles);
 
   const tableCount = profiles.filter((p) => !isViewLike(p)).length;
   const viewCount = profiles.filter((p) => isView(p)).length;
@@ -4371,7 +3083,7 @@ async function profileDatasource(opts: ProfileDatasourceOpts): Promise<void> {
   }
 
   // Compute output directories
-  const outputBase = outputDirForDatasource(id);
+  const outputBase = outputDirForDatasource(id, orgId);
   const entitiesOutDir = path.join(outputBase, "entities");
   const metricsOutDir = path.join(outputBase, "metrics");
 
@@ -4379,6 +3091,15 @@ async function profileDatasource(opts: ProfileDatasourceOpts): Promise<void> {
   fs.mkdirSync(entitiesOutDir, { recursive: true });
   fs.mkdirSync(metricsOutDir, { recursive: true });
 
+  // Clean stale entity/metric files from previous runs
+  for (const dir of [entitiesOutDir, metricsOutDir]) {
+    for (const file of fs.readdirSync(dir)) {
+      if (file.endsWith(".yml") || file.endsWith(".yaml")) {
+        fs.unlinkSync(path.join(dir, file));
+      }
+    }
+  }
+
   // Generate entity YAMLs
   console.log(`\nGenerating semantic layer...\n`);
 
@@ -4438,9 +3159,11 @@ async function profileDatasource(opts: ProfileDatasourceOpts): Promise<void> {
     }
   }
 
-  const relativeOutput = id === "default" ? "./semantic/" : `./semantic/${id}/`;
+  const relativeOutput = orgId
+    ? `./semantic/.orgs/${orgId}/`
+    : id === "default" ? "./semantic/" : `./semantic/${id}/`;
   console.log(`
-Done! Semantic layer for "${id}" is at ${relativeOutput}
+Done! Semantic layer written to ${relativeOutput} in ${formatDuration(profilingElapsed)}
 
 Generated:
   - ${profiles.length} entity YAMLs with dimensions, joins, measures, and query patterns${sourceId ? ` (connection: ${id})` : ""}
@@ -4454,6 +3177,82 @@ Next steps:
 `);
 }
 
+// --- Import ---
+
+async function handleImport(args: string[]): Promise<void> {
+  const connectionArg = getFlag(args, "--connection");
+
+  // Determine the API base URL
+  const apiUrl = process.env.ATLAS_API_URL ?? "http://localhost:3001";
+
+  // Build the import request
+  const importUrl = `${apiUrl}/api/v1/admin/semantic/org/import`;
+  const body: Record<string, string> = {};
+  if (connectionArg) body.connectionId = connectionArg;
+
+  // Determine auth header
+  const headers: Record<string, string> = { "Content-Type": "application/json" };
+  if (process.env.ATLAS_API_KEY) headers.Authorization = `Bearer ${process.env.ATLAS_API_KEY}`;
+
+  console.log("Importing semantic layer from disk to DB...\n");
+
+  try {
+    const resp = await fetch(importUrl, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(60_000),
+    });
+
+    if (!resp.ok) {
+      if (resp.status === 401 || resp.status === 403) {
+        console.error("Import failed: authentication required.");
+        console.error("  Set ATLAS_API_KEY environment variable.");
+      } else {
+        let errorMsg = `HTTP ${resp.status}`;
+        try {
+          const json = await resp.json() as { message?: string; error?: string };
+          errorMsg = json.message ?? json.error ?? errorMsg;
+        } catch {
+          // intentionally ignored: JSON parse failed, fall through to text() attempt
+          errorMsg = await resp.text().catch(() => errorMsg);
+        }
+        console.error(`Import failed: ${errorMsg}`);
+      }
+      process.exit(1);
+    }
+
+    const result = await resp.json() as { imported: number; skipped: number; errors: Array<{ file: string; reason: string }>; total: number };
+
+    console.log(`Imported: ${result.imported}`);
+    if (result.skipped > 0) {
+      console.log(`Skipped:  ${result.skipped}`);
+    }
+    console.log(`Total:    ${result.total}`);
+
+    if (result.errors.length > 0) {
+      console.log("\nErrors:");
+      for (const e of result.errors) {
+        console.log(`  ${e.file}: ${e.reason}`);
+      }
+    }
+
+    if (result.imported > 0) {
+      console.log("\nDone! Entities imported to DB. The explore tool and SQL validation will use the updated semantic layer.");
+    }
+  } catch (err) {
+    const detail = err instanceof Error ? err.message : String(err);
+    if (detail.includes("ECONNREFUSED") || detail.includes("fetch failed")) {
+      console.error(`Cannot reach Atlas API at ${apiUrl}. Is the server running?`);
+      console.error("  Start it with: bun run dev:api");
+      console.error("  Set ATLAS_API_URL if the API is not on localhost:3001");
+    } else {
+      console.error(`Import failed: ${detail}`);
+    }
+    process.exit(1);
+  }
+}
+
 // --- Migrate ---
 
 async function handleMigrate(args: string[]): Promise<void> {
@@ -4569,12 +3368,306 @@ async function handleMigrate(args: string[]): Promise<void> {
   }
 }
 
+// --- Help system ---
+
+interface SubcommandHelp {
+  description: string;
+  usage: string;
+  flags?: Array<{ flag: string; description: string }>;
+  subcommands?: Array<{ name: string; description: string }>;
+  examples?: string[];
+}
+
+function printSubcommandHelp(help: SubcommandHelp): void {
+  console.log(`${help.description}\n`);
+  console.log(`Usage: atlas ${help.usage}\n`);
+  if (help.subcommands?.length) {
+    console.log("Subcommands:");
+    const maxLen = Math.max(...help.subcommands.map((s) => s.name.length));
+    for (const s of help.subcommands) {
+      console.log(`  ${s.name.padEnd(maxLen + 2)}${s.description}`);
+    }
+    console.log();
+  }
+  if (help.flags?.length) {
+    console.log("Options:");
+    const maxLen = Math.max(...help.flags.map((f) => f.flag.length));
+    for (const f of help.flags) {
+      console.log(`  ${f.flag.padEnd(maxLen + 2)}${f.description}`);
+    }
+    console.log();
+  }
+  if (help.examples?.length) {
+    console.log("Examples:");
+    for (const ex of help.examples) {
+      console.log(`  ${ex}`);
+    }
+    console.log();
+  }
+}
+
+const SUBCOMMAND_HELP: Record<string, SubcommandHelp> = {
+  init: {
+    description: "Profile a database and generate semantic layer YAML files.",
+    usage: "init [options]",
+    flags: [
+      { flag: "--tables <t1,t2>", description: "Profile only specific tables/views (comma-separated)" },
+      { flag: "--schema <name>", description: "PostgreSQL schema name (default: public)" },
+      { flag: "--source <name>", description: "Write to semantic/{name}/ subdirectory (mutually exclusive with --connection)" },
+      { flag: "--connection <name>", description: "Profile a named datasource from atlas.config.ts (mutually exclusive with --source)" },
+      { flag: "--csv <file1.csv,...>", description: "Load CSV files via DuckDB (no DB server needed, requires @duckdb/node-api)" },
+      { flag: "--parquet <f1.parquet,...>", description: "Load Parquet files via DuckDB (requires @duckdb/node-api)" },
+      { flag: "--enrich", description: "Add LLM-enriched descriptions and query patterns (requires API key)" },
+      { flag: "--no-enrich", description: "Explicitly skip LLM enrichment" },
+      { flag: "--force", description: "Continue even if more than 20% of tables fail to profile" },
+      { flag: "--demo [simple|cybersec|ecommerce]", description: "Load a demo dataset then profile (default: simple)" },
+      { flag: "--org <orgId>", description: "Write to semantic/.orgs/{orgId}/ and auto-import to DB (org-scoped mode)" },
+      { flag: "--no-import", description: "Skip auto-import to DB in org-scoped mode (write disk only)" },
+    ],
+    examples: [
+      "atlas init",
+      "atlas init --tables users,orders,products",
+      "atlas init --enrich",
+      "atlas init --demo cybersec",
+      "atlas init --csv sales.csv,products.csv",
+      "atlas init --org org-123",
+    ],
+  },
+  diff: {
+    description: "Compare the database schema against the existing semantic layer. Exits with code 1 if drift is detected.",
+    usage: "diff [options]",
+    flags: [
+      { flag: "--tables <t1,t2>", description: "Diff only specific tables/views" },
+      { flag: "--schema <name>", description: "PostgreSQL schema (falls back to ATLAS_SCHEMA, then public)" },
+      { flag: "--source <name>", description: "Read from semantic/{name}/ subdirectory" },
+    ],
+    examples: [
+      "atlas diff",
+      "atlas diff --tables users,orders",
+      'atlas diff || echo "Schema drift detected!"',
+    ],
+  },
+  query: {
+    description: "Ask a natural language question and get an answer. Requires a running Atlas API server.",
+    usage: 'query "your question" [options]',
+    flags: [
+      { flag: "--json", description: "Raw JSON output (pipe-friendly)" },
+      { flag: "--csv", description: "CSV output (headers + rows, no narrative)" },
+      { flag: "--quiet", description: "Data only — no narrative, SQL, or stats" },
+      { flag: "--auto-approve", description: "Auto-approve any pending actions" },
+      { flag: "--connection <id>", description: "Query a specific datasource" },
+    ],
+    examples: [
+      'atlas query "How many users signed up last month?"',
+      'atlas query "top 10 customers by revenue" --json',
+      'atlas query "monthly revenue by product" --csv > report.csv',
+    ],
+  },
+  doctor: {
+    description: "Alias for 'atlas validate' — validate config, semantic layer, and connectivity.",
+    usage: "doctor",
+    examples: [
+      "atlas doctor",
+    ],
+  },
+  validate: {
+    description: "Validate config, semantic layer, and connectivity. Use --offline to skip connectivity checks.",
+    usage: "validate [options]",
+    flags: [
+      { flag: "--offline", description: "Skip connectivity checks (datasource, provider, internal DB)" },
+    ],
+    examples: [
+      "atlas validate",
+      "atlas validate --offline",
+    ],
+  },
+  mcp: {
+    description: "Start an MCP (Model Context Protocol) server for Claude Desktop, Cursor, and other MCP clients.",
+    usage: "mcp [options]",
+    flags: [
+      { flag: "--transport <stdio|sse>", description: "Transport type (default: stdio)" },
+      { flag: "--port <n>", description: "Port for SSE transport (default: 8080)" },
+    ],
+    examples: [
+      "atlas mcp",
+      "atlas mcp --transport sse --port 9090",
+    ],
+  },
+  import: {
+    description: "Import semantic layer YAML files from disk into the internal DB for the active org.",
+    usage: "import [options]",
+    flags: [
+      { flag: "--connection <name>", description: "Associate imported entities with a named datasource" },
+    ],
+    examples: [
+      "atlas import",
+      "atlas import --connection warehouse",
+    ],
+  },
+  index: {
+    description: "Rebuild the semantic index from current YAML files, or print index statistics.",
+    usage: "index [options]",
+    flags: [
+      { flag: "--stats", description: "Print current index statistics without rebuilding" },
+    ],
+    examples: [
+      "atlas index",
+      "atlas index --stats",
+    ],
+  },
+  learn: {
+    description: "Analyze audit log and propose semantic layer YAML improvements.",
+    usage: "learn [options]",
+    flags: [
+      { flag: "--apply", description: "Write proposed changes to YAML files (default: dry-run)" },
+      { flag: "--suggestions", description: "Generate query suggestions from the audit log" },
+      { flag: "--limit <n>", description: "Max audit log entries to analyze (default: 1000)" },
+      { flag: "--since <date>", description: "Only analyze queries after this date (ISO 8601)" },
+      { flag: "--source <name>", description: "Read from/write to semantic/{name}/ subdirectory" },
+    ],
+    examples: [
+      "atlas learn",
+      "atlas learn --apply",
+      "atlas learn --since 2026-03-01 --limit 500",
+      "atlas learn --source warehouse",
+    ],
+  },
+  migrate: {
+    description: "Generate or apply plugin schema migrations.",
+    usage: "migrate [options]",
+    flags: [
+      { flag: "--apply", description: "Execute migrations against internal database (default: dry-run)" },
+    ],
+    examples: [
+      "atlas migrate",
+      "atlas migrate --apply",
+    ],
+  },
+  plugin: {
+    description: "Manage Atlas plugins.",
+    usage: "plugin <list|create|add>",
+    subcommands: [
+      { name: "list", description: "List installed plugins from atlas.config.ts" },
+      { name: "create <name> --type <type>", description: "Scaffold a new plugin (datasource|context|interaction|action|sandbox)" },
+      { name: "add <package-name>", description: "Install a plugin package" },
+    ],
+    examples: [
+      "atlas plugin list",
+      "atlas plugin create my-plugin --type datasource",
+      "atlas plugin add @useatlas/plugin-bigquery",
+    ],
+  },
+  eval: {
+    description: "Run the evaluation pipeline against demo schemas to measure text-to-SQL accuracy.",
+    usage: "eval [options]",
+    flags: [
+      { flag: "--schema <name>", description: "Filter by demo dataset (not a PostgreSQL schema; e.g. simple, cybersec, ecommerce)" },
+      { flag: "--category <name>", description: "Filter by category" },
+      { flag: "--difficulty <level>", description: "Filter by difficulty (simple|medium|complex)" },
+      { flag: "--id <case-id>", description: "Run a single case" },
+      { flag: "--limit <n>", description: "Max cases to evaluate" },
+      { flag: "--resume <file>", description: "Resume from existing JSONL results file" },
+      { flag: "--baseline", description: "Save results as new baseline" },
+      { flag: "--compare <file.jsonl>", description: "Diff against baseline (exit 1 on regression)" },
+      { flag: "--csv", description: "CSV output" },
+      { flag: "--json", description: "JSON summary output" },
+    ],
+    examples: [
+      "atlas eval",
+      "atlas eval --schema cybersec --difficulty complex",
+      "atlas eval --baseline",
+    ],
+  },
+  smoke: {
+    description: "Run end-to-end smoke tests against a running Atlas deployment.",
+    usage: "smoke [options]",
+    flags: [
+      { flag: "--target <url>", description: "API base URL (default: http://localhost:3001)" },
+      { flag: "--api-key <key>", description: "Bearer auth token" },
+      { flag: "--timeout <ms>", description: "Per-check timeout (default: 30000)" },
+      { flag: "--verbose", description: "Show full response bodies on failure" },
+      { flag: "--json", description: "Machine-readable JSON output" },
+    ],
+    examples: [
+      "atlas smoke",
+      "atlas smoke --target https://api.example.com --api-key sk-...",
+    ],
+  },
+  benchmark: {
+    description: "Run the BIRD benchmark for text-to-SQL accuracy evaluation.",
+    usage: "benchmark [options]",
+    flags: [
+      { flag: "--bird-path <path>", description: "Path to the downloaded BIRD dev directory (required)" },
+      { flag: "--limit <n>", description: "Max questions to evaluate" },
+      { flag: "--db <name>", description: "Filter to a single database" },
+      { flag: "--csv", description: "CSV output" },
+      { flag: "--resume <file>", description: "Resume from existing JSONL results file" },
+    ],
+    examples: [
+      "atlas benchmark --bird-path ./bird-dev",
+      "atlas benchmark --bird-path ./bird-dev --db california_schools --limit 50",
+    ],
+  },
+  completions: {
+    description: "Output a shell completion script.",
+    usage: "completions <bash|zsh|fish>",
+    examples: [
+      'eval "$(atlas completions bash)"',
+      'eval "$(atlas completions zsh)"',
+      "atlas completions fish > ~/.config/fish/completions/atlas.fish",
+    ],
+  },
+};
+
+function printOverviewHelp(): void {
+  console.log(
+    "Atlas CLI — profile databases, generate semantic layers, and query your data.\n\n" +
+    "Usage: atlas <command> [options]\n\n" +
+    "Commands:\n" +
+    "  init          Profile DB and generate semantic layer\n" +
+    "  import        Import semantic YAML files from disk into DB\n" +
+    "  index         Rebuild or inspect the semantic index\n" +
+    "  learn         Analyze audit log and propose YAML improvements\n" +
+    "  diff          Compare DB schema against existing semantic layer\n" +
+    "  query         Ask a question via the Atlas API\n" +
+    "  validate      Validate config, semantic layer, and connectivity\n" +
+    "  doctor        Alias for validate\n" +
+    "  eval          Run eval pipeline against demo schemas\n" +
+    "  smoke         Run E2E smoke tests against a running Atlas deployment\n" +
+    "  migrate       Generate/apply plugin schema migrations\n" +
+    "  plugin        Manage plugins (list, create, add)\n" +
+    "  benchmark     Run BIRD benchmark for text-to-SQL accuracy\n" +
+    "  mcp           Start MCP server (stdio or SSE transport)\n" +
+    "  completions   Output shell completion script (bash, zsh, fish)\n\n" +
+    "Run atlas <command> --help for detailed usage of any command."
+  );
+}
+
+/** Check if args contain --help or -h for a subcommand. */
+function wantsHelp(args: string[]): boolean {
+  return args.includes("--help") || args.includes("-h");
+}
+
 // --- Main ---
 
 async function main() {
   const args = process.argv.slice(2);
   const command = args[0];
 
+  // Top-level help: atlas --help, atlas -h, or no command
+  if (!command || command === "--help" || command === "-h") {
+    printOverviewHelp();
+    process.exit(0);
+  }
+
+  // Per-subcommand --help
+  if (wantsHelp(args) && command in SUBCOMMAND_HELP) {
+    printSubcommandHelp(SUBCOMMAND_HELP[command]);
+    process.exit(0);
+  }
+
+  await checkEnvFile(command);
+
   if (command === "query") {
     return handleQuery(args);
   }
@@ -4601,17 +3694,28 @@ async function main() {
   }
 
   if (command === "doctor") {
-    const { runDoctor } = await import("../src/doctor");
-    const exitCode = await runDoctor();
+    // doctor is an alias for validate with relaxed exit codes:
+    // Sandbox and Internal DB failures don't contribute to exit 1
+    const { runValidate } = await import("../src/validate");
+    const exitCode = await runValidate({ mode: "doctor" });
     process.exit(exitCode);
   }
 
   if (command === "validate") {
     const { runValidate } = await import("../src/validate");
-    const exitCode = await runValidate();
+    const offline = args.includes("--offline");
+    const exitCode = await runValidate({ offline });
     process.exit(exitCode);
   }
 
+  if (command === "index") {
+    return handleIndex(args);
+  }
+
+  if (command === "learn") {
+    return handleLearn(args);
+  }
+
   if (command === "diff") {
     return handleDiff(args);
   }
@@ -4676,6 +3780,10 @@ async function main() {
     return;
   }
 
+  if (command === "import") {
+    return handleImport(args);
+  }
+
   if (command === "migrate") {
     return handleMigrate(args);
   }
@@ -4685,56 +3793,8 @@ async function main() {
   }
 
   if (command !== "init") {
-    console.log(
-      "Usage: bun run atlas -- <init|diff|query|doctor|validate|eval|smoke|migrate|plugin|benchmark|mcp|completions> [options]\n\n" +
-      "Commands:\n" +
-      "  init          Profile DB and generate semantic layer\n" +
-      "  diff          Compare DB schema against existing semantic layer\n" +
-      "  query         Ask a question via the Atlas API\n" +
-      "  doctor        Validate environment, connectivity, and configuration\n" +
-      "  validate      Check config and semantic layer YAML files (offline)\n" +
-      "  eval          Run eval pipeline against demo schemas\n" +
-      "  smoke         Run E2E smoke tests against a running Atlas deployment\n" +
-      "  migrate       Generate/apply plugin schema migrations\n" +
-      "  plugin        Manage plugins (list, create, add)\n" +
-      "  benchmark     Run BIRD benchmark for text-to-SQL accuracy\n" +
-      "  mcp           Start MCP server (stdio default, --transport sse --port N for SSE)\n" +
-      "  completions   Output shell completion script (bash, zsh, fish)\n\n" +
-      "Options (init):\n" +
-      "  --tables t1,t2         Only specific tables/views\n" +
-      "  --schema <name>        PostgreSQL schema (default: public)\n" +
-      "  --source <name>        Write to semantic/{name}/ subdirectory (per-source layout)\n" +
-      "  --connection <name>    Profile a datasource from atlas.config.ts\n" +
-      "  --csv file1.csv[,...]  Load CSV files via DuckDB (no DB server needed)\n" +
-      "  --parquet f1.parquet[,...] Load Parquet files via DuckDB\n" +
-      "  --enrich               Profile + LLM enrichment (needs API key)\n" +
-      "  --no-enrich            Explicitly skip LLM enrichment\n" +
-      "  --demo [simple|cybersec|ecommerce]  Load demo dataset then profile\n\n" +
-      "Options (diff):\n" +
-      "  --tables t1,t2         Only diff specific tables/views\n" +
-      "  --schema <name>        PostgreSQL schema (default: public)\n" +
-      "  --source <name>        Read from semantic/{name}/ subdirectory\n\n" +
-      "Options (query):\n" +
-      '  atlas query "question"     Ask a question (requires running API server)\n' +
-      "  --json                     Raw JSON output (pipe-friendly)\n" +
-      "  --csv                      CSV output (headers + rows, no narrative)\n" +
-      "  --quiet                    Data only — no narrative, SQL, or stats\n" +
-      "  --auto-approve             Auto-approve any pending actions\n" +
-      "  --connection <id>          Query a specific datasource\n\n" +
-      "Options (migrate):\n" +
-      "  atlas migrate              Dry run — show SQL that would be executed\n" +
-      "  --apply                    Execute migrations against internal DB\n\n" +
-      "Options (smoke):\n" +
-      "  --target <url>             API base URL (default: http://localhost:3001)\n" +
-      "  --api-key <key>            Bearer auth token\n" +
-      "  --timeout <ms>             Per-check timeout (default: 30000)\n" +
-      "  --verbose                  Show full response bodies on failure\n" +
-      "  --json                     Machine-readable JSON output\n\n" +
-      "Options (plugin):\n" +
-      "  atlas plugin list                          List installed plugins\n" +
-      "  atlas plugin create <name> --type <type>   Scaffold a new plugin\n" +
-      "  atlas plugin add <package-name>            Install a plugin package"
-    );
+    console.error(`Unknown command: ${command}\n`);
+    printOverviewHelp();
     process.exit(1);
   }
 
@@ -4744,6 +3804,7 @@ async function main() {
   const sourceArg = requireFlagIdentifier(args, "--source", "source name");
   const connectionArg = requireFlagIdentifier(args, "--connection", "connection name");
   const demoDataset = parseDemoArg(args);
+  const forceInit = args.includes("--force");
   const csvArg = getFlag(args, "--csv");
   const parquetArg = getFlag(args, "--parquet");
   const hasDocumentFiles = !!(csvArg || parquetArg);
@@ -4792,15 +3853,31 @@ async function main() {
     // Profile the DuckDB database
     console.log("Profiling DuckDB tables...\n");
     const duckFilterTables = filterTables ?? tableNames;
-    const profiles = await profileDuckDB(dbPath, duckFilterTables);
+    const duckProgress = createProgressTracker();
+    const duckStart = Date.now();
+    const duckResult = await profileDuckDB(dbPath, duckFilterTables, undefined, duckProgress);
+    let { profiles } = duckResult;
+    duckProgress.onComplete(profiles.length, Date.now() - duckStart);
 
     if (profiles.length === 0) {
       console.error("\nError: No tables were successfully profiled.");
       process.exit(1);
     }
 
+    // Warn about any profiling errors
+    if (duckResult.errors.length > 0) {
+      const total = profiles.length + duckResult.errors.length;
+      logProfilingErrors(duckResult.errors, total);
+      const { shouldAbort } = checkFailureThreshold(duckResult, forceInit);
+      if (shouldAbort) {
+        console.error(`\nUse \`--force\` to continue anyway.`);
+        process.exit(1);
+      }
+      console.warn(`Continuing with ${profiles.length} successfully profiled tables.\n`);
+    }
+
     // Run profiler heuristics
-    analyzeTableProfiles(profiles);
+    profiles = analyzeTableProfiles(profiles);
 
     console.log(`\nFound ${profiles.length} table(s):\n`);
     for (const p of profiles) {
@@ -4811,6 +3888,15 @@ async function main() {
     fs.mkdirSync(entitiesOutDir, { recursive: true });
     fs.mkdirSync(metricsOutDir, { recursive: true });
 
+    // Clean stale entity/metric files from previous runs
+    for (const dir of [entitiesOutDir, metricsOutDir]) {
+      for (const file of fs.readdirSync(dir)) {
+        if (file.endsWith(".yml") || file.endsWith(".yaml")) {
+          fs.unlinkSync(path.join(dir, file));
+        }
+      }
+    }
+
     console.log(`\nGenerating semantic layer...\n`);
 
     // DuckDB uses PostgreSQL-compatible SQL — "public" schema is not meaningful
@@ -4889,6 +3975,20 @@ Next steps:
     shouldEnrich = providerConfigured;
   }
 
+  // --- Detect org-scoped mode ---
+  // When DATABASE_URL is set and managed auth is active, atlas init writes
+  // to semantic/.orgs/{orgId}/ and auto-imports to the internal DB.
+  const noImport = args.includes("--no-import");
+  let orgId: string | undefined;
+  if (process.env.DATABASE_URL && process.env.BETTER_AUTH_SECRET) {
+    // Org-scoped mode is available. The orgId comes from the active session.
+    // For CLI use, accept ATLAS_ORG_ID env var or --org flag.
+    orgId = getFlag(args, "--org") ?? process.env.ATLAS_ORG_ID;
+    if (orgId) {
+      console.log(`Org-scoped mode: writing to semantic/.orgs/${orgId}/\n`);
+    }
+  }
+
   // --- Resolve datasource list ---
 
   // Try loading atlas.config.ts
@@ -5048,6 +4148,8 @@ Next steps:
         shouldEnrich,
         explicitEnrich,
         demoDataset: isMultiSource ? null : demoDataset,
+        force: forceInit,
+        orgId,
       });
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
@@ -5071,6 +4173,66 @@ Next steps:
     console.error(`${"=".repeat(60)}`);
     process.exit(1);
   }
+
+  // --- Auto-import to DB in org-scoped mode ---
+  if (orgId && !noImport) {
+    console.log("\nImporting entities to internal DB...\n");
+
+    const apiUrl = process.env.ATLAS_API_URL ?? "http://localhost:3001";
+    const importUrl = `${apiUrl}/api/v1/admin/semantic/org/import`;
+    const importHeaders: Record<string, string> = { "Content-Type": "application/json" };
+    if (process.env.ATLAS_API_KEY) importHeaders.Authorization = `Bearer ${process.env.ATLAS_API_KEY}`;
+
+    // For each datasource, import with its connection ID
+    let anyImported = false;
+    for (const ds of datasources) {
+      const importBody: Record<string, string> = {};
+      if (ds.id !== "default") importBody.connectionId = ds.id;
+
+      try {
+        const resp = await fetch(importUrl, {
+          method: "POST",
+          headers: importHeaders,
+          body: JSON.stringify(importBody),
+          signal: AbortSignal.timeout(60_000),
+        });
+
+        if (resp.ok) {
+          const result = await resp.json() as { imported: number; skipped: number; total: number };
+          console.log(`  Imported ${result.imported} entities${ds.id !== "default" ? ` (connection: ${ds.id})` : ""}`);
+          if (result.imported > 0) anyImported = true;
+        } else {
+          let errorMsg = `HTTP ${resp.status}`;
+          try {
+            const json = await resp.json() as { message?: string; error?: string };
+            errorMsg = json.message ?? json.error ?? errorMsg;
+          } catch {
+            // intentionally ignored: JSON parse failed, fall through to text() attempt
+            errorMsg = await resp.text().catch(() => errorMsg);
+          }
+          console.warn(`  Warning: Import failed for ${ds.id}: ${errorMsg}`);
+          console.warn("  Run 'atlas import' later to retry.\n");
+        }
+      } catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        if (detail.includes("ECONNREFUSED") || detail.includes("fetch failed")) {
+          console.warn("  Warning: Atlas API not reachable — skipping auto-import.");
+          console.warn("  Set ATLAS_API_URL if the API is not on localhost:3001");
+          console.warn("  Start the API server and run 'atlas import' to import manually.\n");
+          break; // Don't try remaining datasources
+        }
+        console.warn(`  Warning: Import failed for ${ds.id}: ${detail}`);
+      }
+    }
+
+    if (!anyImported && datasources.length > 0) {
+      console.warn("\nNo entities were imported to the DB. Files were written to disk successfully.");
+      console.warn("Run 'atlas import' once the API server is available to complete the import.");
+      if (!process.env.ATLAS_API_KEY) {
+        console.warn("Hint: set ATLAS_API_KEY for CLI authentication.\n");
+      }
+    }
+  }
 }
 
 export function getFlag(args: string[], flag: string): string | undefined {