@useatlas/create 0.0.5 → 0.0.7

package/templates/docker/src/lib/semantic/diff.ts

@@ -0,0 +1,267 @@
+/**
+ * Semantic layer diff — compares live database schema against entity YAML files.
+ *
+ * Core comparison logic extracted from the CLI `atlas diff` command so both
+ * CLI and API can share the same diff engine.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import type { SemanticTableDiff, SemanticDiffResponse } from "@useatlas/types";
+import { createLogger } from "@atlas/api/lib/logger";
+import { connections } from "@atlas/api/lib/db/connection";
+import { getSemanticRoot, readYamlFile, discoverEntities } from "./files";
+
+const log = createLogger("semantic-diff");
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface EntitySnapshot {
+  table: string;
+  columns: Map<string, string>; // column name → normalized type
+  foreignKeys: Set<string>;     // "from_col→target_table.target_col"
+}
+
+export interface DiffResult {
+  newTables: string[];
+  removedTables: string[];
+  tableDiffs: SemanticTableDiff[];
+  unchangedCount: number;
+}
+
+// ---------------------------------------------------------------------------
+// mapSQLType — normalizes raw SQL types to semantic dimension types
+// ---------------------------------------------------------------------------
+
+export function mapSQLType(sqlType: string): string {
+  const unwrapped = sqlType.replace(/Nullable\((.+)\)/g, "$1").replace(/LowCardinality\((.+)\)/g, "$1");
+  const t = unwrapped.toLowerCase();
+  if (t.includes("interval") || t.includes("money")) return "string";
+  if (
+    t.includes("int") ||
+    t.includes("float") ||
+    t.includes("real") ||
+    t.includes("numeric") ||
+    t.includes("decimal") ||
+    t.includes("double") ||
+    t === "currency" ||
+    t === "percent" ||
+    t === "long"
+  )
+    return "number";
+  if (t.startsWith("bool")) return "boolean";
+  if (t.includes("date") || t.includes("time") || t.includes("timestamp"))
+    return "date";
+  return "string";
+}
+
+// ---------------------------------------------------------------------------
+// parseEntityYAML — build EntitySnapshot from a YAML document
+// ---------------------------------------------------------------------------
+
+export function parseEntityYAML(doc: Record<string, unknown>): EntitySnapshot {
+  const table = doc.table as string;
+  const columns = new Map<string, string>();
+  const foreignKeys = new Set<string>();
+
+  const rawDimensions = doc.dimensions ?? [];
+  if (!Array.isArray(rawDimensions)) {
+    return { table, columns, foreignKeys };
+  }
+  const dimensions = rawDimensions as Record<string, unknown>[];
+  for (const dim of dimensions) {
+    if (dim.virtual) continue;
+    if (typeof dim.name !== "string" || typeof dim.type !== "string") continue;
+    columns.set(dim.name, dim.type);
+  }
+
+  const rawJoins = doc.joins ?? [];
+  if (!Array.isArray(rawJoins)) {
+    return { table, columns, foreignKeys };
+  }
+  const joins = rawJoins as Record<string, unknown>[];
+  for (const join of joins) {
+    const joinCols = join.join_columns as { from: string; to: string } | undefined;
+    const targetEntity = join.target_entity as string | undefined;
+    if (!joinCols || !targetEntity) continue;
+    if (typeof joinCols.from !== "string" || typeof joinCols.to !== "string") continue;
+    const targetTable = targetEntity.replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase();
+    foreignKeys.add(`${joinCols.from}→${targetTable}.${joinCols.to}`);
+  }
+
+  return { table, columns, foreignKeys };
+}
+
+// ---------------------------------------------------------------------------
+// computeDiff — pure comparison of DB vs YAML snapshots
+// ---------------------------------------------------------------------------
+
+export function computeDiff(
+  dbSnapshots: Map<string, EntitySnapshot>,
+  yamlSnapshots: Map<string, EntitySnapshot>,
+): DiffResult {
+  const dbTables = new Set(dbSnapshots.keys());
+  const yamlTables = new Set(yamlSnapshots.keys());
+
+  const newTables = [...dbTables].filter((t) => !yamlTables.has(t)).sort();
+  const removedTables = [...yamlTables].filter((t) => !dbTables.has(t)).sort();
+
+  const tableDiffs: SemanticTableDiff[] = [];
+  let unchangedCount = 0;
+
+  for (const table of [...dbTables].filter((t) => yamlTables.has(t)).sort()) {
+    const db = dbSnapshots.get(table)!;
+    const yml = yamlSnapshots.get(table)!;
+
+    const addedColumns: { name: string; type: string }[] = [];
+    const removedColumns: { name: string; type: string }[] = [];
+    const typeChanges: { name: string; yamlType: string; dbType: string }[] = [];
+
+    for (const [name, type] of db.columns) {
+      if (!yml.columns.has(name)) {
+        addedColumns.push({ name, type });
+      } else if (yml.columns.get(name) !== type) {
+        typeChanges.push({ name, yamlType: yml.columns.get(name)!, dbType: type });
+      }
+    }
+
+    for (const [name, type] of yml.columns) {
+      if (!db.columns.has(name)) {
+        removedColumns.push({ name, type });
+      }
+    }
+
+    if (addedColumns.length > 0 || removedColumns.length > 0 || typeChanges.length > 0) {
+      tableDiffs.push({ table, addedColumns, removedColumns, typeChanges });
+    } else {
+      unchangedCount++;
+    }
+  }
+
+  return { newTables, removedTables, tableDiffs, unchangedCount };
+}
+
+// ---------------------------------------------------------------------------
+// getDBSchema — query information_schema for table/column metadata
+// ---------------------------------------------------------------------------
+
+export async function getDBSchema(connectionId: string = "default"): Promise<Map<string, EntitySnapshot>> {
+  const conn = connections.get(connectionId);
+  const dbType = connections.getDBType(connectionId);
+
+  let sql: string;
+  if (dbType === "mysql") {
+    sql = `SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = DATABASE() ORDER BY table_name, ordinal_position`;
+  } else if (dbType === "postgres") {
+    sql = `SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = current_schema() ORDER BY table_name, ordinal_position`;
+  } else {
+    throw new Error(`Schema diff is not yet supported for ${dbType} connections. Supported: postgres, mysql.`);
+  }
+
+  let result;
+  try {
+    result = await conn.query(sql, 15000);
+  } catch (err) {
+    throw new Error(`Database schema query failed for connection "${connectionId}": ${err instanceof Error ? err.message : String(err)}`, { cause: err });
+  }
+
+  const snapshots = new Map<string, EntitySnapshot>();
+
+  for (const row of result.rows) {
+    const tableName = String(row.table_name);
+    const columnName = String(row.column_name);
+    const dataType = String(row.data_type);
+
+    if (!snapshots.has(tableName)) {
+      snapshots.set(tableName, { table: tableName, columns: new Map(), foreignKeys: new Set() });
+    }
+    snapshots.get(tableName)!.columns.set(columnName, mapSQLType(dataType));
+  }
+
+  return snapshots;
+}
+
+// ---------------------------------------------------------------------------
+// getYAMLSnapshots — read entity YAML files for a given connection
+// ---------------------------------------------------------------------------
+
+export function getYAMLSnapshots(
+  connectionId: string = "default",
+): { snapshots: Map<string, EntitySnapshot>; warnings: string[] } {
+  const root = getSemanticRoot();
+  const snapshots = new Map<string, EntitySnapshot>();
+  const warnings: string[] = [];
+
+  if (!fs.existsSync(root)) {
+    warnings.push(`Semantic root not found: ${root}. Run 'atlas init' to create it.`);
+    return { snapshots, warnings };
+  }
+
+  // Discover entities to find their source/connection mapping
+  const { entities, warnings: discoverWarnings } = discoverEntities(root);
+  if (discoverWarnings.length > 0) {
+    warnings.push(...discoverWarnings);
+  }
+
+  for (const entity of entities) {
+    // Match entities to the requested connection:
+    // - entities with connection=null/undefined belong to "default"
+    // - entities with an explicit connection belong to that connection
+    const entityConnection = entity.connection ?? entity.source ?? "default";
+    // "default" source means flat layout → default connection
+    const effectiveConnection = entityConnection === "default" ? "default" : entityConnection;
+    if (effectiveConnection !== connectionId) continue;
+
+    // Resolve entity file path
+    const entitiesDir = entity.source === "default"
+      ? path.join(root, "entities")
+      : path.join(root, entity.source, "entities");
+    const filePath = path.join(entitiesDir, `${entity.table}.yml`);
+
+    if (!fs.existsSync(filePath)) continue;
+
+    try {
+      const doc = readYamlFile(filePath) as Record<string, unknown>;
+      if (!doc || typeof doc.table !== "string") continue;
+      const snapshot = parseEntityYAML(doc);
+      snapshots.set(snapshot.table, snapshot);
+    } catch (err) {
+      const msg = `Failed to parse ${entity.table}.yml: ${err instanceof Error ? err.message : String(err)}`;
+      log.warn({ err: err instanceof Error ? err : new Error(String(err)), filePath }, msg);
+      warnings.push(msg);
+    }
+  }
+
+  return { snapshots, warnings };
+}
+
+// ---------------------------------------------------------------------------
+// runDiff — orchestrates the full diff for a connection
+// ---------------------------------------------------------------------------
+
+export async function runDiff(connectionId: string = "default"): Promise<SemanticDiffResponse> {
+  const dbSnapshots = await getDBSchema(connectionId);
+  const { snapshots: yamlSnapshots, warnings } = getYAMLSnapshots(connectionId);
+
+  const diff = computeDiff(dbSnapshots, yamlSnapshots);
+
+  const total = diff.newTables.length + diff.removedTables.length + diff.tableDiffs.length + diff.unchangedCount;
+
+  return {
+    connection: connectionId,
+    newTables: diff.newTables,
+    removedTables: diff.removedTables,
+    tableDiffs: diff.tableDiffs,
+    unchangedCount: diff.unchangedCount,
+    summary: {
+      total,
+      new: diff.newTables.length,
+      removed: diff.removedTables.length,
+      changed: diff.tableDiffs.length,
+      unchanged: diff.unchangedCount,
+    },
+    ...(warnings.length > 0 && { warnings }),
+  };
+}

package/templates/docker/src/lib/semantic/entities.ts

@@ -0,0 +1,167 @@
+/**
+ * CRUD helpers for org-scoped semantic entities stored in the internal DB.
+ *
+ * Entity types: "entity", "metric", "glossary", "catalog".
+ * Each entity is stored as raw YAML content keyed by (orgId, entityType, name).
+ */
+
+import { internalQuery, hasInternalDB } from "@atlas/api/lib/db/internal";
+import { createLogger } from "@atlas/api/lib/logger";
+
+const log = createLogger("semantic-entities");
+
+export type SemanticEntityType = "entity" | "metric" | "glossary" | "catalog";
+
+export interface SemanticEntityRow {
+  id: string;
+  org_id: string;
+  entity_type: SemanticEntityType;
+  name: string;
+  yaml_content: string;
+  connection_id: string | null;
+  created_at: string;
+  updated_at: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Upsert a semantic entity for an org.
+ * Uses ON CONFLICT on (org_id, entity_type, name) to update if exists.
+ */
+export async function upsertEntity(
+  orgId: string,
+  entityType: SemanticEntityType,
+  name: string,
+  yamlContent: string,
+  connectionId?: string,
+): Promise<void> {
+  if (!hasInternalDB()) {
+    throw new Error("Internal DB required for org-scoped semantic entities");
+  }
+  await internalQuery(
+    `INSERT INTO semantic_entities (org_id, entity_type, name, yaml_content, connection_id)
+     VALUES ($1, $2, $3, $4, $5)
+     ON CONFLICT (org_id, entity_type, name)
+     DO UPDATE SET yaml_content = EXCLUDED.yaml_content,
+                   connection_id = EXCLUDED.connection_id,
+                   updated_at = now()`,
+    [orgId, entityType, name, yamlContent, connectionId ?? null],
+  );
+}
+
+/**
+ * List all semantic entities for an org, optionally filtered by type.
+ */
+export async function listEntities(
+  orgId: string,
+  entityType?: SemanticEntityType,
+): Promise<SemanticEntityRow[]> {
+  if (!hasInternalDB()) return [];
+
+  if (entityType) {
+    return internalQuery<SemanticEntityRow>(
+      `SELECT id, org_id, entity_type, name, yaml_content, connection_id, created_at, updated_at
+       FROM semantic_entities
+       WHERE org_id = $1 AND entity_type = $2
+       ORDER BY name`,
+      [orgId, entityType],
+    );
+  }
+
+  return internalQuery<SemanticEntityRow>(
+    `SELECT id, org_id, entity_type, name, yaml_content, connection_id, created_at, updated_at
+     FROM semantic_entities
+     WHERE org_id = $1
+     ORDER BY entity_type, name`,
+    [orgId],
+  );
+}
+
+/**
+ * Get a single semantic entity by org, type, and name.
+ */
+export async function getEntity(
+  orgId: string,
+  entityType: SemanticEntityType,
+  name: string,
+): Promise<SemanticEntityRow | null> {
+  if (!hasInternalDB()) return null;
+
+  const rows = await internalQuery<SemanticEntityRow>(
+    `SELECT id, org_id, entity_type, name, yaml_content, connection_id, created_at, updated_at
+     FROM semantic_entities
+     WHERE org_id = $1 AND entity_type = $2 AND name = $3`,
+    [orgId, entityType, name],
+  );
+  return rows[0] ?? null;
+}
+
+/**
+ * Delete a semantic entity by org, type, and name.
+ * Returns true if a row was deleted.
+ */
+export async function deleteEntity(
+  orgId: string,
+  entityType: SemanticEntityType,
+  name: string,
+): Promise<boolean> {
+  if (!hasInternalDB()) return false;
+
+  const rows = await internalQuery<{ id: string }>(
+    `DELETE FROM semantic_entities
+     WHERE org_id = $1 AND entity_type = $2 AND name = $3
+     RETURNING id`,
+    [orgId, entityType, name],
+  );
+  return rows.length > 0;
+}
+
+/**
+ * Count entities for an org, optionally by type.
+ */
+export async function countEntities(
+  orgId: string,
+  entityType?: SemanticEntityType,
+): Promise<number> {
+  if (!hasInternalDB()) return 0;
+
+  const rows = entityType
+    ? await internalQuery<{ count: string }>(
+        `SELECT COUNT(*)::TEXT AS count FROM semantic_entities WHERE org_id = $1 AND entity_type = $2`,
+        [orgId, entityType],
+      )
+    : await internalQuery<{ count: string }>(
+        `SELECT COUNT(*)::TEXT AS count FROM semantic_entities WHERE org_id = $1`,
+        [orgId],
+      );
+  return parseInt(rows[0]?.count ?? "0", 10);
+}
+
+/**
+ * Bulk upsert entities for an org. Each entity is upserted individually —
+ * failures are logged and skipped (partial imports are expected).
+ * Used by the import endpoint.
+ */
+export async function bulkUpsertEntities(
+  orgId: string,
+  entities: Array<{ entityType: SemanticEntityType; name: string; yamlContent: string; connectionId?: string }>,
+): Promise<number> {
+  if (!hasInternalDB()) {
+    throw new Error("Internal DB required for org-scoped semantic entities");
+  }
+  if (entities.length === 0) return 0;
+
+  let upserted = 0;
+  for (const e of entities) {
+    try {
+      await upsertEntity(orgId, e.entityType, e.name, e.yamlContent, e.connectionId);
+      upserted++;
+    } catch (err) {
+      log.warn(
+        { orgId, entityType: e.entityType, name: e.name, err: err instanceof Error ? err.message : String(err) },
+        "Failed to upsert semantic entity — skipping",
+      );
+    }
+  }
+  return upserted;
+}