@useatlas/create 0.0.5 → 0.0.7

package/templates/docker/src/lib/semantic/files.ts

@@ -0,0 +1,283 @@
+/**
+ * Shared semantic layer file utilities.
+ *
+ * Reads entity/metric/glossary YAML files from the semantic/ directory.
+ * Used by both admin and public semantic API routes.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import * as yaml from "js-yaml";
+import { createLogger } from "@atlas/api/lib/logger";
+
+const log = createLogger("semantic-files");
+
+// ---------------------------------------------------------------------------
+// Semantic layer root
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the semantic layer root directory.
+ *
+ * Defaults to `{cwd}/semantic`. Override with `ATLAS_SEMANTIC_ROOT` for
+ * development, testing, or non-standard directory layouts.
+ */
+export function getSemanticRoot(): string {
+  const envRoot = process.env.ATLAS_SEMANTIC_ROOT;
+  if (envRoot !== undefined) {
+    if (!envRoot) {
+      throw new Error(
+        "ATLAS_SEMANTIC_ROOT is set but empty — remove it to use the default, or provide a path",
+      );
+    }
+    return path.resolve(envRoot);
+  }
+  return path.resolve(process.cwd(), "semantic");
+}
+
+// ---------------------------------------------------------------------------
+// Path traversal guard
+// ---------------------------------------------------------------------------
+
+/** Reject entity names that could escape the semantic root. */
+export function isValidEntityName(name: string): boolean {
+  return !!(
+    name &&
+    !name.includes("/") &&
+    !name.includes("\\") &&
+    !name.includes("..") &&
+    !name.includes("\0")
+  );
+}
+
+// ---------------------------------------------------------------------------
+// YAML reading helpers
+// ---------------------------------------------------------------------------
+
+export function readYamlFile(filePath: string): unknown {
+  const content = fs.readFileSync(filePath, "utf-8");
+  return yaml.load(content);
+}
+
+export interface EntitySummary {
+  table: string;
+  description: string;
+  columnCount: number;
+  joinCount: number;
+  measureCount: number;
+  connection: string | null;
+  type: string | null;
+  source: string;
+}
+
+/**
+ * Discover all entity YAML files from semantic/entities/ and
+ * semantic/{source}/entities/. Entities in the top-level entities/
+ * directory are tagged with source "default"; those under
+ * semantic/{name}/entities/ use the subdirectory name as source.
+ */
+interface DiscoverEntitiesResult {
+  entities: EntitySummary[];
+  warnings: string[];
+}
+
+export function discoverEntities(root: string): DiscoverEntitiesResult {
+  const entities: EntitySummary[] = [];
+  const warnings: string[] = [];
+
+  const defaultDir = path.join(root, "entities");
+  if (fs.existsSync(defaultDir)) {
+    loadEntitiesFromDir(defaultDir, "default", root, entities, warnings);
+  }
+
+  // Per-source subdirectories (e.g. semantic/warehouse/entities/)
+  const RESERVED_DIRS = new Set(["entities", "metrics"]);
+  if (fs.existsSync(root)) {
+    try {
+      const entries = fs.readdirSync(root, { withFileTypes: true });
+      for (const entry of entries) {
+        if (!entry.isDirectory() || RESERVED_DIRS.has(entry.name)) continue;
+        const subEntities = path.join(root, entry.name, "entities");
+        if (fs.existsSync(subEntities)) {
+          loadEntitiesFromDir(subEntities, entry.name, root, entities, warnings);
+        }
+      }
+    } catch (err) {
+      log.warn({ err: err instanceof Error ? err : new Error(String(err)), root }, "Failed to scan semantic root for per-source directories");
+      warnings.push("Failed to read semantic root directory");
+    }
+  }
+
+  return { entities, warnings };
+}
+
+function loadEntitiesFromDir(dir: string, source: string, root: string, out: EntitySummary[], warnings: string[]): void {
+  let files: string[];
+  try {
+    files = fs.readdirSync(dir).filter((f) => f.endsWith(".yml"));
+  } catch (err) {
+    log.warn({ err: err instanceof Error ? err : new Error(String(err)), dir, source }, "Failed to read entities directory");
+    warnings.push(`Failed to read directory: ${path.relative(root, dir)}`);
+    return;
+  }
+
+  for (const file of files) {
+    try {
+      const raw = readYamlFile(path.join(dir, file)) as Record<string, unknown>;
+      if (!raw || typeof raw !== "object") continue;
+      if (!raw.table) {
+        warnings.push(`Entity file missing required 'table' field: ${path.relative(root, path.join(dir, file))}`);
+        continue;
+      }
+
+      const dimensions = raw.dimensions && typeof raw.dimensions === "object"
+        ? Object.keys(raw.dimensions)
+        : [];
+      const joins = Array.isArray(raw.joins) ? raw.joins : (raw.joins && typeof raw.joins === "object" ? Object.keys(raw.joins) : []);
+      const measures = Array.isArray(raw.measures) ? raw.measures : (raw.measures && typeof raw.measures === "object" ? Object.keys(raw.measures) : []);
+
+      out.push({
+        table: String(raw.table),
+        description: typeof raw.description === "string" ? raw.description : "",
+        columnCount: dimensions.length,
+        joinCount: Array.isArray(joins) ? joins.length : 0,
+        measureCount: Array.isArray(measures) ? measures.length : 0,
+        connection: typeof raw.connection === "string" ? raw.connection : null,
+        type: typeof raw.type === "string" ? raw.type : null,
+        source,
+      });
+    } catch (err) {
+      log.warn({ err: err instanceof Error ? err : new Error(String(err)), file, dir, source }, "Failed to parse entity YAML file");
+      warnings.push(`Failed to parse entity: ${path.relative(root, path.join(dir, file))}`);
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Table discovery (with columns)
+// ---------------------------------------------------------------------------
+
+import type { TableInfo, TableColumn } from "@useatlas/types";
+export type { TableInfo };
+
+/**
+ * Discover all entity YAML files and return a simplified table view
+ * with column details. Used by the public `GET /api/v1/tables` endpoint.
+ */
+interface DiscoverTablesResult {
+  tables: TableInfo[];
+  warnings: string[];
+}
+
+export function discoverTables(root: string): DiscoverTablesResult {
+  const tables: TableInfo[] = [];
+  const warnings: string[] = [];
+
+  const defaultDir = path.join(root, "entities");
+  if (fs.existsSync(defaultDir)) {
+    loadTablesFromDir(defaultDir, root, tables, warnings);
+  }
+
+  const RESERVED_DIRS = new Set(["entities", "metrics"]);
+  if (fs.existsSync(root)) {
+    try {
+      const entries = fs.readdirSync(root, { withFileTypes: true });
+      for (const entry of entries) {
+        if (!entry.isDirectory() || RESERVED_DIRS.has(entry.name)) continue;
+        const subEntities = path.join(root, entry.name, "entities");
+        if (fs.existsSync(subEntities)) {
+          loadTablesFromDir(subEntities, root, tables, warnings);
+        }
+      }
+    } catch (err) {
+      log.warn({ err: err instanceof Error ? err : new Error(String(err)), root }, "Failed to scan semantic root for tables");
+      warnings.push("Failed to read semantic root directory");
+    }
+  }
+
+  return { tables, warnings };
+}
+
+function loadTablesFromDir(dir: string, root: string, out: TableInfo[], warnings: string[]): void {
+  let files: string[];
+  try {
+    files = fs.readdirSync(dir).filter((f) => f.endsWith(".yml"));
+  } catch (err) {
+    log.warn({ err: err instanceof Error ? err : new Error(String(err)), dir }, "Failed to read entities directory for tables");
+    warnings.push(`Failed to read directory: ${path.relative(root, dir)}`);
+    return;
+  }
+
+  for (const file of files) {
+    try {
+      const raw = readYamlFile(path.join(dir, file)) as Record<string, unknown>;
+      if (!raw || typeof raw !== "object") continue;
+      if (!raw.table) {
+        warnings.push(`Entity file missing required 'table' field: ${path.relative(root, path.join(dir, file))}`);
+        continue;
+      }
+
+      const columns: TableColumn[] = [];
+      const dims = raw.dimensions;
+      if (dims && typeof dims === "object") {
+        if (Array.isArray(dims)) {
+          for (const d of dims) {
+            if (d && typeof d === "object" && typeof d.name === "string") {
+              columns.push({
+                name: d.name,
+                type: typeof d.type === "string" ? d.type : "string",
+                description: typeof d.description === "string" ? d.description : "",
+              });
+            }
+          }
+        } else {
+          for (const [key, val] of Object.entries(dims)) {
+            const dim = val as Record<string, unknown> | undefined;
+            columns.push({
+              name: key,
+              type: typeof dim?.type === "string" ? dim.type : "string",
+              description: typeof dim?.description === "string" ? dim.description : "",
+            });
+          }
+        }
+      }
+
+      out.push({
+        table: String(raw.table),
+        description: typeof raw.description === "string" ? raw.description : "",
+        columns,
+      });
+    } catch (err) {
+      log.warn({ err: err instanceof Error ? err : new Error(String(err)), file, dir }, "Failed to parse entity YAML for tables");
+      warnings.push(`Failed to parse entity: ${path.relative(root, path.join(dir, file))}`);
+    }
+  }
+}
+
+/**
+ * Find a specific entity YAML file by table name. Searches default
+ * entities/ and all per-source subdirectories.
+ * Caller must validate `name` with isValidEntityName() first.
+ */
+export function findEntityFile(root: string, name: string): string | null {
+  const defaultDir = path.join(root, "entities");
+  const defaultFile = path.join(defaultDir, `${name}.yml`);
+  if (fs.existsSync(defaultFile)) return defaultFile;
+
+  // Search per-source subdirectories
+  const RESERVED_DIRS = new Set(["entities", "metrics"]);
+  if (fs.existsSync(root)) {
+    try {
+      const entries = fs.readdirSync(root, { withFileTypes: true });
+      for (const entry of entries) {
+        if (!entry.isDirectory() || RESERVED_DIRS.has(entry.name)) continue;
+        const subFile = path.join(root, entry.name, "entities", `${name}.yml`);
+        if (fs.existsSync(subFile)) return subFile;
+      }
+    } catch (err) {
+      log.warn({ err: err instanceof Error ? err : new Error(String(err)), root, name }, "Failed to scan subdirectories for entity file");
+    }
+  }
+
+  return null;
+}

package/templates/docker/src/lib/semantic/index.ts

@@ -0,0 +1,27 @@
+/**
+ * Semantic layer — barrel export.
+ *
+ * Re-exports the public API from whitelist.ts (the original semantic.ts).
+ * Other submodules are imported directly via their paths:
+ *   - @atlas/api/lib/semantic/files   (YAML file discovery/parsing)
+ *   - @atlas/api/lib/semantic/sync    (dual-write DB sync)
+ *   - @atlas/api/lib/semantic/search  (pre-indexed semantic layer summary)
+ *   - @atlas/api/lib/semantic/diff    (schema diffing)
+ *   - @atlas/api/lib/semantic/entities (DB query helper)
+ *
+ * Test-only helpers (_resetPluginEntities, _resetOrgWhitelists,
+ * _resetOrgSemanticIndexes) are exported from whitelist.ts directly.
+ */
+
+export {
+  getWhitelistedTables,
+  getCrossSourceJoins,
+  registerPluginEntities,
+  _resetWhitelists,
+  loadOrgWhitelist,
+  getOrgWhitelistedTables,
+  invalidateOrgWhitelist,
+  invalidateOrgSemanticIndex,
+  getOrgSemanticIndex,
+} from "./whitelist";
+export type { CrossSourceJoin } from "./whitelist";

package/templates/docker/src/lib/{semantic-index.ts → semantic/search.ts}

@@ -14,6 +14,7 @@ import * as fs from "fs";
 import * as path from "path";
 import * as yaml from "js-yaml";
 import { createLogger } from "@atlas/api/lib/logger";
+import { getSemanticRoot as getDefaultSemanticRoot } from "./files";
 
 const log = createLogger("semantic-index");
 
@@ -32,6 +33,7 @@ interface EntityDimension {
   description?: string;
   sample_values?: unknown[];
   primary_key?: boolean;
+  virtual?: boolean;
 }
 
 interface EntityMeasure {
@@ -110,7 +112,7 @@ let _cachedEntityCount = 0;
 export function getSemanticIndex(semanticRoot?: string): string {
   if (_cachedIndex !== null) return _cachedIndex;
 
-  const root = semanticRoot ?? path.resolve(process.cwd(), "semantic");
+  const root = semanticRoot ?? getDefaultSemanticRoot();
   _cachedIndex = buildSemanticIndex(root);
   return _cachedIndex;
 }
@@ -126,6 +128,59 @@ export function invalidateSemanticIndex(): void {
   _cachedEntityCount = 0;
 }
 
+export interface SemanticIndexStats {
+  entities: number;
+  dimensions: number;
+  measures: number;
+  metrics: number;
+  glossaryTerms: number;
+  keywords: number;
+}
+
+/** Collect stats from the semantic layer without building the full index text. */
+export function getSemanticIndexStats(semanticRoot: string): SemanticIndexStats {
+  const entities = loadEntities(semanticRoot);
+  const metrics = loadMetrics(semanticRoot);
+  const glossary = loadGlossary(semanticRoot);
+
+  let dimensionCount = 0;
+  let measureCount = 0;
+  const keywords = new Set<string>();
+
+  for (const e of entities) {
+    const dims = e.dimensions ?? [];
+    const meas = e.measures ?? [];
+    dimensionCount += dims.length;
+    measureCount += meas.length;
+
+    if (e.table) keywords.add(e.table.toLowerCase());
+    if (e.name) keywords.add(e.name.toLowerCase());
+    for (const d of dims) {
+      if (d.name) keywords.add(d.name.toLowerCase());
+    }
+    for (const m of meas) {
+      if (m.name) keywords.add(m.name.toLowerCase());
+    }
+  }
+
+  for (const m of metrics) {
+    if (m.name) keywords.add(m.name.toLowerCase());
+  }
+
+  for (const t of glossary) {
+    if (t.term) keywords.add(t.term.toLowerCase());
+  }
+
+  return {
+    entities: entities.length,
+    dimensions: dimensionCount,
+    measures: measureCount,
+    metrics: metrics.length,
+    glossaryTerms: glossary.length,
+    keywords: keywords.size,
+  };
+}
+
 /**
  * Build a compressed text summary of the entire semantic layer.
  */
@@ -235,13 +290,29 @@ function formatEntity(
   if (full) {
     // Full mode: list all columns with types
     if (entity.dimensions && entity.dimensions.length > 0) {
-      lines.push("Columns:");
-      for (const dim of entity.dimensions) {
-        const name = dim.name ?? dim.sql ?? "?";
-        const type = dim.type ?? "unknown";
-        const pk = dim.primary_key ? " PK" : "";
-        const desc = dim.description ? ` — ${dim.description}` : "";
-        lines.push(`  - ${name} (${type}${pk})${desc}`);
+      const realCols = entity.dimensions.filter((d) => !d.virtual);
+      const virtualCols = entity.dimensions.filter((d) => d.virtual);
+
+      if (realCols.length > 0) {
+        lines.push("Columns:");
+        for (const dim of realCols) {
+          const name = dim.name ?? dim.sql ?? "?";
+          const type = dim.type ?? "unknown";
+          const pk = dim.primary_key ? " PK" : "";
+          const desc = dim.description ? ` — ${dim.description}` : "";
+          lines.push(`  - ${name} (${type}${pk})${desc}`);
+        }
+      }
+
+      if (virtualCols.length > 0) {
+        lines.push("Virtual columns (NOT real columns — use the SQL expression inline):");
+        for (const dim of virtualCols) {
+          const name = dim.name ?? "?";
+          const type = dim.type ?? "unknown";
+          const desc = dim.description ? ` — ${dim.description}` : "";
+          const sql = dim.sql ? ` sql: ${dim.sql.replace(/\n/g, " ")}` : "";
+          lines.push(`  - ${name} (${type})${desc}${sql}`);
+        }
       }
     }
 
@@ -289,7 +360,7 @@ function formatEntity(
     if (measureCount > 0) parts.push(`${measureCount} measures`);
     if (joinCount > 0)
       parts.push(
-        `joins: ${entity.joins!.map((j) => j.target_entity).join(", ")}`,
+        `joins: ${(entity.joins ?? []).map((j) => j.target_entity).join(", ")}`,
       );
     lines.push(parts.join(" | "));
   }