@useatlas/create 0.0.5 → 0.0.7

package/templates/docker/src/lib/{semantic.ts → semantic/whitelist.ts}

@@ -10,14 +10,22 @@
  * `warehouse`). When no entity specifies a connection and no per-source
  * subdirectories are present, all connections share the same whitelist
  * (backward compat with single-DB).
+ *
+ * **Org scoping:** When an orgId is active, the whitelist is loaded from
+ * the internal DB (`semantic_entities` table). The semantic index is built
+ * from persistent on-disk files at `{semanticRoot}/.orgs/{orgId}/`, maintained
+ * by the dual-write sync layer (`./sync.ts`). When no orgId is
+ * present (CLI, self-hosted without orgs), file-based YAML is used
+ * (existing behavior).
  */
 
 import * as fs from "fs";
 import * as path from "path";
 import * as yaml from "js-yaml";
 import { z } from "zod";
+import { getSemanticRoot as getDefaultSemanticRoot } from "./files";
 import { createLogger } from "@atlas/api/lib/logger";
-import { invalidateSemanticIndex } from "@atlas/api/lib/semantic-index";
+import { invalidateSemanticIndex } from "./search";
 
 const log = createLogger("semantic");
 
@@ -147,7 +155,7 @@ function loadEntitiesFromDir(
  * structure, not per-source subdirectories. These are skipped when scanning for
  * source-specific directories.
  */
-const RESERVED_DIRS = new Set(["entities", "metrics"]);
+const RESERVED_DIRS = new Set(["entities", "metrics", ".orgs"]);
 
 /**
  * Load entity YAMLs and partition tables by connection ID.
@@ -174,7 +182,7 @@ function loadTablesByConnection(
     return byConnection;
   }
 
-  const root = semanticRoot ?? path.resolve(process.cwd(), "semantic");
+  const root = semanticRoot ?? getDefaultSemanticRoot();
 
   // 1. Default entities (backward compat — semantic/entities/*.yml)
   loadEntitiesFromDir(path.join(root, "entities"), "default", byConnection, crossJoins);
@@ -387,3 +395,181 @@ export function _resetPluginEntities(): void {
   _pluginEntities.clear();
   _whitelists.clear();
 }
+
+// ---------------------------------------------------------------------------
+// Org-scoped whitelist (DB-backed)
+// ---------------------------------------------------------------------------
+
+/**
+ * Per-org whitelist cache: Map<orgId, Map<connectionId, Set<tableName>>>.
+ * Populated by `loadOrgWhitelist()` before the agent loop starts.
+ * Invalidated by `invalidateOrgWhitelist(orgId)` on entity CRUD.
+ */
+const _orgWhitelists = new Map<string, Map<string, Set<string>>>();
+
+/**
+ * Load the table whitelist for an org from the internal DB.
+ *
+ * Parses stored YAML content using the same EntityShape validation as
+ * file-based loading. Results are cached per-org.
+ *
+ * @param orgId - Organization ID to load entities for.
+ * @returns Map of connectionId → Set<tableName>.
+ */
+export async function loadOrgWhitelist(orgId: string): Promise<Map<string, Set<string>>> {
+  const cached = _orgWhitelists.get(orgId);
+  if (cached) return cached;
+
+  const { listEntities } = await import("@atlas/api/lib/semantic/entities");
+  const rows = await listEntities(orgId, "entity");
+
+  const byConnection = new Map<string, Set<string>>();
+  let parseFailures = 0;
+  for (const row of rows) {
+    try {
+      const raw = yaml.load(row.yaml_content);
+      const parsed = EntityShape.safeParse(raw);
+      if (!parsed.success) {
+        parseFailures++;
+        log.warn({ orgId, entity: row.name, err: parsed.error.message }, "Skipping org entity — failed to validate");
+        continue;
+      }
+      const connId = parsed.data.connection ?? row.connection_id ?? "default";
+      if (!byConnection.has(connId)) byConnection.set(connId, new Set());
+      const tables = byConnection.get(connId)!;
+      const parts = parsed.data.table.split(".");
+      tables.add(parts[parts.length - 1].toLowerCase());
+      tables.add(parsed.data.table.toLowerCase());
+    } catch (err) {
+      parseFailures++;
+      log.warn(
+        { orgId, entity: row.name, err: err instanceof Error ? err.message : String(err) },
+        "Skipping org entity — failed to parse YAML",
+      );
+    }
+  }
+
+  if (parseFailures === rows.length && rows.length > 0) {
+    log.error({ orgId, entityCount: rows.length, parseFailures }, "All org entities failed to parse — whitelist is empty");
+  }
+
+  _orgWhitelists.set(orgId, byConnection);
+  const totalTables = Array.from(byConnection.values()).reduce((s, set) => s + set.size, 0);
+  log.info({ orgId, entityCount: rows.length, parsedCount: rows.length - parseFailures, tableCount: totalTables, connections: Array.from(byConnection.keys()) }, "Loaded org whitelist from DB");
+  return byConnection;
+}
+
+/**
+ * Get whitelisted tables for an org + connection.
+ *
+ * Must be called after `loadOrgWhitelist(orgId)` — returns empty set if
+ * the org whitelist has not been loaded yet.
+ */
+export function getOrgWhitelistedTables(orgId: string, connectionId: string = "default"): Set<string> {
+  const byConnection = _orgWhitelists.get(orgId);
+  if (!byConnection) {
+    log.warn({ orgId, connectionId }, "Org whitelist not loaded — all tables will be rejected");
+    return new Set();
+  }
+
+  const hasNonDefault = Array.from(byConnection.keys()).some((k) => k !== "default");
+  let tables: Set<string>;
+  if (!hasNonDefault) {
+    tables = new Set(byConnection.get("default") ?? []);
+  } else {
+    tables = new Set(byConnection.get(connectionId) ?? []);
+  }
+
+  // Merge plugin-provided entities (same behavior as file-based getWhitelistedTables)
+  const pluginTables = _pluginEntities.get(connectionId);
+  if (pluginTables && pluginTables.size > 0) {
+    for (const t of pluginTables) tables.add(t);
+  }
+
+  return tables;
+}
+
+/** Invalidate the cached whitelist for an org (call after entity CRUD). */
+export function invalidateOrgWhitelist(orgId: string): void {
+  _orgWhitelists.delete(orgId);
+  invalidateOrgSemanticIndex(orgId);
+}
+
+/** Clear all org whitelist caches. For testing. */
+export function _resetOrgWhitelists(): void {
+  _orgWhitelists.clear();
+}
+
+// ---------------------------------------------------------------------------
+// Org-scoped semantic index
+// ---------------------------------------------------------------------------
+
+/** Per-org semantic index cache. */
+const _orgSemanticIndexes = new Map<string, string>();
+
+/** Invalidate the cached semantic index for an org. */
+export function invalidateOrgSemanticIndex(orgId: string): void {
+  _orgSemanticIndexes.delete(orgId);
+}
+
+/**
+ * Get or build the semantic index for an org.
+ *
+ * Reads from the persistent org directory at `{semanticRoot}/.orgs/{orgId}/`
+ * maintained by the dual-write sync layer (`./sync.ts`). If the
+ * directory is empty or missing, triggers a DB-to-disk sync first.
+ *
+ * On sync failure (e.g. transient DB outage), returns an uncached result
+ * built from whatever is on disk — the next call will retry the sync.
+ */
+export async function getOrgSemanticIndex(orgId: string): Promise<string> {
+  const cached = _orgSemanticIndexes.get(orgId);
+  if (cached !== undefined) return cached;
+
+  const { getSemanticRoot, syncAllEntitiesToDisk } = await import("@atlas/api/lib/semantic/sync");
+  const orgRoot = getSemanticRoot(orgId);
+
+  // Ensure the org directory exists on disk (may be first access after boot)
+  const entitiesDir = path.join(orgRoot, "entities");
+  let hasFiles = false;
+  let syncFailed = false;
+  try {
+    const entries = fs.readdirSync(entitiesDir);
+    hasFiles = entries.some((e) => e.endsWith(".yml"));
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+      log.warn(
+        { orgId, err: err instanceof Error ? err.message : String(err) },
+        "Unexpected error reading org entities directory",
+      );
+    }
+  }
+
+  if (!hasFiles) {
+    // Rebuild from DB before building the index
+    try {
+      await syncAllEntitiesToDisk(orgId);
+    } catch (err) {
+      syncFailed = true;
+      log.warn(
+        { orgId, err: err instanceof Error ? err.message : String(err) },
+        "Failed to sync org entities to disk for semantic index — returning uncached result",
+      );
+    }
+  }
+
+  const { buildSemanticIndex } = await import("@atlas/api/lib/semantic/search");
+  const index = buildSemanticIndex(orgRoot);
+
+  // Don't cache if sync failed — next call should retry
+  if (!syncFailed) {
+    _orgSemanticIndexes.set(orgId, index);
+  }
+  return index;
+}
+
+/** Clear all org semantic index caches. For testing. */
+export function _resetOrgSemanticIndexes(): void {
+  _orgSemanticIndexes.clear();
+}
+