@useatlas/create 0.0.5 → 0.0.7

package/templates/nextjs-standalone/bin/__tests__/failure-threshold.test.ts

@@ -0,0 +1,148 @@
+/**
+ * Tests for the profiling failure threshold logic.
+ *
+ * Verifies that checkFailureThreshold correctly determines when to abort
+ * based on the ratio of failed tables, and respects the --force override.
+ */
+import { describe, it, expect } from "bun:test";
+import type { TableProfile, ProfilingResult } from "../atlas";
+import { checkFailureThreshold, logProfilingErrors } from "../atlas";
+
+function makeResult(successCount: number, errorCount: number): ProfilingResult {
+  const profiles: TableProfile[] = Array.from({ length: successCount }, (_, i) => ({
+    table_name: `table_${i}`,
+    object_type: "table" as const,
+    row_count: 100,
+    columns: [],
+    primary_key_columns: [],
+    foreign_keys: [],
+    inferred_foreign_keys: [],
+    profiler_notes: [],
+    table_flags: { possibly_abandoned: false, possibly_denormalized: false },
+  }));
+  const errors = Array.from({ length: errorCount }, (_, i) => ({
+    table: `failed_table_${i}`,
+    error: `permission denied for table failed_table_${i}`,
+  }));
+  return { profiles, errors };
+}
+
+describe("checkFailureThreshold", () => {
+  it("returns shouldAbort=false when there are no errors", () => {
+    const result = makeResult(10, 0);
+    const { shouldAbort, failureRate } = checkFailureThreshold(result, false);
+    expect(shouldAbort).toBe(false);
+    expect(failureRate).toBe(0);
+  });
+
+  it("returns shouldAbort=false when failure rate is below 20%", () => {
+    // 1/10 = 10%
+    const result = makeResult(9, 1);
+    const { shouldAbort, failureRate } = checkFailureThreshold(result, false);
+    expect(shouldAbort).toBe(false);
+    expect(failureRate).toBeCloseTo(0.1);
+  });
+
+  it("returns shouldAbort=false when failure rate is exactly 20%", () => {
+    // 2/10 = 20% — threshold is strictly >20%, so this should not abort
+    const result = makeResult(8, 2);
+    const { shouldAbort, failureRate } = checkFailureThreshold(result, false);
+    expect(shouldAbort).toBe(false);
+    expect(failureRate).toBeCloseTo(0.2);
+  });
+
+  it("returns shouldAbort=true when failure rate exceeds 20%", () => {
+    // 3/10 = 30%
+    const result = makeResult(7, 3);
+    const { shouldAbort, failureRate } = checkFailureThreshold(result, false);
+    expect(shouldAbort).toBe(true);
+    expect(failureRate).toBeCloseTo(0.3);
+  });
+
+  it("returns shouldAbort=true when all tables fail (0 profiles)", () => {
+    // 5/5 = 100% — note: in practice, the "profiles.length === 0" check
+    // fires before this, but the threshold logic should still be correct
+    const result = makeResult(0, 5);
+    const { shouldAbort, failureRate } = checkFailureThreshold(result, false);
+    expect(shouldAbort).toBe(true);
+    expect(failureRate).toBe(1);
+  });
+
+  it("returns shouldAbort=false when --force is set, even above threshold", () => {
+    // 8/10 = 80% but force=true
+    const result = makeResult(2, 8);
+    const { shouldAbort, failureRate } = checkFailureThreshold(result, true);
+    expect(shouldAbort).toBe(false);
+    expect(failureRate).toBeCloseTo(0.8);
+  });
+
+  it("returns shouldAbort=false when --force is set with 100% failure", () => {
+    const result = makeResult(0, 10);
+    const { shouldAbort } = checkFailureThreshold(result, true);
+    expect(shouldAbort).toBe(false);
+  });
+
+  it("handles single table success", () => {
+    const result = makeResult(1, 0);
+    const { shouldAbort, failureRate } = checkFailureThreshold(result, false);
+    expect(shouldAbort).toBe(false);
+    expect(failureRate).toBe(0);
+  });
+
+  it("handles single table failure", () => {
+    // 1/1 = 100%
+    const result = makeResult(0, 1);
+    const { shouldAbort } = checkFailureThreshold(result, false);
+    expect(shouldAbort).toBe(true);
+  });
+
+  it("boundary: 21% failure rate triggers abort", () => {
+    // ~21.4% = 3/14
+    const result = makeResult(11, 3);
+    const { shouldAbort, failureRate } = checkFailureThreshold(result, false);
+    expect(shouldAbort).toBe(true);
+    expect(failureRate).toBeGreaterThan(0.2);
+  });
+});
+
+describe("logProfilingErrors", () => {
+  it("caps preview at 5 errors", () => {
+    const errors = Array.from({ length: 8 }, (_, i) => ({
+      table: `t${i}`,
+      error: `err${i}`,
+    }));
+    // Should not throw — just verifying it handles overflow
+    const origWarn = console.warn;
+    const logged: string[] = [];
+    console.warn = (msg: string) => logged.push(msg);
+    try {
+      logProfilingErrors(errors, 20);
+      // Should log header + 5 previews + overflow line = 7 calls
+      expect(logged.length).toBe(7);
+      expect(logged[0]).toContain("8/20");
+      expect(logged[0]).toContain("40%");
+      expect(logged[6]).toContain("... and 3 more");
+    } finally {
+      console.warn = origWarn;
+    }
+  });
+
+  it("does not show overflow for <= 5 errors", () => {
+    const errors = Array.from({ length: 3 }, (_, i) => ({
+      table: `t${i}`,
+      error: `err${i}`,
+    }));
+    const origWarn = console.warn;
+    const logged: string[] = [];
+    console.warn = (msg: string) => logged.push(msg);
+    try {
+      logProfilingErrors(errors, 10);
+      // header + 3 previews = 4 calls, no overflow
+      expect(logged.length).toBe(4);
+      expect(logged[0]).toContain("3/10");
+      expect(logged[0]).toContain("30%");
+    } finally {
+      console.warn = origWarn;
+    }
+  });
+});

package/templates/nextjs-standalone/bin/__tests__/fatal-error-propagation.test.ts

@@ -0,0 +1,267 @@
+/**
+ * Tests that fatal connection errors (ECONNRESET, ECONNREFUSED, etc.)
+ * propagate from column-level catch blocks up to the table-level catch,
+ * rather than being silently swallowed as column warnings.
+ *
+ * Covers fixes for #358 (Postgres profiler had no fatal error detection)
+ * and #359 (column-level fatal error re-throw in all profilers).
+ */
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import { FATAL_ERROR_PATTERN, isFatalConnectionError, ingestIntoDuckDB, profileDuckDB } from "../atlas";
+
+describe("FATAL_ERROR_PATTERN", () => {
+  const fatalCodes = [
+    "ECONNRESET",
+    "ECONNREFUSED",
+    "EHOSTUNREACH",
+    "ENOTFOUND",
+    "EPIPE",
+    "ETIMEDOUT",
+  ];
+
+  for (const code of fatalCodes) {
+    it(`matches ${code}`, () => {
+      expect(FATAL_ERROR_PATTERN.test(`connect ${code}: connection lost`)).toBe(true);
+    });
+
+    it(`matches ${code} case-insensitively`, () => {
+      expect(FATAL_ERROR_PATTERN.test(code.toLowerCase())).toBe(true);
+    });
+  }
+
+  it("does not match non-fatal errors", () => {
+    expect(FATAL_ERROR_PATTERN.test("permission denied for table foo")).toBe(false);
+    expect(FATAL_ERROR_PATTERN.test("column 'bar' does not exist")).toBe(false);
+    expect(FATAL_ERROR_PATTERN.test("syntax error at or near SELECT")).toBe(false);
+  });
+
+  it("matches errors embedded in longer messages", () => {
+    expect(FATAL_ERROR_PATTERN.test("read ECONNRESET at TLSSocket._recv")).toBe(true);
+    expect(FATAL_ERROR_PATTERN.test("connect ECONNREFUSED 127.0.0.1:5432")).toBe(true);
+    expect(FATAL_ERROR_PATTERN.test("getaddrinfo ENOTFOUND db.example.com")).toBe(true);
+  });
+
+  it("does not false-positive on table names containing error substrings", () => {
+    // Word boundaries prevent matching partial words like "EPIPE_logs"
+    expect(FATAL_ERROR_PATTERN.test("permission denied for relation EPIPE_logs")).toBe(false);
+    expect(FATAL_ERROR_PATTERN.test("column ETIMEDOUT_counter not found")).toBe(false);
+  });
+});
+
+describe("isFatalConnectionError", () => {
+  it("detects fatal errors via message", () => {
+    expect(isFatalConnectionError(new Error("read ECONNRESET"))).toBe(true);
+    expect(isFatalConnectionError(new Error("connect ECONNREFUSED 127.0.0.1:5432"))).toBe(true);
+  });
+
+  it("detects fatal errors via error.code", () => {
+    const err = new Error("connection lost");
+    (err as NodeJS.ErrnoException).code = "ECONNRESET";
+    expect(isFatalConnectionError(err)).toBe(true);
+  });
+
+  it("detects fatal errors via cause chain", () => {
+    const cause = new Error("read ECONNRESET");
+    const wrapper = new Error("query failed", { cause });
+    expect(isFatalConnectionError(wrapper)).toBe(true);
+  });
+
+  it("detects fatal errors via cause.code", () => {
+    const cause = new Error("socket closed");
+    (cause as NodeJS.ErrnoException).code = "EPIPE";
+    const wrapper = new Error("query failed", { cause });
+    expect(isFatalConnectionError(wrapper)).toBe(true);
+  });
+
+  it("returns false for non-fatal errors", () => {
+    expect(isFatalConnectionError(new Error("permission denied"))).toBe(false);
+    expect(isFatalConnectionError(new Error("syntax error"))).toBe(false);
+  });
+
+  it("handles non-Error values", () => {
+    expect(isFatalConnectionError("ECONNRESET")).toBe(true);
+    expect(isFatalConnectionError("permission denied")).toBe(false);
+    expect(isFatalConnectionError(42)).toBe(false);
+  });
+
+  it("MySQL-specific fatal codes are detected alongside generic ones", () => {
+    // MySQL profiler uses: PROTOCOL_CONNECTION_LOST | ER_SERVER_SHUTDOWN | ...
+    // These are checked separately in the MySQL table-level catch, not via isFatalConnectionError.
+    // But the generic codes still work:
+    const err = new Error("Connection lost: The server closed the connection. ECONNRESET");
+    expect(isFatalConnectionError(err)).toBe(true);
+  });
+
+  it("Snowflake-specific fatal codes are numeric and checked separately", () => {
+    // Snowflake uses 390100 (auth expired), 390114 (auth invalid), 250001 (connection failure)
+    // These are NOT part of FATAL_ERROR_PATTERN — they are checked via /390100|390114|250001/
+    // in the Snowflake table-level catch. Verify they don't accidentally match:
+    expect(isFatalConnectionError(new Error("SQL error 390100: auth token expired"))).toBe(false);
+    // But generic codes still work for Snowflake:
+    expect(isFatalConnectionError(new Error("ECONNREFUSED"))).toBe(true);
+  });
+});
+
+describe("DuckDB profiler — error propagation behavior", () => {
+  let tmpDir: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "atlas-fatal-test-"));
+  });
+
+  afterEach(() => {
+    if (tmpDir && fs.existsSync(tmpDir)) {
+      fs.rmSync(tmpDir, { recursive: true, force: true });
+    }
+  });
+
+  it("corrupted database triggers a throw (not silent continuation)", async () => {
+    // Run in a subprocess because corrupting a DuckDB file intermittently
+    // triggers a Bun segfault (~1 in 3 runs). Subprocess isolation ensures
+    // the test runner isn't killed. See #992 and:
+    // https://bun.report/1.3.11/lt1af24e28g2EughgC48hp5EA2AA
+    const script = `
+      const fs = require("fs");
+      const path = require("path");
+      const os = require("os");
+      const { ingestIntoDuckDB, profileDuckDB } = require("../atlas");
+
+      (async () => {
+        const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "atlas-fatal-sub-"));
+        try {
+          const csvPath = path.join(tmpDir, "test.csv");
+          fs.writeFileSync(csvPath, "id,name\\n1,Alice\\n2,Bob\\n");
+          const dbPath = path.join(tmpDir, "test.duckdb");
+          await ingestIntoDuckDB(dbPath, [{ path: csvPath, format: "csv" }]);
+
+          // Corrupt the database file
+          const fd = fs.openSync(dbPath, "r+");
+          const garbage = Buffer.alloc(4096, 0xff);
+          fs.writeSync(fd, garbage, 0, garbage.length, 1024);
+          fs.closeSync(fd);
+
+          await profileDuckDB(dbPath);
+          // If we get here, the corruption wasn't detected
+          process.exit(2);
+        } catch {
+          // Expected — corruption detected and thrown
+          process.exit(0);
+        } finally {
+          fs.rmSync(tmpDir, { recursive: true, force: true });
+        }
+      })();
+    `;
+
+    const proc = Bun.spawn(["bun", "-e", script], {
+      cwd: path.dirname(path.dirname(import.meta.path)),
+      stdout: "ignore",
+      stderr: "ignore",
+    });
+    const exitCode = await proc.exited;
+
+    // exit 0 = threw as expected, exit 2 = no throw (bug), anything else = segfault/crash
+    if (exitCode !== 0 && exitCode !== 2) {
+      // Subprocess crashed (likely segfault) — treat as expected since the
+      // corruption was severe enough to crash DuckDB, which is acceptable
+      // error propagation behavior (the process didn't silently continue).
+      return;
+    }
+    expect(exitCode).toBe(0);
+  });
+
+  it("valid database profiles successfully with all columns", async () => {
+    // A valid database should profile without errors — all columns resolve
+    const csvPath = path.join(tmpDir, "data.csv");
+    fs.writeFileSync(csvPath, "id,value\n1,hello\n2,world\n");
+
+    const dbPath = path.join(tmpDir, "test.duckdb");
+    await ingestIntoDuckDB(dbPath, [{ path: csvPath, format: "csv" }]);
+
+    const result = await profileDuckDB(dbPath);
+    expect(result.profiles).toHaveLength(1);
+    expect(result.errors).toHaveLength(0);
+    expect(result.profiles[0].columns.length).toBe(2);
+  });
+});
+
+describe("column-level catch re-throw contract", () => {
+  it("isFatalConnectionError triggers re-throw, non-fatal continues", () => {
+    // Simulates the exact pattern used in all 6 profilers' column-level catches:
+    //
+    //   } catch (colErr) {
+    //     if (isFatalConnectionError(colErr)) throw colErr;
+    //     console.warn(`Warning: ...`);
+    //   }
+
+    const fatalError = new Error("read ECONNRESET");
+    const nonFatalError = new Error("permission denied for relation users");
+
+    function columnCatch(err: Error): "warning" | "rethrow" {
+      if (isFatalConnectionError(err)) throw err;
+      return "warning";
+    }
+
+    expect(() => columnCatch(fatalError)).toThrow("ECONNRESET");
+    expect(columnCatch(nonFatalError)).toBe("warning");
+  });
+
+  it("fatal errors via .code also trigger re-throw", () => {
+    const err = new Error("connection lost");
+    (err as NodeJS.ErrnoException).code = "ECONNREFUSED";
+
+    function columnCatch(e: Error): "warning" | "rethrow" {
+      if (isFatalConnectionError(e)) throw e;
+      return "warning";
+    }
+
+    expect(() => columnCatch(err)).toThrow("connection lost");
+  });
+
+  it("wrapped fatal errors propagate via cause chain", () => {
+    const original = new Error("read EPIPE");
+    const wrapped = new Error("query failed", { cause: original });
+
+    function columnCatch(e: Error): "warning" | "rethrow" {
+      if (isFatalConnectionError(e)) throw e;
+      return "warning";
+    }
+
+    expect(() => columnCatch(wrapped)).toThrow("query failed");
+  });
+
+  it("all six fatal error codes trigger the re-throw path", () => {
+    const codes = ["ECONNRESET", "ECONNREFUSED", "EHOSTUNREACH", "ENOTFOUND", "EPIPE", "ETIMEDOUT"];
+
+    for (const code of codes) {
+      const err = new Error(`connect ${code}: connection lost`);
+      expect(() => {
+        if (isFatalConnectionError(err)) throw err;
+      }).toThrow(code);
+    }
+  });
+
+  it("table-level catch wraps fatal errors with profiling context", () => {
+    const originalError = new Error("read ECONNRESET");
+    const tableName = "users";
+
+    try {
+      // Simulate column-level re-throw
+      throw originalError;
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (isFatalConnectionError(err)) {
+        const wrapped = new Error(`Fatal database error while profiling ${tableName}: ${msg}`, { cause: err });
+        expect(wrapped.message).toContain("Fatal database error");
+        expect(wrapped.message).toContain("users");
+        expect(wrapped.message).toContain("ECONNRESET");
+        expect(wrapped.cause).toBe(originalError);
+        return; // test passed
+      }
+    }
+    // Should not reach here
+    expect(true).toBe(false);
+  });
+});

package/templates/nextjs-standalone/bin/__tests__/profiler-heuristics.test.ts

@@ -595,14 +595,14 @@ describe("analyzeTableProfiles", () => {
       primary_key_columns: ["id"],
     });
 
-    analyzeTableProfiles([agents, heartbeats, legacy]);
+    const [analyzedAgents, analyzedHeartbeats, analyzedLegacy] = analyzeTableProfiles([agents, heartbeats, legacy]);
 
     // FK inference worked
-    expect(heartbeats.inferred_foreign_keys).toHaveLength(1);
+    expect(analyzedHeartbeats.inferred_foreign_keys).toHaveLength(1);
     // Abandoned detection worked
-    expect(legacy.table_flags.possibly_abandoned).toBe(true);
+    expect(analyzedLegacy.table_flags.possibly_abandoned).toBe(true);
     // Arrays initialized
-    expect(agents.profiler_notes).toEqual([]);
-    expect(agents.table_flags).toBeDefined();
+    expect(analyzedAgents.profiler_notes).toEqual([]);
+    expect(analyzedAgents.table_flags).toBeDefined();
   });
 });

package/templates/nextjs-standalone/bin/__tests__/schema-drift.test.ts

@@ -382,6 +382,45 @@ describe("computeDiff", () => {
     expect(diff.removedTables).toEqual(["customers", "orders"]);
     expect(diff.tableDiffs).toEqual([]);
   });
+
+  test("ignores dimension_table vs fact_table type difference (semantic classification)", () => {
+    const dbSnap: EntitySnapshot = {
+      table: "companies",
+      columns: new Map([["id", "number"]]),
+      foreignKeys: new Set(),
+      objectType: "fact_table",
+    };
+    const ymlSnap: EntitySnapshot = {
+      table: "companies",
+      columns: new Map([["id", "number"]]),
+      foreignKeys: new Set(),
+      objectType: "dimension_table",
+    };
+    const db = new Map([["companies", dbSnap]]);
+    const yml = new Map([["companies", ymlSnap]]);
+    const diff = computeDiff(db, yml);
+    expect(diff.tableDiffs).toEqual([]);
+  });
+
+  test("still flags real type changes (table vs view)", () => {
+    const dbSnap: EntitySnapshot = {
+      table: "orders",
+      columns: new Map([["id", "number"]]),
+      foreignKeys: new Set(),
+      objectType: "view",
+    };
+    const ymlSnap: EntitySnapshot = {
+      table: "orders",
+      columns: new Map([["id", "number"]]),
+      foreignKeys: new Set(),
+      objectType: "fact_table",
+    };
+    const db = new Map([["orders", dbSnap]]);
+    const yml = new Map([["orders", ymlSnap]]);
+    const diff = computeDiff(db, yml);
+    expect(diff.tableDiffs).toHaveLength(1);
+    expect(diff.tableDiffs[0].metadataChanges).toContain("type changed: fact_table → view");
+  });
 });
 
 // --- formatDiff ---