ardent-cli 0.0.38 → 0.0.40

package/dist/index.js

@@ -460,9 +460,13 @@ function getAnonymousId() {
 }
 function trackEvent(event, properties = {}) {
   const distinctId = getConfig("userId") ?? getAnonymousId();
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 500);
+  timeout.unref?.();
   fetch(`${getApiUrl()}/v1/posthog/event`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
+    signal: controller.signal,
     body: JSON.stringify({
       events: [{
         event,
@@ -474,14 +478,18 @@ function trackEvent(event, properties = {}) {
       }]
     })
   }).catch(() => {
-  });
+  }).finally(() => clearTimeout(timeout));
 }
 function identifyUser(userId, personProperties = {}) {
   const anonymousId = getAnonymousId();
   setConfig("userId", userId);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 500);
+  timeout.unref?.();
   fetch(`${getApiUrl()}/v1/posthog/event`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
+    signal: controller.signal,
     body: JSON.stringify({
       events: [{
         event: "$identify",
@@ -494,7 +502,7 @@ function identifyUser(userId, personProperties = {}) {
       }]
     })
   }).catch(() => {
-  });
+  }).finally(() => clearTimeout(timeout));
 }
 
 // src/commands/branch/create.ts
@@ -796,6 +804,126 @@ function printDegradedWarnings(warnings) {
   console.log("  Review this connector with: ardent connector list");
 }
 
+// src/lib/operation_stage_display.ts
+var STAGE_DISPLAY = {
+  // engine_setup_worker / postgres_engine_setup
+  "dispatched": "Starting",
+  "preparing": "Preparing",
+  "creating-neon-project": "Provisioning the branch target",
+  "preparing-target-databases": "Preparing target databases",
+  "deploying-pgstream": "Starting replication",
+  "applying-rls": "Applying RLS policies",
+  "storing-credentials": "Storing connection credentials",
+  "validating": "Validating the branch target",
+  // reset_worker / reset_connector
+  "resetting": "Resetting",
+  "deleting-pgstream": "Stopping replication",
+  "rediscovering-source": "Re-checking the source database",
+  "resetting-neon-main": "Resetting the branch target",
+  "creating-target-schemas": "Recreating target schemas",
+  "redeploying-pgstream": "Restarting replication",
+  // environment deploy_worker
+  "loading_config": "Loading environment configuration",
+  "deploying_infrastructure": "Provisioning environment infrastructure",
+  "recording_success": "Finalizing environment",
+  "cleaning_failed_deploy": "Cleaning up failed environment provisioning",
+  // environment destroy_worker
+  "deleting_private_links": "Removing private network links",
+  "destroying_infrastructure": "Tearing down environment infrastructure",
+  "recording_destroy_success": "Finalizing environment teardown",
+  // discovery_worker / run_discovery_crawl (ARD-1098)
+  "connecting": "Connecting to source",
+  "enumerating_databases": "Listing databases",
+  "scanning_database": "Scanning schema",
+  "writing_schema": "Saving discovered schema",
+  "merging_results": "Combining results"
+};
+function humanizeRawStage(_raw) {
+  return "Working";
+}
+function operationStageDisplay(stage) {
+  if (!stage) return "Working";
+  const colonIndex = stage.indexOf(":");
+  const base = colonIndex === -1 ? stage : stage.slice(0, colonIndex);
+  const suffix = colonIndex === -1 ? "" : stage.slice(colonIndex + 1);
+  const label = STAGE_DISPLAY[base] ?? humanizeRawStage(base);
+  return suffix ? `${label} (for ${suffix})` : label;
+}
+
+// src/lib/discover.ts
+var DiscoveryTimeoutError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "DiscoveryTimeoutError";
+  }
+};
+var DiscoveryOperationFailedError = class extends Error {
+  operationError;
+  constructor(operationError) {
+    super(
+      `Discovery failed: ${operationError ?? "unknown error"}. Inspect the connector with \`ardent connector list\`, fix the underlying issue, then re-run \`ardent connector create\` or \`ardent connector rediscover\`.`
+    );
+    this.name = "DiscoveryOperationFailedError";
+    this.operationError = operationError;
+  }
+};
+async function runDiscoveryWithPolling(connectorId, options) {
+  const maxWaitMs = options?.maxWaitMs ?? 10 * 60 * 1e3;
+  const pollIntervalMs = options?.pollIntervalMs ?? 3 * 1e3;
+  let dispatch;
+  try {
+    dispatch = await api.post(
+      `/v1/connectors/${connectorId}/discover`,
+      {}
+    );
+  } catch (err) {
+    if (isGatewayTimeoutError(err)) {
+      throw new Error(
+        "Backend dispatch timed out. The /discover endpoint is now async and should respond in under a few seconds; this may indicate a backend incident. Try again, or check `ardent connector list` to see if a discovery operation was created anyway."
+      );
+    }
+    throw err;
+  }
+  if (options?.onPrerequisites) {
+    options.onPrerequisites(dispatch.prerequisites, dispatch.source_metadata);
+  }
+  const operationId = dispatch.operation_id;
+  const startedAt = Date.now();
+  let lastStage = null;
+  while (Date.now() - startedAt < maxWaitMs) {
+    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+    let op;
+    try {
+      op = await api.get(`/v1/operations/${operationId}`);
+    } catch (pollErr) {
+      if (isGatewayTimeoutError(pollErr)) continue;
+      throw pollErr;
+    }
+    if (op.stage && op.stage !== lastStage) {
+      const progressLabel = op.progress != null ? ` (${op.progress}%)` : "";
+      console.log(`  ${operationStageDisplay(op.stage)}${progressLabel}`);
+      lastStage = op.stage;
+    }
+    if (op.status === "completed") {
+      const result = op.result ?? {};
+      const schemaVersion = typeof result.schema_version === "number" ? result.schema_version : 0;
+      const databasesCount = typeof result.databases_count === "number" ? result.databases_count : 0;
+      return {
+        prerequisites: dispatch.prerequisites,
+        source_metadata: dispatch.source_metadata,
+        schema_version: schemaVersion,
+        databases_count: databasesCount
+      };
+    }
+    if (op.status === "failed") {
+      throw new DiscoveryOperationFailedError(op.error);
+    }
+  }
+  throw new DiscoveryTimeoutError(
+    `Discovery did not complete within ${Math.round(maxWaitMs / 6e4)} minutes. The catalog walk may still be running server-side \u2014 do NOT delete the connector. Check status with: ardent connector list. If you contact Ardent support, reference operation id ${operationId}.`
+  );
+}
+
 // src/lib/connector_status_display.ts
 var ENGINE_STATUS_DISPLAY = {
   healthy: "ready",
@@ -882,46 +1010,6 @@ function assertEngineSetupCompleted(operation, connectorName) {
   );
 }
 
-// src/lib/operation_stage_display.ts
-var STAGE_DISPLAY = {
-  // engine_setup_worker / postgres_engine_setup
-  "dispatched": "Starting",
-  "preparing": "Preparing",
-  "creating-neon-project": "Provisioning the branch target",
-  "preparing-target-databases": "Preparing target databases",
-  "deploying-pgstream": "Starting replication",
-  "applying-rls": "Applying RLS policies",
-  "storing-credentials": "Storing connection credentials",
-  "validating": "Validating the branch target",
-  // reset_worker / reset_connector
-  "resetting": "Resetting",
-  "deleting-pgstream": "Stopping replication",
-  "rediscovering-source": "Re-checking the source database",
-  "resetting-neon-main": "Resetting the branch target",
-  "creating-target-schemas": "Recreating target schemas",
-  "redeploying-pgstream": "Restarting replication",
-  // environment deploy_worker
-  "loading_config": "Loading environment configuration",
-  "deploying_infrastructure": "Provisioning environment infrastructure",
-  "recording_success": "Finalizing environment",
-  "cleaning_failed_deploy": "Cleaning up failed environment provisioning",
-  // environment destroy_worker
-  "deleting_private_links": "Removing private network links",
-  "destroying_infrastructure": "Tearing down environment infrastructure",
-  "recording_destroy_success": "Finalizing environment teardown"
-};
-function humanizeRawStage(_raw) {
-  return "Working";
-}
-function operationStageDisplay(stage) {
-  if (!stage) return "Working";
-  const colonIndex = stage.indexOf(":");
-  const base = colonIndex === -1 ? stage : stage.slice(0, colonIndex);
-  const suffix = colonIndex === -1 ? "" : stage.slice(colonIndex + 1);
-  const label = STAGE_DISPLAY[base] ?? humanizeRawStage(base);
-  return suffix ? `${label} (for ${suffix})` : label;
-}
-
 // src/lib/engine_setup.ts
 var EngineSetupTimeoutError = class extends Error {
   constructor(message) {
@@ -1697,11 +1785,23 @@ async function createAction2(type, url, options) {
       }
     } else {
       console.log("Discovering schema...");
-      const discoverResult = await api.post(
-        `/v1/connectors/${connectorId}/discover`,
-        {}
-      );
-      const discoveryWarnings = discoverResult?.source_metadata?.warnings ?? [];
+      let discoverPollResult;
+      try {
+        discoverPollResult = await runDiscoveryWithPolling(connectorId, {
+          onPrerequisites: (prereqs) => {
+            if (prereqs.pass) {
+              console.log("Connected \u2713 \u2014 scanning your schema\u2026");
+            }
+          }
+        });
+      } catch (discoverErr) {
+        if (discoverErr instanceof DiscoveryTimeoutError || discoverErr instanceof DiscoveryOperationFailedError) {
+          console.error(`\u2717 ${discoverErr.message}`);
+          process.exit(1);
+        }
+        throw discoverErr;
+      }
+      const discoveryWarnings = discoverPollResult.source_metadata?.warnings ?? [];
       if (discoveryWarnings.length > 0) {
         const yellow = "\x1B[33m";
         const reset2 = "\x1B[0m";
@@ -2944,6 +3044,7 @@ var ORIGINAL_SUFFIX = "-ardent-original";
 var SERVICE_SUFFIXES = ["rest", "auth", "storage", "realtime", "pg_meta"];
 var DOCKER_HOST_GATEWAY = "host.docker.internal";
 var REALTIME_LIST_CHANGES_MIGRATION_VERSION = "20230328144023";
+var MIGRATION_IMPORT_TIMEOUT_MS = 15e3;
 var SENSITIVE_DOCKER_ENV_KEYS = /* @__PURE__ */ new Set([
   "DATABASE_URL",
   "DB_PASSWORD",
@@ -3521,19 +3622,68 @@ function dumpLocalStorageMigrations(dbContainer) {
     "--no-privileges"
   ]);
 }
+function dumpLocalRealtimeMigrations(dbContainer) {
+  return runDocker([
+    "exec",
+    dbContainer,
+    "pg_dump",
+    "-U",
+    "postgres",
+    "-d",
+    "postgres",
+    "--data-only",
+    "--table=realtime.schema_migrations",
+    "--column-inserts",
+    "--no-owner",
+    "--no-privileges"
+  ]);
+}
 function storageMigrationsImportSql(dumpSql) {
   const tempTable = "pg_temp.ardent_storage_migrations_import";
-  const tempDumpSql = dumpSql.replaceAll("INSERT INTO storage.migrations", `INSERT INTO ${tempTable}`);
+  const tempDumpSql = retargetPgDumpInserts(dumpSql, "storage", "migrations", tempTable);
   return `
+SET lock_timeout = '2s';
+SET statement_timeout = '10s';
 BEGIN;
 CREATE TEMP TABLE ardent_storage_migrations_import (LIKE storage.migrations INCLUDING DEFAULTS);
 ${tempDumpSql}
 INSERT INTO storage.migrations
 SELECT * FROM ${tempTable}
 ON CONFLICT DO NOTHING;
+COMMIT;
+  `;
+}
+function realtimeMigrationsImportSql(dumpSql) {
+  const tempTable = "pg_temp.ardent_realtime_migrations_import";
+  const tempDumpSql = retargetPgDumpInserts(dumpSql, "realtime", "schema_migrations", tempTable);
+  return `
+SET lock_timeout = '2s';
+SET statement_timeout = '10s';
+BEGIN;
+CREATE SCHEMA IF NOT EXISTS realtime;
+CREATE TABLE IF NOT EXISTS realtime.schema_migrations (
+  version bigint PRIMARY KEY,
+  inserted_at timestamp(0) without time zone
+);
+CREATE TEMP TABLE ardent_realtime_migrations_import (LIKE realtime.schema_migrations INCLUDING DEFAULTS);
+${tempDumpSql}
+INSERT INTO realtime.schema_migrations
+SELECT * FROM ${tempTable}
+ON CONFLICT DO NOTHING;
 COMMIT;
 `;
 }
+function retargetPgDumpInserts(dumpSql, schema, table, tempTable) {
+  const insertPattern = new RegExp(
+    `INSERT INTO (?:${schema}\\.${table}|"${schema}"\\."${table}")`,
+    "g"
+  );
+  const tempDumpSql = dumpSql.replace(insertPattern, `INSERT INTO ${tempTable}`);
+  if (/\bINSERT INTO\b/.test(dumpSql) && tempDumpSql === dumpSql) {
+    throw new Error(`Could not rewrite pg_dump INSERT statements for ${schema}.${table}.`);
+  }
+  return tempDumpSql;
+}
 function waitForOriginalDb(containerName2) {
   const deadline = Date.now() + 3e4;
   while (Date.now() < deadline) {
@@ -3546,6 +3696,81 @@ function waitForOriginalDb(containerName2) {
   }
   throw new Error(`Timed out waiting for ${containerName2} to accept local PostgreSQL connections.`);
 }
+function waitForBranchProxy(originalDbContainer, dbContainer, network, branchUser, branchPassword) {
+  runDocker(["start", originalDbContainer]);
+  try {
+    waitForOriginalDb(originalDbContainer);
+    const dbContainerIpAddress = containerIpAddress(dbContainer, network);
+    const deadline = Date.now() + 6e4;
+    let lastError = "";
+    while (Date.now() < deadline) {
+      try {
+        runDocker([
+          "exec",
+          "-e",
+          `PGPASSWORD=${branchPassword}`,
+          originalDbContainer,
+          "psql",
+          "-h",
+          dbContainerIpAddress,
+          "-p",
+          "5432",
+          "-U",
+          branchUser,
+          "-d",
+          "postgres",
+          "-v",
+          "ON_ERROR_STOP=1",
+          "-c",
+          "SELECT 1"
+        ]);
+        return;
+      } catch (error) {
+        lastError = error instanceof Error ? error.message : String(error);
+        execFileSync("sleep", ["1"]);
+      }
+    }
+    throw new Error(`Timed out waiting for branch DB proxy to accept connections: ${lastError}`);
+  } finally {
+    runDocker(["stop", originalDbContainer]);
+  }
+}
+function waitForBranchProxyWithClientImage(clientImage, network, dbContainer, branchUser, branchPassword) {
+  const deadline = Date.now() + 6e4;
+  let lastError = "";
+  while (Date.now() < deadline) {
+    try {
+      runDocker([
+        "run",
+        "--rm",
+        "--network",
+        network,
+        "--entrypoint",
+        "psql",
+        "-e",
+        `PGPASSWORD=${branchPassword}`,
+        clientImage,
+        "-h",
+        dbContainer,
+        "-p",
+        "5432",
+        "-U",
+        branchUser,
+        "-d",
+        "postgres",
+        "-v",
+        "ON_ERROR_STOP=1",
+        "-c",
+        "SELECT 1"
+      ]);
+      return;
+    } catch (error) {
+      lastError = error instanceof Error ? error.message : String(error);
+      execFileSync("sleep", ["1"]);
+    }
+  }
+  throw new Error(`Timed out waiting for final branch DB proxy to accept connections: ${lastError}`);
+}
 function copyLocalStorageMigrationsToBranch(dumpSql, originalDbContainer, dbContainer, network, branchUser, branchPassword) {
   runDocker(["start", originalDbContainer]);
   try {
@@ -3570,7 +3795,39 @@ function copyLocalStorageMigrationsToBranch(dumpSql, originalDbContainer, dbCont
         "-v",
         "ON_ERROR_STOP=1"
       ],
-      storageMigrationsImportSql(dumpSql)
+      storageMigrationsImportSql(dumpSql),
+      { timeoutMs: MIGRATION_IMPORT_TIMEOUT_MS }
+    );
+  } finally {
+    runDocker(["stop", originalDbContainer]);
+  }
+}
+function copyLocalRealtimeMigrationsToBranch(dumpSql, originalDbContainer, dbContainer, network, branchPassword) {
+  runDocker(["start", originalDbContainer]);
+  try {
+    waitForOriginalDb(originalDbContainer);
+    const dbContainerIpAddress = containerIpAddress(dbContainer, network);
+    runDockerWithInput(
+      [
+        "exec",
+        "-i",
+        "-e",
+        `PGPASSWORD=${branchPassword}`,
+        originalDbContainer,
+        "psql",
+        "-h",
+        dbContainerIpAddress,
+        "-p",
+        "5432",
+        "-U",
+        "supabase_admin",
+        "-d",
+        "postgres",
+        "-v",
+        "ON_ERROR_STOP=1"
+      ],
+      realtimeMigrationsImportSql(dumpSql),
+      { timeoutMs: MIGRATION_IMPORT_TIMEOUT_MS }
     );
   } finally {
     runDocker(["stop", originalDbContainer]);
@@ -3691,7 +3948,6 @@ function applySupabaseRealtimeCompatibility(projectId, originalDbContainer, dbCo
           supabaseRealtimeCompatibilitySql(),
           { timeoutMs: 7e3 }
         );
-        runDocker(["restart", realtimeContainer]);
         return;
       } catch (error) {
         lastError = error instanceof Error ? error.message : String(error);
@@ -3713,6 +3969,42 @@ function kongApiUrl(projectId) {
   const port = first?.split(":").pop();
   return port ? `http://127.0.0.1:${port}` : void 0;
 }
+function requireLinkedServiceContainersRunning(projectId) {
+  const stoppedServices = [];
+  for (const servicePrefix of SERVICE_SUFFIXES) {
+    const name = containerName(servicePrefix, projectId);
+    if (!dockerExists(name)) {
+      continue;
+    }
+    const state = runDocker([
+      "inspect",
+      "-f",
+      "{{.State.Status}}",
+      name
+    ]);
+    if (state !== "running") {
+      stoppedServices.push(`${name}=${state}`);
+    }
+  }
+  if (stoppedServices.length > 0) {
+    throw new Error(`Linked Supabase service container is not running: ${stoppedServices.join(", ")}`);
+  }
+}
+async function closeHealthResponse(response) {
+  try {
+    await response.body?.cancel();
+  } catch {
+  }
+}
+async function fetchHealthCheck(url) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 5e3);
+  try {
+    return await fetch(url, { signal: controller.signal });
+  } finally {
+    clearTimeout(timeout);
+  }
+}
 async function waitForHealth(projectId) {
   const apiUrl = kongApiUrl(projectId);
   if (!apiUrl) {
@@ -3722,12 +4014,22 @@ async function waitForHealth(projectId) {
   let lastError = "";
   while (Date.now() < deadline) {
     try {
-      const authResponse = await fetch(`${apiUrl}/auth/v1/health`);
-      const restResponse = await fetch(`${apiUrl}/rest/v1/`);
-      if (authResponse.ok && restResponse.ok) {
-        return;
+      requireLinkedServiceContainersRunning(projectId);
+      let authResponse;
+      let restResponse;
+      try {
+        authResponse = await fetchHealthCheck(`${apiUrl}/auth/v1/health`);
+        restResponse = await fetchHealthCheck(`${apiUrl}/rest/v1/`);
+        if (authResponse.ok && restResponse.ok) {
+          return;
+        }
+        lastError = `auth=${authResponse.status}, rest=${restResponse.status}`;
+      } finally {
+        await Promise.all([
+          authResponse ? closeHealthResponse(authResponse) : Promise.resolve(),
+          restResponse ? closeHealthResponse(restResponse) : Promise.resolve()
+        ]);
       }
-      lastError = `auth=${authResponse.status}, rest=${restResponse.status}`;
     } catch (error) {
       lastError = error instanceof Error ? error.message : String(error);
     }
@@ -3794,11 +4096,16 @@ async function linkSupabaseAction(branchName, options = {}) {
     }
     console.log(`Linking Supabase project ${projectId} to branch ${branch.name}...`);
     const storageMigrationsDump = dumpLocalStorageMigrations(dbContainer);
+    const realtimeContainer = containerName("realtime", projectId);
+    const realtimeMigrationsDump = dockerExists(realtimeContainer) ? dumpLocalRealtimeMigrations(dbContainer) : "";
     const originalDbContainerState = inspectContainer(dbContainer);
     const dbPortBindings = portPublishArgs(originalDbContainerState);
     runDocker(["stop", dbContainer]);
     runDocker(["rename", dbContainer, originalDbContainer]);
     startProxyContainer(projectId, network, dbContainer, parsedBranchUrl, []);
+    console.log("Checking branch database connection...");
+    waitForBranchProxy(originalDbContainer, dbContainer, network, branchUser, branchPassword);
+    console.log("Copying local Supabase Storage migration state...");
     copyLocalStorageMigrationsToBranch(
       storageMigrationsDump,
       originalDbContainer,
@@ -3807,19 +4114,17 @@ async function linkSupabaseAction(branchName, options = {}) {
       branchUser,
       branchPassword
     );
-    for (const servicePrefix of SERVICE_SUFFIXES) {
-      const service = recreateServiceForBranch(
-        servicePrefix,
-        projectId,
-        network,
+    if (realtimeMigrationsDump) {
+      console.log("Copying local Supabase Realtime migration state...");
+      copyLocalRealtimeMigrationsToBranch(
+        realtimeMigrationsDump,
+        originalDbContainer,
         dbContainer,
-        branchUser,
+        network,
         branchPassword
       );
-      if (service) {
-        createdServices.push(service);
-      }
     }
+    console.log("Applying Supabase Realtime compatibility...");
     applySupabaseRealtimeCompatibility(
       projectId,
       originalDbContainer,
@@ -3829,6 +4134,27 @@ async function linkSupabaseAction(branchName, options = {}) {
     );
     runDocker(["rm", "-f", dbContainer]);
     startProxyContainer(projectId, network, dbContainer, parsedBranchUrl, dbPortBindings);
+    console.log("Starting linked Supabase database proxy...");
+    waitForBranchProxyWithClientImage(
+      originalDbContainerState.Config.Image,
+      network,
+      dbContainer,
+      branchUser,
+      branchPassword
+    );
+    for (const servicePrefix of SERVICE_SUFFIXES) {
+      const service = recreateServiceForBranch(
+        servicePrefix,
+        projectId,
+        network,
+        dbContainer,
+        branchUser,
+        branchPassword
+      );
+      if (service) {
+        createdServices.push(service);
+      }
+    }
     const kong = containerName("kong", projectId);
     if (dockerExists(kong)) {
       runDocker(["restart", kong]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ardent-cli",
-  "version": "0.0.38",
+  "version": "0.0.40",
   "description": "Git for Data infrastructure",
   "type": "module",
   "bin": {