convex 1.46.0 → 1.47.0

package/src/cli/lib/checkForLargeIndexDeletion.ts

@@ -11,9 +11,12 @@ import { formatIndex } from "./indexes.js";
 import { promptYesNo } from "./utils/prompts.js";
 import { Span } from "./tracing.js";
 import { StartPushRequest } from "./deployApi/startPush.js";
-import { evaluatePush, evaluateSchema } from "./deploy2.js";
+import { evaluatePush } from "./deploy2.js";
 import { DeveloperIndexConfig } from "./deployApi/finishPush.js";
+import { EvaluateSchemaResponse } from "./deployApi/evaluateSchema.js";
 import { runSystemQuery } from "./run.js";
+import { SchemaEvaluation } from "./schemaEvaluation.js";
+import { integerFromEnv } from "./utils/utils.js";
 
 const MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING = 100_000;
 
@@ -37,24 +40,35 @@ export async function checkForLargeIndexDeletion({
   span,
   request,
   options,
+  schemaEvaluation,
   askForConfirmation,
 }: {
   ctx: Context;
   span: Span;
   request: StartPushRequest;
   options: CheckOptions;
+  schemaEvaluation: SchemaEvaluation;
   askForConfirmation: boolean;
 }): Promise<void> {
   changeSpinner("Verifying that the push isn’t deleting large indexes...");
 
-  const deletedIndexes =
-    (await deletedIndexesFromSchemaEvaluation({
+  const schemaEvaluationResponse = await schemaEvaluation(span);
+  let deletedIndexes: DeletedIndex[];
+  if (schemaEvaluationResponse !== null) {
+    deletedIndexes = deletedIndexesFromSchemaEvaluation(
+      schemaEvaluationResponse,
+    );
+  } else {
+    logVerbose(
+      "Schema evaluation unavailable, sizing deleted indexes per table",
+    );
+    deletedIndexes = await deletedIndexesFromTableSizes({
       ctx,
       span,
       request,
       options,
-    })) ??
-    (await deletedIndexesFromTableSizes({ ctx, span, request, options }));
+    });
+  }
 
   if (deletedIndexes.length === 0) {
     logFinishedStep("No indexes are deleted by this push");
@@ -84,9 +98,11 @@ The documents that are in the index won’t be deleted, but the index will need
 to be backfilled again if you want to restore it later.
 `);
 
+  // Either --skip-large-indexes-check or its deletion-only predecessor
+  // --allow-deleting-large-indexes gets here, so name neither.
   if (!askForConfirmation) {
     logFinishedStep(
-      "Proceeding with push since --allow-deleting-large-indexes is set",
+      "Proceeding with push since deleting large indexes was allowed by flag",
     );
     return;
   }
@@ -97,7 +113,7 @@ to be backfilled again if you want to restore it later.
       errorType: "fatal",
       printedMessage: `To confirm the push:
 • run the deploy command in an ${chalkStderr.bold("interactive terminal")}
-• or run the deploy command with the ${chalkStderr.bold("--allow-deleting-large-indexes")} flag`,
+• or run the deploy command with the ${chalkStderr.bold("--skip-large-indexes-check")} flag`,
     });
   }
 
@@ -120,49 +136,24 @@ to be backfilled again if you want to restore it later.
 
 // Preferred source: one schema evaluation reports every dropped index
 // together with its table's document count, requiring only the
-// `deployment:deploy` permission the push itself needs. Returns null when the
-// deployment doesn't support the endpoint, in which case the caller falls
-// back to sizing tables one system query at a time.
-export async function deletedIndexesFromSchemaEvaluation({
-  ctx,
-  span,
-  request,
-  options,
-}: {
-  ctx: Context;
-  span: Span;
-  request: StartPushRequest;
-  options: CheckOptions;
-}): Promise<DeletedIndex[] | null> {
-  try {
-    const response = await evaluateSchema(
-      ctx,
-      span,
-      request,
-      options,
-      /* bestEffort */ true,
-    );
-    return Object.entries(response.componentSchemaEvaluations).flatMap(
-      ([componentPath, prediction]) =>
-        prediction.indexes
-          .filter((index) => index.change === "dropped")
-          .map((index) => ({
-            componentPath,
-            index,
-            documentsCount: index.numDocs,
-            replacedBy:
-              prediction.indexes.find(
-                (other) =>
-                  other.change === "added" && other.name === index.name,
-              ) ?? null,
-          })),
-    );
-  } catch (error: unknown) {
-    logVerbose(
-      `Schema evaluation unavailable, sizing deleted indexes per table: ${error instanceof Error ? error.message : String(error)}`,
-    );
-    return null;
-  }
+// `deployment:deploy` permission the push itself needs.
+export function deletedIndexesFromSchemaEvaluation(
+  response: EvaluateSchemaResponse,
+): DeletedIndex[] {
+  return Object.entries(response.componentSchemaEvaluations).flatMap(
+    ([componentPath, prediction]) =>
+      prediction.indexes
+        .filter((index) => index.change === "dropped")
+        .map((index) => ({
+          componentPath,
+          index,
+          documentsCount: index.numDocs,
+          replacedBy:
+            prediction.indexes.find(
+              (other) => other.change === "added" && other.name === index.name,
+            ) ?? null,
+        })),
+  );
 }
 
 // Fallback for deployments without `evaluate_schema`: diff indexes with
@@ -268,12 +259,8 @@ function getTableName(index: DeveloperIndexConfig) {
 }
 
 function minDocumentsForIndexDeleteWarning(): number {
-  const envValue = process.env.CONVEX_MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING;
-  if (envValue !== undefined) {
-    const parsed = parseInt(envValue, 10);
-    if (!isNaN(parsed)) {
-      return parsed;
-    }
-  }
-  return MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING;
+  return integerFromEnv(
+    "CONVEX_MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING",
+    MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING,
+  );
 }

package/src/cli/lib/checkForSlowSchemaValidation.ts

@@ -1,41 +1,36 @@
 import { chalkStderr } from "chalk";
-import { Context } from "../../bundler/context.js";
-import { logMessage, logVerbose } from "../../bundler/log.js";
+import { logMessage } from "../../bundler/log.js";
 import { Span } from "./tracing.js";
-import { StartPushRequest } from "./deployApi/startPush.js";
 import {
   EvaluateSchemaResponse,
   TablePrediction,
 } from "./deployApi/evaluateSchema.js";
-import { evaluateSchema } from "./deploy2.js";
-import { formatSize } from "./utils/utils.js";
+import { SchemaEvaluation } from "./schemaEvaluation.js";
+import { formatSize, integerFromEnv } from "./utils/utils.js";
+import {
+  formatDiscardedStagedValidator,
+  formatValidationProgress,
+} from "./formatValidationProgress.js";
 
 const MIN_BYTES_FOR_SCHEMA_WALK_WARNING = 1 << 27; // 128 MiB
 
 export async function checkForSlowSchemaValidation({
-  ctx,
   span,
-  request,
-  options,
+  schemaEvaluation,
+  dryRun,
 }: {
-  ctx: Context;
   span: Span;
-  request: StartPushRequest;
-  options: {
-    url: string;
-    deploymentName: string | null;
-    adminKey: string;
-  };
+  schemaEvaluation: SchemaEvaluation;
+  dryRun: boolean;
 }): Promise<void> {
-  const response = await evaluateSchemaBestEffort({
-    ctx,
-    span,
-    request,
-    options,
-  });
+  const response = await schemaEvaluation(span);
   if (response === null) {
     return;
   }
+  reportStagedValidators(response);
+  if (!dryRun) {
+    return;
+  }
 
   const walkedTables = Object.entries(
     response.componentSchemaEvaluations,
@@ -56,6 +51,9 @@ export async function checkForSlowSchemaValidation({
     return;
   }
 
+  const shouldSuggestStaging = walkedTables.some(
+    ({ table }) => !table.canSkipAfterStagedValidation,
+  );
   logMessage(`⚠️  This schema change requires checking every document in the following ${walkedTables.length === 1 ? "table" : "tables"} against your new schema, totaling ${chalkStderr.yellowBright(formatSize(totalBytes))}. This deploy may take a while:
 
 ${walkedTables
@@ -63,45 +61,58 @@ ${walkedTables
     formatWalkedTable({ componentPath, table }),
   )
   .join("\n")}
-`);
+${
+  shouldSuggestStaging
+    ? `\nTip: stage the new validator (\`defineTable({...}).staged({...})\`) to validate it in the background without blocking deploys.`
+    : ""
+}`);
 }
 
-// This warning is advisory: if evaluation fails, fall back to skipping it
-// rather than aborting the push over a check that isn't required for
-// correctness. Plausibly transient failures — network errors and 5xx
-// responses (e.g. table summaries still bootstrapping) — are retried with
-// backoff by the shared fetch wrapper (see `pushCode`); anything else is
-// deterministic, including a 2xx whose shape the CLI doesn't recognize, so
-// it skips the check immediately.
-async function evaluateSchemaBestEffort({
-  ctx,
-  span,
-  request,
-  options,
-}: {
-  ctx: Context;
-  span: Span;
-  request: StartPushRequest;
-  options: {
-    url: string;
-    deploymentName: string | null;
-    adminKey: string;
-  };
-}): Promise<EvaluateSchemaResponse | null> {
-  try {
-    return await evaluateSchema(
-      ctx,
-      span,
-      request,
-      options,
-      /* bestEffort */ true,
-    );
-  } catch (error: unknown) {
-    logVerbose(
-      `Skipping slow schema validation check: ${error instanceof Error ? error.message : String(error)}`,
+// The prediction includes staged validators from active and in-flight schemas.
+function reportStagedValidators(response: EvaluateSchemaResponse) {
+  const stagedTables = Object.entries(
+    response.componentSchemaEvaluations,
+  ).flatMap(([componentPath, prediction]) => {
+    const discarded = new Map(
+      (prediction.discardedStagedValidators ?? []).map((validator) => [
+        validator.tableName,
+        validator,
+      ]),
     );
-    return null;
+    return prediction.tables
+      .filter((table) => table.staged !== undefined)
+      .map((table) => ({
+        componentPath,
+        table,
+        discarded: discarded.get(table.name),
+      }));
+  });
+  if (stagedTables.length === 0) {
+    return;
   }
+  const lines = stagedTables.map(({ componentPath, table, discarded }) => {
+    const componentPrefix =
+      componentPath !== "" ? `${chalkStderr.gray(componentPath)}: ` : "";
+    // The state describes the validator this push discards, so advertising
+    // it as ready to promote would contradict the unstaging warning.
+    if (discarded !== undefined) {
+      return `  ${componentPrefix}${table.name}: ${formatDiscardedStagedValidator(discarded)}`;
+    }
+    const staged = table.staged!;
+    switch (staged.state) {
+      case "pending": {
+        return `  ${componentPrefix}${table.name}: validating in the background (${formatValidationProgress(staged)})`;
+      }
+      case "valid":
+        // `supersetOfStagedValidated` means this push already promotes it.
+        return table.outcome === "supersetOfStagedValidated"
+          ? `  ${componentPrefix}${table.name}: ${chalkStderr.green("validated")}. This deploy applies the staged schema without scanning the table.`
+          : `  ${componentPrefix}${table.name}: ${chalkStderr.green("validated")}. Safe to apply the staged schema changes and deploy.`;
+      case "failed":
+        return `  ${componentPrefix}${table.name}: ${chalkStderr.red("failed")}: ${staged.error}${chalkStderr.gray(" (fix the offending documents and redeploy to revalidate)")}`;
+    }
+  });
+  logMessage(`Staged schema validation status:\n${lines.join("\n")}\n`);
 }
 
 function formatWalkedTable({
@@ -115,16 +126,16 @@ function formatWalkedTable({
     componentPath !== "" ? `${chalkStderr.gray(componentPath)}: ` : "";
   const docsFormatted = `${table.numDocs.toLocaleString()} documents, `;
   const sizeFormatted = formatSize(table.sizeBytes);
-  return `  ${componentPrefix}${table.name} (${docsFormatted}${sizeFormatted})`;
+  let stagedNote = "";
+  if (table.staged?.state === "pending" && table.canSkipAfterStagedValidation) {
+    stagedNote = `. Staged schema validation in progress. Wait for validation to complete before deploying with the staged changes. Deploying now will block on a full table revalidation scan.`;
+  }
+  return `  ${componentPrefix}${table.name} (${docsFormatted}${sizeFormatted})${stagedNote}`;
 }
 
 function minBytesForSchemaWalkWarning(): number {
-  const envValue = process.env.CONVEX_MIN_BYTES_FOR_SCHEMA_WALK_WARNING;
-  if (envValue !== undefined) {
-    const parsed = parseInt(envValue, 10);
-    if (!isNaN(parsed)) {
-      return parsed;
-    }
-  }
-  return MIN_BYTES_FOR_SCHEMA_WALK_WARNING;
+  return integerFromEnv(
+    "CONVEX_MIN_BYTES_FOR_SCHEMA_WALK_WARNING",
+    MIN_BYTES_FOR_SCHEMA_WALK_WARNING,
+  );
 }

package/src/cli/lib/checkForStagedValidatorDeletion.ts

@@ -0,0 +1,104 @@
+import { chalkStderr } from "chalk";
+import { Context } from "../../bundler/context.js";
+import { logFinishedStep, logMessage, stopSpinner } from "../../bundler/log.js";
+import { promptYesNo } from "./utils/prompts.js";
+import {
+  DiscardedStagedValidator,
+  EvaluateSchemaResponse,
+} from "./deployApi/evaluateSchema.js";
+import { formatDiscardedStagedValidator } from "./formatValidationProgress.js";
+
+export type StagedValidatorDeletionCheck =
+  | "ask for confirmation"
+  | "has confirmation"
+  | "no verification";
+
+/**
+ * Confirm before discarding pending or successful staged validation. Unstaging
+ * stops write-time checks, so restaging requires a fresh scan.
+ *
+ * A dry run only reports what the push would discard, so `--dry-run` previews
+ * the confirmation the real push will ask for.
+ */
+export async function checkForStagedValidatorDeletion({
+  ctx,
+  response,
+  askForConfirmation,
+  dryRun = false,
+}: {
+  ctx: Context;
+  response: EvaluateSchemaResponse | null;
+  askForConfirmation: boolean;
+  dryRun?: boolean;
+}): Promise<void> {
+  // `sharedSchemaEvaluation` has already logged why evaluation failed.
+  if (response === null) {
+    return;
+  }
+  const discarded = Object.entries(response.componentSchemaEvaluations).flatMap(
+    ([componentPath, prediction]) =>
+      (prediction.discardedStagedValidators ?? []).map((validator) => ({
+        componentPath,
+        validator,
+      })),
+  );
+  if (discarded.length === 0) {
+    return;
+  }
+
+  logMessage(`\u26a0\ufe0f  This push ${chalkStderr.bold(dryRun ? "would discard" : "discards")} staged schema validation for the following ${discarded.length === 1 ? "table" : "tables"}:
+
+${discarded.map(formatDiscarded).join("\n")}
+
+A discarded staged validator restarts validation from zero if staged again.
+`);
+
+  if (dryRun) {
+    return;
+  }
+
+  if (!askForConfirmation) {
+    logFinishedStep(
+      "Proceeding with push since --allow-unstaging-schema is set",
+    );
+    return;
+  }
+
+  if (!process.stdin.isTTY) {
+    return ctx.crash({
+      exitCode: 1,
+      errorType: "fatal",
+      printedMessage: `To confirm the push:
+\u2022 run the deploy command in an ${chalkStderr.bold("interactive terminal")}
+\u2022 or run the deploy command with the ${chalkStderr.bold("--allow-unstaging-schema")} flag`,
+    });
+  }
+
+  stopSpinner();
+  if (
+    !(await promptYesNo(ctx, {
+      message: `Discard staged validation for ${discarded.length === 1 ? "this table" : "these tables"}?`,
+      default: false,
+    }))
+  ) {
+    return ctx.crash({
+      exitCode: 1,
+      errorType: "fatal",
+      printedMessage: `Canceling push`,
+    });
+  }
+
+  logFinishedStep("Proceeding with push.");
+}
+
+function formatDiscarded({
+  componentPath,
+  validator,
+}: {
+  componentPath: string;
+  validator: DiscardedStagedValidator;
+}): string {
+  const componentPrefix =
+    componentPath !== "" ? `${chalkStderr.gray(componentPath)}:` : "";
+  return `\u26d4 ${componentPrefix}${validator.tableName}: ${formatDiscardedStagedValidator(validator)}`;
+}

package/src/cli/lib/codegen.ts

@@ -46,7 +46,6 @@ import {
   rootComponentApiCJS,
 } from "../codegen_templates/component_api.js";
 import { functionsDir } from "./utils/utils.js";
-import { LargeIndexDeletionCheck } from "./indexes.js";
 
 const PRESERVED_GENERATED_ENTRIES = new Set(["ai"]);
 
@@ -81,8 +80,6 @@ export type CodegenOptions = {
   liveComponentSources: boolean;
   debugNodeApis: boolean;
   systemUdfs: boolean;
-  largeIndexDeletionCheck: LargeIndexDeletionCheck;
-  warnOnSlowSchemaValidation: boolean;
   codegenOnlyThisComponent?: string | undefined;
 };
 

package/src/cli/lib/command.ts

@@ -260,6 +260,7 @@ export async function normalizeDevOptions(
     debugNodeApis?: boolean;
     liveComponentSources?: boolean;
     pushAllModules?: boolean;
+    allowUnstagingSchema?: boolean;
     while?: string;
   },
 ): Promise<{
@@ -279,6 +280,7 @@ export async function normalizeDevOptions(
   debugNodeApis: boolean;
   liveComponentSources: boolean;
   pushAllModules: boolean;
+  allowUnstagingSchema: boolean;
 }> {
   if (cmdOptions.runComponent && !cmdOptions.run) {
     return await ctx.crash({
@@ -332,6 +334,7 @@ export async function normalizeDevOptions(
     debugNodeApis: !!cmdOptions.debugNodeApis,
     liveComponentSources: !!cmdOptions.liveComponentSources,
     pushAllModules: !!cmdOptions.pushAllModules,
+    allowUnstagingSchema: !!cmdOptions.allowUnstagingSchema,
   };
 }
 

package/src/cli/lib/components/definition/bundle.ts

@@ -645,9 +645,11 @@ export async function bundleImplementations({
     functions: Bundle[];
     definitionPath: ComponentDefinitionPath;
   }[];
+  buildDebugInfos: BuildDebugInfo[];
 }> {
   let appImplementation;
   const componentImplementations = [];
+  const buildDebugInfos: BuildDebugInfo[] = [];
 
   // For --component-dir flag, skip bundling root implementations (no real code to bundle)
   const directoriesToBundle = rootComponentDirectory.syntheticComponentImport
@@ -660,13 +662,25 @@ export async function bundleImplementations({
       rootComponentDirectory.path,
       directory.path,
     );
+    // definitionPath is the canonical form
+    const definitionPath = toComponentDefinitionPath(
+      rootComponentDirectory,
+      directory,
+    );
     let schema;
-    if (ctx.fs.exists(path.resolve(resolvedPath, "schema.ts"))) {
-      schema =
-        (await bundleSchema(ctx, resolvedPath, extraConditions))[0] || null;
-    } else if (ctx.fs.exists(path.resolve(resolvedPath, "schema.js"))) {
-      schema =
-        (await bundleSchema(ctx, resolvedPath, extraConditions))[0] || null;
+    if (
+      ctx.fs.exists(path.resolve(resolvedPath, "schema.ts")) ||
+      ctx.fs.exists(path.resolve(resolvedPath, "schema.js"))
+    ) {
+      const schemaResult = await bundleSchema(
+        ctx,
+        resolvedPath,
+        extraConditions,
+      );
+      buildDebugInfos.push(
+        buildDebugInfo(directory, definitionPath, "schema", schemaResult),
+      );
+      schema = schemaResult.modules[0] || null;
     } else {
       schema = null;
     }
@@ -676,6 +690,7 @@ export async function bundleImplementations({
       modules: Bundle[];
       externalDependencies: Map<string, string>;
       bundledModuleNames: Set<string>;
+      metafile: Metafile | null;
     } = await bundle({
       ctx,
       dir: resolvedPath,
@@ -693,6 +708,9 @@ export async function bundleImplementations({
         printedMessage: "external dependencies not supported",
       });
     }
+    buildDebugInfos.push(
+      buildDebugInfo(directory, definitionPath, "convex", convexResult),
+    );
     const functions = convexResult.modules;
     if (isRoot) {
       if (verbose) {
@@ -702,6 +720,7 @@ export async function bundleImplementations({
         modules: Bundle[];
         externalDependencies: Map<string, string>;
         bundledModuleNames: Set<string>;
+        metafile: Metafile | null;
       } = await bundle({
         ctx,
         dir: resolvedPath,
@@ -713,6 +732,9 @@ export async function bundleImplementations({
         extraConditions,
         includeSourcesContent,
       });
+      buildDebugInfos.push(
+        buildDebugInfo(directory, definitionPath, "node", nodeResult),
+      );
 
       const externalNodeDependencies: NodeDependency[] = [];
       for (const [
@@ -724,10 +746,15 @@ export async function bundleImplementations({
           version: moduleVersion,
         });
       }
-      const authBundle = await bundleAuthConfig(ctx, resolvedPath);
+      const authResult = await bundleAuthConfig(ctx, resolvedPath);
+      buildDebugInfos.push(
+        buildDebugInfo(directory, definitionPath, "auth", authResult),
+      );
       appImplementation = {
         schema,
-        functions: functions.concat(nodeResult.modules).concat(authBundle),
+        functions: functions
+          .concat(nodeResult.modules)
+          .concat(authResult.modules),
         externalNodeDependencies,
       };
     } else {
@@ -757,11 +784,6 @@ export async function bundleImplementations({
           });
         }
       }
-      // definitionPath is the canonical form
-      const definitionPath = toComponentDefinitionPath(
-        rootComponentDirectory,
-        directory,
-      );
       componentImplementations.push({ definitionPath, schema, functions });
     }
   }
@@ -783,7 +805,39 @@ export async function bundleImplementations({
     }
   }
 
-  return { appImplementation, componentImplementations };
+  return { appImplementation, componentImplementations, buildDebugInfos };
+}
+
+/**
+ * Debug information for a build in a push.
+ */
+export type BuildDebugInfo = {
+  directory: ComponentDirectory;
+  definitionPath: ComponentDefinitionPath;
+  kind: "convex" | "node" | "schema" | "auth";
+  metafile: Metafile | null;
+  modules: Bundle[];
+  externalDependencies: Map<string, string>;
+};
+
+function buildDebugInfo(
+  directory: ComponentDirectory,
+  definitionPath: ComponentDefinitionPath,
+  kind: BuildDebugInfo["kind"],
+  result: {
+    modules: Bundle[];
+    metafile: Metafile | null;
+    externalDependencies?: Map<string, string>;
+  },
+): BuildDebugInfo {
+  return {
+    directory,
+    definitionPath,
+    kind,
+    metafile: result.metafile,
+    modules: result.modules,
+    externalDependencies: result.externalDependencies ?? new Map(),
+  };
 }
 
 async function registerEsbuildReads(