convex 1.45.0 → 1.47.0

package/src/cli/lib/checkForLargeIndexDeletion.ts

@@ -4,6 +4,7 @@ import {
   changeSpinner,
   logFinishedStep,
   logMessage,
+  logVerbose,
   stopSpinner,
 } from "../../bundler/log.js";
 import { formatIndex } from "./indexes.js";
@@ -11,107 +12,85 @@ import { promptYesNo } from "./utils/prompts.js";
 import { Span } from "./tracing.js";
 import { StartPushRequest } from "./deployApi/startPush.js";
 import { evaluatePush } from "./deploy2.js";
-import { DeveloperIndexConfig, IndexDiff } from "./deployApi/finishPush.js";
+import { DeveloperIndexConfig } from "./deployApi/finishPush.js";
+import { EvaluateSchemaResponse } from "./deployApi/evaluateSchema.js";
 import { runSystemQuery } from "./run.js";
+import { SchemaEvaluation } from "./schemaEvaluation.js";
+import { integerFromEnv } from "./utils/utils.js";
 
 const MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING = 100_000;
 
+type CheckOptions = {
+  url: string;
+  deploymentName: string | null;
+  adminKey: string;
+};
+
+type DeletedIndex = {
+  // The component instance path ("" for the root component), matching the
+  // keys of both `componentSchemaEvaluations` and `indexDiffs`.
+  componentPath: string;
+  index: DeveloperIndexConfig;
+  documentsCount: number;
+  replacedBy: DeveloperIndexConfig | null;
+};
+
 export async function checkForLargeIndexDeletion({
   ctx,
   span,
   request,
   options,
+  schemaEvaluation,
   askForConfirmation,
 }: {
   ctx: Context;
   span: Span;
   request: StartPushRequest;
-  options: {
-    url: string;
-    deploymentName: string | null;
-    adminKey: string;
-  };
+  options: CheckOptions;
+  schemaEvaluation: SchemaEvaluation;
   askForConfirmation: boolean;
 }): Promise<void> {
   changeSpinner("Verifying that the push isn’t deleting large indexes...");
 
-  const { schemaChange } = await evaluatePush(ctx, span, request, options);
-
-  const indexDiffs = schemaChange.indexDiffs ?? {};
-  const deletedIndexes = Object.entries(indexDiffs).flatMap(
-    ([componentDefinitionPath, indexDiff]) =>
-      indexDiff.removed_indexes.map((index) => ({
-        componentDefinitionPath,
-        index,
-      })),
-  );
+  const schemaEvaluationResponse = await schemaEvaluation(span);
+  let deletedIndexes: DeletedIndex[];
+  if (schemaEvaluationResponse !== null) {
+    deletedIndexes = deletedIndexesFromSchemaEvaluation(
+      schemaEvaluationResponse,
+    );
+  } else {
+    logVerbose(
+      "Schema evaluation unavailable, sizing deleted indexes per table",
+    );
+    deletedIndexes = await deletedIndexesFromTableSizes({
+      ctx,
+      span,
+      request,
+      options,
+    });
+  }
 
   if (deletedIndexes.length === 0) {
     logFinishedStep("No indexes are deleted by this push");
     return;
   }
 
-  const tablesWithDeletedIndexes = [
-    ...new Set(
-      deletedIndexes.map(
-        ({ componentDefinitionPath, index }) =>
-          `${componentDefinitionPath}:${getTableName(index)}`,
-      ),
-    ),
-  ].map((str) => {
-    const [componentDefinitionPath, table] = str.split(":");
-    return { componentDefinitionPath, table };
-  });
-  changeSpinner("Checking whether the deleted indexes are on large tables...");
-  const documentCounts = await Promise.all(
-    tablesWithDeletedIndexes.map(
-      async ({ componentDefinitionPath, table }) => ({
-        componentDefinitionPath,
-        table,
-        count: (await runSystemQuery(ctx, {
-          deploymentUrl: options.url,
-          adminKey: options.adminKey,
-          functionName: "_system/cli/tableSize:default",
-          componentPath: componentDefinitionPath,
-          args: { tableName: table },
-        })) as number,
-      }),
-    ),
-  );
-  const deletedIndexesWithDocumentsCount = deletedIndexes.map(
-    ({ componentDefinitionPath, index }) => ({
-      componentDefinitionPath,
-      index,
-      count: documentCounts.find(
-        (count) =>
-          count.table === getTableName(index) &&
-          count.componentDefinitionPath === componentDefinitionPath,
-      )!.count,
-    }),
-  );
-
   const minDocumentsForWarning = minDocumentsForIndexDeleteWarning();
   if (
-    !deletedIndexesWithDocumentsCount.some(
-      ({ count }) => count >= minDocumentsForWarning,
+    !deletedIndexes.some(
+      ({ documentsCount }) => documentsCount >= minDocumentsForWarning,
     )
   ) {
     logFinishedStep("No large indexes are deleted by this push");
     return;
   }
 
-  logMessage(`⚠️  This code push will ${chalkStderr.bold("delete")} the following ${deletedIndexesWithDocumentsCount.length === 1 ? "index" : "indexes"}
+  logMessage(`⚠️  This code push will ${chalkStderr.bold("delete")} the following ${deletedIndexes.length === 1 ? "index" : "indexes"}
 from your production deployment (${options.url}):
 
-${deletedIndexesWithDocumentsCount
-  .map(({ componentDefinitionPath, index, count }) =>
-    formatDeletedIndex({
-      componentDefinitionPath,
-      index,
-      indexDiff: indexDiffs[componentDefinitionPath],
-      documentsCount: count,
-      minDocumentsForWarning,
-    }),
+${deletedIndexes
+  .map((deletedIndex) =>
+    formatDeletedIndex({ ...deletedIndex, minDocumentsForWarning }),
   )
   .join("\n")}
 
@@ -119,9 +98,11 @@ The documents that are in the index won’t be deleted, but the index will need
 to be backfilled again if you want to restore it later.
 `);
 
+  // Either --skip-large-indexes-check or its deletion-only predecessor
+  // --allow-deleting-large-indexes gets here, so name neither.
   if (!askForConfirmation) {
     logFinishedStep(
-      "Proceeding with push since --allow-deleting-large-indexes is set",
+      "Proceeding with push since deleting large indexes was allowed by flag",
     );
     return;
   }
@@ -132,14 +113,14 @@ to be backfilled again if you want to restore it later.
       errorType: "fatal",
       printedMessage: `To confirm the push:
 • run the deploy command in an ${chalkStderr.bold("interactive terminal")}
-• or run the deploy command with the ${chalkStderr.bold("--allow-deleting-large-indexes")} flag`,
+• or run the deploy command with the ${chalkStderr.bold("--skip-large-indexes-check")} flag`,
     });
   }
 
   stopSpinner();
   if (
     !(await promptYesNo(ctx, {
-      message: `Delete ${deletedIndexesWithDocumentsCount.length === 1 ? "this index" : "these indexes"}?`,
+      message: `Delete ${deletedIndexes.length === 1 ? "this index" : "these indexes"}?`,
       default: false,
     }))
   ) {
@@ -153,30 +134,112 @@ to be backfilled again if you want to restore it later.
   logFinishedStep("Proceeding with push.");
 }
 
+// Preferred source: one schema evaluation reports every dropped index
+// together with its table's document count, requiring only the
+// `deployment:deploy` permission the push itself needs.
+export function deletedIndexesFromSchemaEvaluation(
+  response: EvaluateSchemaResponse,
+): DeletedIndex[] {
+  return Object.entries(response.componentSchemaEvaluations).flatMap(
+    ([componentPath, prediction]) =>
+      prediction.indexes
+        .filter((index) => index.change === "dropped")
+        .map((index) => ({
+          componentPath,
+          index,
+          documentsCount: index.numDocs,
+          replacedBy:
+            prediction.indexes.find(
+              (other) => other.change === "added" && other.name === index.name,
+            ) ?? null,
+        })),
+  );
+}
+
+// Fallback for deployments without `evaluate_schema`: diff indexes with
+// `evaluate_push`, then size each affected table with a system query. The
+// system query needs the `deployment:data:view` permission, which a
+// deploy-scoped key may lack.
+export async function deletedIndexesFromTableSizes({
+  ctx,
+  span,
+  request,
+  options,
+}: {
+  ctx: Context;
+  span: Span;
+  request: StartPushRequest;
+  options: CheckOptions;
+}): Promise<DeletedIndex[]> {
+  const { schemaChange } = await evaluatePush(ctx, span, request, options);
+
+  const indexDiffs = schemaChange.indexDiffs ?? {};
+  const deletedIndexes = Object.entries(indexDiffs).flatMap(
+    ([componentPath, indexDiff]) =>
+      indexDiff.removed_indexes.map((index) => ({
+        componentPath,
+        index,
+        replacedBy:
+          indexDiff.added_indexes.find((other) => other.name === index.name) ??
+          null,
+      })),
+  );
+
+  if (deletedIndexes.length === 0) {
+    return [];
+  }
+
+  const tablesWithDeletedIndexes = [
+    ...new Set(
+      deletedIndexes.map(
+        ({ componentPath, index }) => `${componentPath}:${getTableName(index)}`,
+      ),
+    ),
+  ].map((str) => {
+    const [componentPath, table] = str.split(":");
+    return { componentPath, table };
+  });
+  changeSpinner("Checking whether the deleted indexes are on large tables...");
+  const documentCounts = await Promise.all(
+    tablesWithDeletedIndexes.map(async ({ componentPath, table }) => ({
+      componentPath,
+      table,
+      count: (await runSystemQuery(ctx, {
+        deploymentUrl: options.url,
+        adminKey: options.adminKey,
+        functionName: "_system/cli/tableSize:default",
+        componentPath,
+        args: { tableName: table },
+      })) as number,
+    })),
+  );
+  return deletedIndexes.map(({ componentPath, index, replacedBy }) => ({
+    componentPath,
+    index,
+    replacedBy,
+    documentsCount: documentCounts.find(
+      (count) =>
+        count.table === getTableName(index) &&
+        count.componentPath === componentPath,
+    )!.count,
+  }));
+}
+
 function formatDeletedIndex({
-  componentDefinitionPath,
+  componentPath,
   index,
-  indexDiff,
   documentsCount,
+  replacedBy,
   minDocumentsForWarning,
-}: {
-  componentDefinitionPath: string;
-  index: DeveloperIndexConfig;
-  indexDiff: IndexDiff;
-  documentsCount: number;
-  minDocumentsForWarning: number;
-}) {
+}: DeletedIndex & { minDocumentsForWarning: number }) {
   const componentNameFormatted =
-    componentDefinitionPath !== ""
-      ? `${chalkStderr.gray(componentDefinitionPath)}:`
-      : "";
+    componentPath !== "" ? `${chalkStderr.gray(componentPath)}:` : "";
 
   const documentsCountFormatted =
     documentsCount >= minDocumentsForWarning
       ? `  ${chalkStderr.yellowBright(`⚠️  ${documentsCount.toLocaleString()} documents`)}`
       : `  ${documentsCount.toLocaleString()} ${documentsCount === 1 ? "document" : "documents"}`;
 
-  const replacedBy = indexDiff.added_indexes.find((i) => i.name === index.name);
   const replacedByFormatted = replacedBy
     ? `\n   ${chalkStderr.green("→ replaced by:")} ${formatIndex(replacedBy)}`
     : "";
@@ -196,12 +259,8 @@ function getTableName(index: DeveloperIndexConfig) {
 }
 
 function minDocumentsForIndexDeleteWarning(): number {
-  const envValue = process.env.CONVEX_MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING;
-  if (envValue !== undefined) {
-    const parsed = parseInt(envValue, 10);
-    if (!isNaN(parsed)) {
-      return parsed;
-    }
-  }
-  return MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING;
+  return integerFromEnv(
+    "CONVEX_MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING",
+    MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING,
+  );
 }

package/src/cli/lib/checkForSlowSchemaValidation.ts

@@ -0,0 +1,141 @@
+import { chalkStderr } from "chalk";
+import { logMessage } from "../../bundler/log.js";
+import { Span } from "./tracing.js";
+import {
+  EvaluateSchemaResponse,
+  TablePrediction,
+} from "./deployApi/evaluateSchema.js";
+import { SchemaEvaluation } from "./schemaEvaluation.js";
+import { formatSize, integerFromEnv } from "./utils/utils.js";
+import {
+  formatDiscardedStagedValidator,
+  formatValidationProgress,
+} from "./formatValidationProgress.js";
+
+const MIN_BYTES_FOR_SCHEMA_WALK_WARNING = 1 << 27; // 128 MiB
+
+export async function checkForSlowSchemaValidation({
+  span,
+  schemaEvaluation,
+  dryRun,
+}: {
+  span: Span;
+  schemaEvaluation: SchemaEvaluation;
+  dryRun: boolean;
+}): Promise<void> {
+  const response = await schemaEvaluation(span);
+  if (response === null) {
+    return;
+  }
+  reportStagedValidators(response);
+  if (!dryRun) {
+    return;
+  }
+
+  const walkedTables = Object.entries(
+    response.componentSchemaEvaluations,
+  ).flatMap(([componentPath, prediction]) =>
+    prediction.tables
+      .filter((table) => table.outcome === "mustWalk")
+      .map((table) => ({ componentPath, table })),
+  );
+  if (walkedTables.length === 0) {
+    return;
+  }
+
+  const totalBytes = walkedTables.reduce(
+    (sum, { table }) => sum + table.sizeBytes,
+    0,
+  );
+  if (totalBytes < minBytesForSchemaWalkWarning()) {
+    return;
+  }
+
+  const shouldSuggestStaging = walkedTables.some(
+    ({ table }) => !table.canSkipAfterStagedValidation,
+  );
+  logMessage(`⚠️  This schema change requires checking every document in the following ${walkedTables.length === 1 ? "table" : "tables"} against your new schema, totaling ${chalkStderr.yellowBright(formatSize(totalBytes))}. This deploy may take a while:
+
+${walkedTables
+  .map(({ componentPath, table }) =>
+    formatWalkedTable({ componentPath, table }),
+  )
+  .join("\n")}
+${
+  shouldSuggestStaging
+    ? `\nTip: stage the new validator (\`defineTable({...}).staged({...})\`) to validate it in the background without blocking deploys.`
+    : ""
+}`);
+}
+
+// The prediction includes staged validators from active and in-flight schemas.
+function reportStagedValidators(response: EvaluateSchemaResponse) {
+  const stagedTables = Object.entries(
+    response.componentSchemaEvaluations,
+  ).flatMap(([componentPath, prediction]) => {
+    const discarded = new Map(
+      (prediction.discardedStagedValidators ?? []).map((validator) => [
+        validator.tableName,
+        validator,
+      ]),
+    );
+    return prediction.tables
+      .filter((table) => table.staged !== undefined)
+      .map((table) => ({
+        componentPath,
+        table,
+        discarded: discarded.get(table.name),
+      }));
+  });
+  if (stagedTables.length === 0) {
+    return;
+  }
+  const lines = stagedTables.map(({ componentPath, table, discarded }) => {
+    const componentPrefix =
+      componentPath !== "" ? `${chalkStderr.gray(componentPath)}: ` : "";
+    // The state describes the validator this push discards, so advertising
+    // it as ready to promote would contradict the unstaging warning.
+    if (discarded !== undefined) {
+      return `  ${componentPrefix}${table.name}: ${formatDiscardedStagedValidator(discarded)}`;
+    }
+    const staged = table.staged!;
+    switch (staged.state) {
+      case "pending": {
+        return `  ${componentPrefix}${table.name}: validating in the background (${formatValidationProgress(staged)})`;
+      }
+      case "valid":
+        // `supersetOfStagedValidated` means this push already promotes it.
+        return table.outcome === "supersetOfStagedValidated"
+          ? `  ${componentPrefix}${table.name}: ${chalkStderr.green("validated")}. This deploy applies the staged schema without scanning the table.`
+          : `  ${componentPrefix}${table.name}: ${chalkStderr.green("validated")}. Safe to apply the staged schema changes and deploy.`;
+      case "failed":
+        return `  ${componentPrefix}${table.name}: ${chalkStderr.red("failed")}: ${staged.error}${chalkStderr.gray(" (fix the offending documents and redeploy to revalidate)")}`;
+    }
+  });
+  logMessage(`Staged schema validation status:\n${lines.join("\n")}\n`);
+}
+
+function formatWalkedTable({
+  componentPath,
+  table,
+}: {
+  componentPath: string;
+  table: TablePrediction;
+}) {
+  const componentPrefix =
+    componentPath !== "" ? `${chalkStderr.gray(componentPath)}: ` : "";
+  const docsFormatted = `${table.numDocs.toLocaleString()} documents, `;
+  const sizeFormatted = formatSize(table.sizeBytes);
+  let stagedNote = "";
+  if (table.staged?.state === "pending" && table.canSkipAfterStagedValidation) {
+    stagedNote = `. Staged schema validation in progress. Wait for validation to complete before deploying with the staged changes. Deploying now will block on a full table revalidation scan.`;
+  }
+  return `  ${componentPrefix}${table.name} (${docsFormatted}${sizeFormatted})${stagedNote}`;
+}
+
+function minBytesForSchemaWalkWarning(): number {
+  return integerFromEnv(
+    "CONVEX_MIN_BYTES_FOR_SCHEMA_WALK_WARNING",
+    MIN_BYTES_FOR_SCHEMA_WALK_WARNING,
+  );
+}

package/src/cli/lib/checkForStagedValidatorDeletion.ts

@@ -0,0 +1,104 @@
+import { chalkStderr } from "chalk";
+import { Context } from "../../bundler/context.js";
+import { logFinishedStep, logMessage, stopSpinner } from "../../bundler/log.js";
+import { promptYesNo } from "./utils/prompts.js";
+import {
+  DiscardedStagedValidator,
+  EvaluateSchemaResponse,
+} from "./deployApi/evaluateSchema.js";
+import { formatDiscardedStagedValidator } from "./formatValidationProgress.js";
+
+export type StagedValidatorDeletionCheck =
+  | "ask for confirmation"
+  | "has confirmation"
+  | "no verification";
+
+/**
+ * Confirm before discarding pending or successful staged validation. Unstaging
+ * stops write-time checks, so restaging requires a fresh scan.
+ *
+ * A dry run only reports what the push would discard, so `--dry-run` previews
+ * the confirmation the real push will ask for.
+ */
+export async function checkForStagedValidatorDeletion({
+  ctx,
+  response,
+  askForConfirmation,
+  dryRun = false,
+}: {
+  ctx: Context;
+  response: EvaluateSchemaResponse | null;
+  askForConfirmation: boolean;
+  dryRun?: boolean;
+}): Promise<void> {
+  // `sharedSchemaEvaluation` has already logged why evaluation failed.
+  if (response === null) {
+    return;
+  }
+  const discarded = Object.entries(response.componentSchemaEvaluations).flatMap(
+    ([componentPath, prediction]) =>
+      (prediction.discardedStagedValidators ?? []).map((validator) => ({
+        componentPath,
+        validator,
+      })),
+  );
+  if (discarded.length === 0) {
+    return;
+  }
+
+  logMessage(`\u26a0\ufe0f  This push ${chalkStderr.bold(dryRun ? "would discard" : "discards")} staged schema validation for the following ${discarded.length === 1 ? "table" : "tables"}:
+
+${discarded.map(formatDiscarded).join("\n")}
+
+A discarded staged validator restarts validation from zero if staged again.
+`);
+
+  if (dryRun) {
+    return;
+  }
+
+  if (!askForConfirmation) {
+    logFinishedStep(
+      "Proceeding with push since --allow-unstaging-schema is set",
+    );
+    return;
+  }
+
+  if (!process.stdin.isTTY) {
+    return ctx.crash({
+      exitCode: 1,
+      errorType: "fatal",
+      printedMessage: `To confirm the push:
+\u2022 run the deploy command in an ${chalkStderr.bold("interactive terminal")}
+\u2022 or run the deploy command with the ${chalkStderr.bold("--allow-unstaging-schema")} flag`,
+    });
+  }
+
+  stopSpinner();
+  if (
+    !(await promptYesNo(ctx, {
+      message: `Discard staged validation for ${discarded.length === 1 ? "this table" : "these tables"}?`,
+      default: false,
+    }))
+  ) {
+    return ctx.crash({
+      exitCode: 1,
+      errorType: "fatal",
+      printedMessage: `Canceling push`,
+    });
+  }
+
+  logFinishedStep("Proceeding with push.");
+}
+
+function formatDiscarded({
+  componentPath,
+  validator,
+}: {
+  componentPath: string;
+  validator: DiscardedStagedValidator;
+}): string {
+  const componentPrefix =
+    componentPath !== "" ? `${chalkStderr.gray(componentPath)}:` : "";
+  return `\u26d4 ${componentPrefix}${validator.tableName}: ${formatDiscardedStagedValidator(validator)}`;
+}

package/src/cli/lib/codegen.ts

@@ -46,7 +46,6 @@ import {
   rootComponentApiCJS,
 } from "../codegen_templates/component_api.js";
 import { functionsDir } from "./utils/utils.js";
-import { LargeIndexDeletionCheck } from "./indexes.js";
 
 const PRESERVED_GENERATED_ENTRIES = new Set(["ai"]);
 
@@ -81,7 +80,6 @@ export type CodegenOptions = {
   liveComponentSources: boolean;
   debugNodeApis: boolean;
   systemUdfs: boolean;
-  largeIndexDeletionCheck: LargeIndexDeletionCheck;
   codegenOnlyThisComponent?: string | undefined;
 };
 

package/src/cli/lib/command.ts

@@ -260,6 +260,7 @@ export async function normalizeDevOptions(
     debugNodeApis?: boolean;
     liveComponentSources?: boolean;
     pushAllModules?: boolean;
+    allowUnstagingSchema?: boolean;
     while?: string;
   },
 ): Promise<{
@@ -279,6 +280,7 @@ export async function normalizeDevOptions(
   debugNodeApis: boolean;
   liveComponentSources: boolean;
   pushAllModules: boolean;
+  allowUnstagingSchema: boolean;
 }> {
   if (cmdOptions.runComponent && !cmdOptions.run) {
     return await ctx.crash({
@@ -332,6 +334,7 @@ export async function normalizeDevOptions(
     debugNodeApis: !!cmdOptions.debugNodeApis,
     liveComponentSources: !!cmdOptions.liveComponentSources,
     pushAllModules: !!cmdOptions.pushAllModules,
+    allowUnstagingSchema: !!cmdOptions.allowUnstagingSchema,
   };
 }