convex 1.45.0 → 1.46.0

package/src/cli/lib/checkForLargeIndexDeletion.ts

@@ -4,18 +4,34 @@ import {
   changeSpinner,
   logFinishedStep,
   logMessage,
+  logVerbose,
   stopSpinner,
 } from "../../bundler/log.js";
 import { formatIndex } from "./indexes.js";
 import { promptYesNo } from "./utils/prompts.js";
 import { Span } from "./tracing.js";
 import { StartPushRequest } from "./deployApi/startPush.js";
-import { evaluatePush } from "./deploy2.js";
-import { DeveloperIndexConfig, IndexDiff } from "./deployApi/finishPush.js";
+import { evaluatePush, evaluateSchema } from "./deploy2.js";
+import { DeveloperIndexConfig } from "./deployApi/finishPush.js";
 import { runSystemQuery } from "./run.js";
 
 const MIN_DOCUMENTS_FOR_INDEX_DELETE_WARNING = 100_000;
 
+type CheckOptions = {
+  url: string;
+  deploymentName: string | null;
+  adminKey: string;
+};
+
+type DeletedIndex = {
+  // The component instance path ("" for the root component), matching the
+  // keys of both `componentSchemaEvaluations` and `indexDiffs`.
+  componentPath: string;
+  index: DeveloperIndexConfig;
+  documentsCount: number;
+  replacedBy: DeveloperIndexConfig | null;
+};
+
 export async function checkForLargeIndexDeletion({
   ctx,
   span,
@@ -26,92 +42,41 @@ export async function checkForLargeIndexDeletion({
   ctx: Context;
   span: Span;
   request: StartPushRequest;
-  options: {
-    url: string;
-    deploymentName: string | null;
-    adminKey: string;
-  };
+  options: CheckOptions;
   askForConfirmation: boolean;
 }): Promise<void> {
   changeSpinner("Verifying that the push isn’t deleting large indexes...");
 
-  const { schemaChange } = await evaluatePush(ctx, span, request, options);
-
-  const indexDiffs = schemaChange.indexDiffs ?? {};
-  const deletedIndexes = Object.entries(indexDiffs).flatMap(
-    ([componentDefinitionPath, indexDiff]) =>
-      indexDiff.removed_indexes.map((index) => ({
-        componentDefinitionPath,
-        index,
-      })),
-  );
+  const deletedIndexes =
+    (await deletedIndexesFromSchemaEvaluation({
+      ctx,
+      span,
+      request,
+      options,
+    })) ??
+    (await deletedIndexesFromTableSizes({ ctx, span, request, options }));
 
   if (deletedIndexes.length === 0) {
     logFinishedStep("No indexes are deleted by this push");
     return;
   }
 
-  const tablesWithDeletedIndexes = [
-    ...new Set(
-      deletedIndexes.map(
-        ({ componentDefinitionPath, index }) =>
-          `${componentDefinitionPath}:${getTableName(index)}`,
-      ),
-    ),
-  ].map((str) => {
-    const [componentDefinitionPath, table] = str.split(":");
-    return { componentDefinitionPath, table };
-  });
-  changeSpinner("Checking whether the deleted indexes are on large tables...");
-  const documentCounts = await Promise.all(
-    tablesWithDeletedIndexes.map(
-      async ({ componentDefinitionPath, table }) => ({
-        componentDefinitionPath,
-        table,
-        count: (await runSystemQuery(ctx, {
-          deploymentUrl: options.url,
-          adminKey: options.adminKey,
-          functionName: "_system/cli/tableSize:default",
-          componentPath: componentDefinitionPath,
-          args: { tableName: table },
-        })) as number,
-      }),
-    ),
-  );
-  const deletedIndexesWithDocumentsCount = deletedIndexes.map(
-    ({ componentDefinitionPath, index }) => ({
-      componentDefinitionPath,
-      index,
-      count: documentCounts.find(
-        (count) =>
-          count.table === getTableName(index) &&
-          count.componentDefinitionPath === componentDefinitionPath,
-      )!.count,
-    }),
-  );
-
   const minDocumentsForWarning = minDocumentsForIndexDeleteWarning();
   if (
-    !deletedIndexesWithDocumentsCount.some(
-      ({ count }) => count >= minDocumentsForWarning,
+    !deletedIndexes.some(
+      ({ documentsCount }) => documentsCount >= minDocumentsForWarning,
     )
   ) {
     logFinishedStep("No large indexes are deleted by this push");
     return;
   }
 
-  logMessage(`⚠️  This code push will ${chalkStderr.bold("delete")} the following ${deletedIndexesWithDocumentsCount.length === 1 ? "index" : "indexes"}
+  logMessage(`⚠️  This code push will ${chalkStderr.bold("delete")} the following ${deletedIndexes.length === 1 ? "index" : "indexes"}
 from your production deployment (${options.url}):
 
-${deletedIndexesWithDocumentsCount
-  .map(({ componentDefinitionPath, index, count }) =>
-    formatDeletedIndex({
-      componentDefinitionPath,
-      index,
-      indexDiff: indexDiffs[componentDefinitionPath],
-      documentsCount: count,
-      minDocumentsForWarning,
-    }),
+${deletedIndexes
+  .map((deletedIndex) =>
+    formatDeletedIndex({ ...deletedIndex, minDocumentsForWarning }),
   )
   .join("\n")}
 
@@ -139,7 +104,7 @@ to be backfilled again if you want to restore it later.
   stopSpinner();
   if (
     !(await promptYesNo(ctx, {
-      message: `Delete ${deletedIndexesWithDocumentsCount.length === 1 ? "this index" : "these indexes"}?`,
+      message: `Delete ${deletedIndexes.length === 1 ? "this index" : "these indexes"}?`,
       default: false,
     }))
   ) {
@@ -153,30 +118,137 @@ to be backfilled again if you want to restore it later.
   logFinishedStep("Proceeding with push.");
 }
 
+// Preferred source: one schema evaluation reports every dropped index
+// together with its table's document count, requiring only the
+// `deployment:deploy` permission the push itself needs. Returns null when the
+// deployment doesn't support the endpoint, in which case the caller falls
+// back to sizing tables one system query at a time.
+export async function deletedIndexesFromSchemaEvaluation({
+  ctx,
+  span,
+  request,
+  options,
+}: {
+  ctx: Context;
+  span: Span;
+  request: StartPushRequest;
+  options: CheckOptions;
+}): Promise<DeletedIndex[] | null> {
+  try {
+    const response = await evaluateSchema(
+      ctx,
+      span,
+      request,
+      options,
+      /* bestEffort */ true,
+    );
+    return Object.entries(response.componentSchemaEvaluations).flatMap(
+      ([componentPath, prediction]) =>
+        prediction.indexes
+          .filter((index) => index.change === "dropped")
+          .map((index) => ({
+            componentPath,
+            index,
+            documentsCount: index.numDocs,
+            replacedBy:
+              prediction.indexes.find(
+                (other) =>
+                  other.change === "added" && other.name === index.name,
+              ) ?? null,
+          })),
+    );
+  } catch (error: unknown) {
+    logVerbose(
+      `Schema evaluation unavailable, sizing deleted indexes per table: ${error instanceof Error ? error.message : String(error)}`,
+    );
+    return null;
+  }
+}
+
+// Fallback for deployments without `evaluate_schema`: diff indexes with
+// `evaluate_push`, then size each affected table with a system query. The
+// system query needs the `deployment:data:view` permission, which a
+// deploy-scoped key may lack.
+export async function deletedIndexesFromTableSizes({
+  ctx,
+  span,
+  request,
+  options,
+}: {
+  ctx: Context;
+  span: Span;
+  request: StartPushRequest;
+  options: CheckOptions;
+}): Promise<DeletedIndex[]> {
+  const { schemaChange } = await evaluatePush(ctx, span, request, options);
+
+  const indexDiffs = schemaChange.indexDiffs ?? {};
+  const deletedIndexes = Object.entries(indexDiffs).flatMap(
+    ([componentPath, indexDiff]) =>
+      indexDiff.removed_indexes.map((index) => ({
+        componentPath,
+        index,
+        replacedBy:
+          indexDiff.added_indexes.find((other) => other.name === index.name) ??
+          null,
+      })),
+  );
+
+  if (deletedIndexes.length === 0) {
+    return [];
+  }
+
+  const tablesWithDeletedIndexes = [
+    ...new Set(
+      deletedIndexes.map(
+        ({ componentPath, index }) => `${componentPath}:${getTableName(index)}`,
+      ),
+    ),
+  ].map((str) => {
+    const [componentPath, table] = str.split(":");
+    return { componentPath, table };
+  });
+  changeSpinner("Checking whether the deleted indexes are on large tables...");
+  const documentCounts = await Promise.all(
+    tablesWithDeletedIndexes.map(async ({ componentPath, table }) => ({
+      componentPath,
+      table,
+      count: (await runSystemQuery(ctx, {
+        deploymentUrl: options.url,
+        adminKey: options.adminKey,
+        functionName: "_system/cli/tableSize:default",
+        componentPath,
+        args: { tableName: table },
+      })) as number,
+    })),
+  );
+  return deletedIndexes.map(({ componentPath, index, replacedBy }) => ({
+    componentPath,
+    index,
+    replacedBy,
+    documentsCount: documentCounts.find(
+      (count) =>
+        count.table === getTableName(index) &&
+        count.componentPath === componentPath,
+    )!.count,
+  }));
+}
+
 function formatDeletedIndex({
-  componentDefinitionPath,
+  componentPath,
   index,
-  indexDiff,
   documentsCount,
+  replacedBy,
   minDocumentsForWarning,
-}: {
-  componentDefinitionPath: string;
-  index: DeveloperIndexConfig;
-  indexDiff: IndexDiff;
-  documentsCount: number;
-  minDocumentsForWarning: number;
-}) {
+}: DeletedIndex & { minDocumentsForWarning: number }) {
   const componentNameFormatted =
-    componentDefinitionPath !== ""
-      ? `${chalkStderr.gray(componentDefinitionPath)}:`
-      : "";
+    componentPath !== "" ? `${chalkStderr.gray(componentPath)}:` : "";
 
   const documentsCountFormatted =
     documentsCount >= minDocumentsForWarning
       ? `  ${chalkStderr.yellowBright(`⚠️  ${documentsCount.toLocaleString()} documents`)}`
       : `  ${documentsCount.toLocaleString()} ${documentsCount === 1 ? "document" : "documents"}`;
 
-  const replacedBy = indexDiff.added_indexes.find((i) => i.name === index.name);
   const replacedByFormatted = replacedBy
     ? `\n   ${chalkStderr.green("→ replaced by:")} ${formatIndex(replacedBy)}`
     : "";

package/src/cli/lib/checkForSlowSchemaValidation.ts

@@ -0,0 +1,130 @@
+import { chalkStderr } from "chalk";
+import { Context } from "../../bundler/context.js";
+import { logMessage, logVerbose } from "../../bundler/log.js";
+import { Span } from "./tracing.js";
+import { StartPushRequest } from "./deployApi/startPush.js";
+import {
+  EvaluateSchemaResponse,
+  TablePrediction,
+} from "./deployApi/evaluateSchema.js";
+import { evaluateSchema } from "./deploy2.js";
+import { formatSize } from "./utils/utils.js";
+
+const MIN_BYTES_FOR_SCHEMA_WALK_WARNING = 1 << 27; // 128 MiB
+
+export async function checkForSlowSchemaValidation({
+  ctx,
+  span,
+  request,
+  options,
+}: {
+  ctx: Context;
+  span: Span;
+  request: StartPushRequest;
+  options: {
+    url: string;
+    deploymentName: string | null;
+    adminKey: string;
+  };
+}): Promise<void> {
+  const response = await evaluateSchemaBestEffort({
+    ctx,
+    span,
+    request,
+    options,
+  });
+  if (response === null) {
+    return;
+  }
+
+  const walkedTables = Object.entries(
+    response.componentSchemaEvaluations,
+  ).flatMap(([componentPath, prediction]) =>
+    prediction.tables
+      .filter((table) => table.outcome === "mustWalk")
+      .map((table) => ({ componentPath, table })),
+  );
+  if (walkedTables.length === 0) {
+    return;
+  }
+
+  const totalBytes = walkedTables.reduce(
+    (sum, { table }) => sum + table.sizeBytes,
+    0,
+  );
+  if (totalBytes < minBytesForSchemaWalkWarning()) {
+    return;
+  }
+
+  logMessage(`⚠️  This schema change requires checking every document in the following ${walkedTables.length === 1 ? "table" : "tables"} against your new schema, totaling ${chalkStderr.yellowBright(formatSize(totalBytes))}. This deploy may take a while:
+
+${walkedTables
+  .map(({ componentPath, table }) =>
+    formatWalkedTable({ componentPath, table }),
+  )
+  .join("\n")}
+`);
+}
+
+// This warning is advisory: if evaluation fails, fall back to skipping it
+// rather than aborting the push over a check that isn't required for
+// correctness. Plausibly transient failures — network errors and 5xx
+// responses (e.g. table summaries still bootstrapping) — are retried with
+// backoff by the shared fetch wrapper (see `pushCode`); anything else is
+// deterministic, including a 2xx whose shape the CLI doesn't recognize, so
+// it skips the check immediately.
+async function evaluateSchemaBestEffort({
+  ctx,
+  span,
+  request,
+  options,
+}: {
+  ctx: Context;
+  span: Span;
+  request: StartPushRequest;
+  options: {
+    url: string;
+    deploymentName: string | null;
+    adminKey: string;
+  };
+}): Promise<EvaluateSchemaResponse | null> {
+  try {
+    return await evaluateSchema(
+      ctx,
+      span,
+      request,
+      options,
+      /* bestEffort */ true,
+    );
+  } catch (error: unknown) {
+    logVerbose(
+      `Skipping slow schema validation check: ${error instanceof Error ? error.message : String(error)}`,
+    );
+    return null;
+  }
+}
+
+function formatWalkedTable({
+  componentPath,
+  table,
+}: {
+  componentPath: string;
+  table: TablePrediction;
+}) {
+  const componentPrefix =
+    componentPath !== "" ? `${chalkStderr.gray(componentPath)}: ` : "";
+  const docsFormatted = `${table.numDocs.toLocaleString()} documents, `;
+  const sizeFormatted = formatSize(table.sizeBytes);
+  return `  ${componentPrefix}${table.name} (${docsFormatted}${sizeFormatted})`;
+}
+
+function minBytesForSchemaWalkWarning(): number {
+  const envValue = process.env.CONVEX_MIN_BYTES_FOR_SCHEMA_WALK_WARNING;
+  if (envValue !== undefined) {
+    const parsed = parseInt(envValue, 10);
+    if (!isNaN(parsed)) {
+      return parsed;
+    }
+  }
+  return MIN_BYTES_FOR_SCHEMA_WALK_WARNING;
+}

package/src/cli/lib/codegen.ts

@@ -82,6 +82,7 @@ export type CodegenOptions = {
   debugNodeApis: boolean;
   systemUdfs: boolean;
   largeIndexDeletionCheck: LargeIndexDeletionCheck;
+  warnOnSlowSchemaValidation: boolean;
   codegenOnlyThisComponent?: string | undefined;
 };
 

package/src/cli/lib/components.ts

@@ -52,6 +52,7 @@ import { DeploymentType } from "./api.js";
 import { deploymentDashboardUrlPage } from "./dashboard.js";
 import { formatIndex, LargeIndexDeletionCheck } from "./indexes.js";
 import { checkForLargeIndexDeletion } from "./checkForLargeIndexDeletion.js";
+import { checkForSlowSchemaValidation } from "./checkForSlowSchemaValidation.js";
 import { LogManager } from "./logs.js";
 import { createHash } from "crypto";
 import { Bundle, BundleHash } from "../../bundler/index.js";
@@ -75,6 +76,10 @@ export type PushOptions = {
   pushAllModules: boolean;
   logManager?: LogManager | undefined;
   largeIndexDeletionCheck: LargeIndexDeletionCheck;
+  // Whether a dry run should warn when the schema change would block the
+  // deploy on a slow table walk. On for deploys; off for commands whose dry
+  // runs never deploy (e.g. codegen).
+  warnOnSlowSchemaValidation: boolean;
   message: string | null;
 };
 
@@ -229,6 +234,7 @@ async function startComponentsPushAndCodegen(
     pushAllModules?: boolean;
     debugNodeApis: boolean;
     largeIndexDeletionCheck: LargeIndexDeletionCheck;
+    warnOnSlowSchemaValidation: boolean;
     codegenOnlyThisComponent?: string | undefined;
   },
 ): Promise<StartPushResponse | null> {
@@ -458,6 +464,17 @@ async function startComponentsPushAndCodegen(
     );
   }
 
+  if (options.dryRun && options.warnOnSlowSchemaValidation) {
+    await parentSpan.enterAsync("checkForSlowSchemaValidation", (span) =>
+      checkForSlowSchemaValidation({
+        ctx,
+        span,
+        request: startPushRequest,
+        options,
+      }),
+    );
+  }
+
   changeSpinner("Uploading functions to Convex...");
   const startPushResponse = await parentSpan.enterAsync("startPush", (span) =>
     startPush(ctx, span, startPushRequest, options),

package/src/cli/lib/deploy2.ts

@@ -22,6 +22,10 @@ import {
   startPushResponse,
   StartPushResponse,
 } from "./deployApi/startPush.js";
+import {
+  EvaluateSchemaResponse,
+  evaluateSchemaResponse,
+} from "./deployApi/evaluateSchema.js";
 import {
   AppDefinitionConfig,
   ComponentDefinitionConfig,
@@ -103,6 +107,35 @@ export async function evaluatePush(
   return evaluatePushResponse.parse(response);
 }
 
+export async function evaluateSchema(
+  ctx: Context,
+  span: Span,
+  request: StartPushRequest,
+  options: {
+    url: string;
+    deploymentName: string | null;
+    deploymentType?: DeploymentType;
+  },
+  // When set, a failed evaluation throws the raw error instead of crashing
+  // the process, so best-effort callers (e.g. advisory checks) can recover.
+  bestEffort?: boolean,
+): Promise<EvaluateSchemaResponse> {
+  const response = await pushCode(
+    ctx,
+    span,
+    request,
+    options,
+    "/api/deploy2/evaluate_schema",
+    bestEffort,
+  );
+  return evaluateSchemaResponse.parse(response);
+}
+
+// Retries for best-effort requests, where a transient failure should be
+// ridden out if possible but the command proceeds without the result
+// otherwise.
+const BEST_EFFORT_RETRIES = 5;
+
 async function pushCode(
   ctx: Context,
   span: Span,
@@ -112,7 +145,11 @@ async function pushCode(
     deploymentName: string | null;
     deploymentType?: DeploymentType;
   },
-  endpoint: "/api/deploy2/start_push" | "/api/deploy2/evaluate_push",
+  endpoint:
+    | "/api/deploy2/start_push"
+    | "/api/deploy2/evaluate_push"
+    | "/api/deploy2/evaluate_schema",
+  bestEffort?: boolean,
 ): Promise<unknown> {
   // Log a summary of the push request instead of the full object
   const unchangedModuleCount =
@@ -136,6 +173,18 @@ async function pushCode(
     deploymentUrl: options.url,
     adminKey: request.adminKey,
     onError,
+    // Best-effort callers tolerate failure, so retry only plausibly
+    // transient conditions — network errors and 5xx responses (e.g. table
+    // summaries still bootstrapping) — and fail fast on everything else,
+    // like a 404 from a deployment whose backend doesn't yet expose the
+    // endpoint.
+    ...(bestEffort
+      ? {
+          maxRetries: BEST_EFFORT_RETRIES,
+          shouldRetry: (error: Error | null, response: Response | null) =>
+            error !== null || (response !== null && response.status >= 500),
+        }
+      : {}),
   });
   try {
     const response = await fetch(endpoint, {
@@ -149,6 +198,11 @@ async function pushCode(
     });
     return await response.json();
   } catch (error: unknown) {
+    if (bestEffort) {
+      // The caller handles this itself instead of crashing the process.
+      // eslint-disable-next-line no-restricted-syntax
+      throw error;
+    }
     return await handlePushConfigError(
       ctx,
       error,
@@ -453,6 +507,7 @@ export async function deployToDeployment(
     largeIndexDeletionCheck: options.allowDeletingLargeIndexes
       ? "has confirmation"
       : "ask for confirmation",
+    warnOnSlowSchemaValidation: true,
     message: options.message,
   };
   showSpinner(`Deploying to ${url}...${options.dryRun ? " [dry run]" : ""}`);

package/src/cli/lib/deployApi/evaluateSchema.ts

@@ -0,0 +1,62 @@
+import { z } from "zod";
+import { componentDefinitionPath, componentPath } from "./paths.js";
+import { developerIndexConfig } from "./finishPush.js";
+import { looseObject } from "./utils.js";
+
+// Hand-mirrors `TableValidationOutcome` in
+// `crates/common/src/schemas/mod.rs` (`#[serde(rename_all = "camelCase")]`),
+// like the rest of this deployApi directory: there's no codegen sharing
+// types between the Rust backend and this package.
+export const tableValidationOutcome = z.enum([
+  "notValidated",
+  "supersetOfEnforced",
+  "supersetOfShape",
+  "mustWalk",
+]);
+export type TableValidationOutcome = z.infer<typeof tableValidationOutcome>;
+
+export const indexChangePrediction = z.enum([
+  "added",
+  "identical",
+  "enabled",
+  "disabled",
+  "dropped",
+]);
+export type IndexChangePrediction = z.infer<typeof indexChangePrediction>;
+
+export const tablePrediction = looseObject({
+  name: z.string(),
+  outcome: tableValidationOutcome,
+  numDocs: z.number(),
+  sizeBytes: z.number(),
+});
+export type TablePrediction = z.infer<typeof tablePrediction>;
+
+export const indexPrediction = z.intersection(
+  developerIndexConfig,
+  looseObject({
+    change: indexChangePrediction,
+    needsBackfill: z.boolean(),
+    numDocs: z.number(),
+  }),
+);
+export type IndexPrediction = z.infer<typeof indexPrediction>;
+
+export const componentSchemaPrediction = looseObject({
+  definitionPath: z.string(),
+  schemaValidation: z.boolean(),
+  tables: z.array(tablePrediction),
+  indexes: z.array(indexPrediction),
+});
+export type ComponentSchemaPrediction = z.infer<
+  typeof componentSchemaPrediction
+>;
+
+export const evaluateSchemaResponse = looseObject({
+  componentSchemaEvaluations: z.record(
+    componentPath,
+    componentSchemaPrediction,
+  ),
+  newComponentDefinitions: z.array(componentDefinitionPath),
+});
+export type EvaluateSchemaResponse = z.infer<typeof evaluateSchemaResponse>;

package/src/cli/lib/deployApi/finishPush.ts

@@ -45,7 +45,7 @@ export const cronDiff = looseObject({
 });
 export type CronDiff = z.infer<typeof cronDiff>;
 
-const developerIndexConfig = z.intersection(
+export const developerIndexConfig = z.intersection(
   z.discriminatedUnion("type", [
     looseObject({
       name: z.string(),

package/src/cli/lib/deployApi/startPush.ts

@@ -29,7 +29,7 @@ export type StartPushRequest = z.infer<typeof startPushRequest>;
 export const schemaChange = looseObject({
   allocatedComponentIds: z.any(),
   schemaIds: z.any(),
-  indexDiffs: z.record(componentDefinitionPath, indexDiff).optional(),
+  indexDiffs: z.record(componentPath, indexDiff).optional(),
 });
 export type SchemaChange = z.infer<typeof schemaChange>;
 

package/src/cli/lib/dev.ts

@@ -148,6 +148,7 @@ export async function devAgainstDeployment(
         pushAllModules: devOptions.pushAllModules,
         logManager, // Pass logManager to control logs during deploy
         largeIndexDeletionCheck: "no verification", // `convex dev` can’t push to prod
+        warnOnSlowSchemaValidation: true,
         message: null,
       },
       devOptions,

package/src/cli/lib/generatedFunctionLogsApi.ts

@@ -103,6 +103,8 @@ export interface components {
             /** Format: int64 */
             databaseWriteDocuments: number;
             /** Format: int64 */
+            databaseWriteIndexRows: number;
+            /** Format: int64 */
             storageReadBytes: number;
             /** Format: int64 */
             storageWriteBytes: number;