convex 1.46.0 → 1.48.0-alpha.0

package/src/cli/configure.ts

@@ -1,4 +1,5 @@
 import { chalkStderr } from "chalk";
+import { handleNoAccessToConfiguredProject } from "./account.js";
 import { Context } from "../bundler/context.js";
 import {
   logFailure,
@@ -122,17 +123,22 @@ export async function deploymentCredentialsOrConfigure(
     chosenConfiguration,
     cmdOptions,
   );
-  const { convexSiteUrl: siteUrl } = await fetchDeploymentCanonicalUrls(ctx, {
-    adminKey: selectedDeployment.adminKey,
-    deploymentUrl: selectedDeployment.url,
-  });
+  const { convexCloudUrl, convexSiteUrl: siteUrl } =
+    await fetchDeploymentCanonicalUrls(ctx, {
+      adminKey: selectedDeployment.adminKey,
+      deploymentUrl: selectedDeployment.url,
+    });
+  const applicationUrl = deploymentUrlForApplicationEnv(
+    selectedDeployment,
+    convexCloudUrl,
+  );
 
   if (selectedDeployment.deploymentFields !== null) {
     // Set the `CONVEX_DEPLOYMENT` env var + the `CONVEX_URL` env var
     await updateEnvAndConfigForDeploymentSelection(
       ctx,
       {
-        url: selectedDeployment.url,
+        url: applicationUrl,
         siteUrl,
         deploymentName: selectedDeployment.deploymentFields.deploymentName,
         teamSlug: selectedDeployment.deploymentFields.teamSlug,
@@ -160,6 +166,21 @@ export async function deploymentCredentialsOrConfigure(
   };
 }
 
+export function deploymentUrlForApplicationEnv(
+  selectedDeployment: {
+    url: string;
+    deploymentFields: { deploymentType: DeploymentType } | null;
+  },
+  canonicalCloudUrl: string,
+): string {
+  const deploymentType = selectedDeployment.deploymentFields?.deploymentType;
+  return deploymentType !== undefined &&
+    deploymentType !== "local" &&
+    deploymentType !== "anonymous"
+    ? canonicalCloudUrl
+    : selectedDeployment.url;
+}
+
 export async function _deploymentCredentialsOrConfigure(
   ctx: Context,
   deploymentSelection: DeploymentSelection,
@@ -330,11 +351,26 @@ async function handleDeploymentWithinProject(
     return result;
   }
 
-  const accessResult = await checkAccessToSelectedProject(
+  let accessResult = await checkAccessToSelectedProject(
     ctx,
     deploymentSelection.targetProject,
   );
-  if (accessResult.kind === "noAccess") {
+  while (accessResult.kind === "noAccess") {
+    const next =
+      cmdOptions.team !== undefined && cmdOptions.project !== undefined
+        ? "chooseProject"
+        : await handleNoAccessToConfiguredProject(
+            ctx,
+            deploymentSelection.targetProject,
+            cmdOptions.envFile,
+          );
+    if (next === "retry") {
+      accessResult = await checkAccessToSelectedProject(
+        ctx,
+        deploymentSelection.targetProject,
+      );
+      continue;
+    }
     logMessage("You don't have access to the selected project.");
     const result = await handleChooseProject(
       ctx,
@@ -608,14 +644,14 @@ async function selectExistingProject(
   devDeployment: "cloud" | "local";
 }> {
   const {
-    team: { slug: teamSlug },
+    team: { id: teamId, slug: teamSlug },
     chosen,
   } = await validateOrSelectTeam(ctx, config.team, "Team:");
 
   const projectSlug = await validateOrSelectProject(
     ctx,
     config.project,
-    teamSlug,
+    teamId,
     "Configure project",
     "Project:",
   );

package/src/cli/deploy.ts

@@ -111,10 +111,23 @@ Same format as .env.local or .env files, and overrides them.`,
     ).hideHelp(),
   )
   .addOption(
-    new Option("--allow-deleting-large-indexes")
-      .hideHelp()
-      .conflicts("preview-create")
-      .conflicts("preview-name"),
+    new Option(
+      "--skip-large-indexes-check",
+      "Skip the confirmation prompt when this push creates, changes, or deletes an index on a large table. Creating or changing an index blocks the deploy until it is backfilled; consider staging it instead so it backfills in the background.",
+    ),
+  )
+  .addOption(
+    // Predecessor of --skip-large-indexes-check, kept hidden so existing
+    // deploy scripts keep working. It still skips only the deletion
+    // confirmation: scripts that opted into that never agreed to let a
+    // blocking backfill through.
+    new Option("--allow-deleting-large-indexes").hideHelp(),
+  )
+  .addOption(
+    new Option(
+      "--allow-unstaging-schema",
+      "Skip the confirmation prompt when this push discards a staged table validator that is validating or already validated.",
+    ),
   )
   .showHelpAfterError()
   .action(async (cmdOptions) => {
@@ -181,6 +194,7 @@ Same format as .env.local or .env files, and overrides them.`,
           ...cmdOptions,
           previewName: previewName ?? undefined,
           reuse,
+          skipLargeIndexesCheck: cmdOptions.skipLargeIndexesCheck ?? false,
           message: cmdOptions.message ?? getDefaultDeployMessage(),
         },
       );
@@ -204,8 +218,10 @@ Same format as .env.local or .env files, and overrides them.`,
       await deployToExistingDeployment(ctx, deploymentSelection, {
         ...cmdOptions,
         skipWorkosCheck: cmdOptions.skipWorkosCheck ?? false,
+        skipLargeIndexesCheck: cmdOptions.skipLargeIndexesCheck ?? false,
         allowDeletingLargeIndexes:
           cmdOptions.allowDeletingLargeIndexes ?? false,
+        allowUnstagingSchema: cmdOptions.allowUnstagingSchema ?? false,
         message: cmdOptions.message ?? getDefaultDeployMessage(),
       });
     }
@@ -237,6 +253,7 @@ async function deployToNewPreviewDeployment(
     debug?: boolean | undefined;
     debugBundlePath?: string | undefined;
     skipWorkosCheck?: boolean | undefined;
+    skipLargeIndexesCheck: boolean;
     message: string | null;
   },
 ) {
@@ -319,7 +336,8 @@ async function deployToNewPreviewDeployment(
   });
 
   const pushOptions: PushOptions = {
-    deploymentName: null,
+    deploymentName: data.deploymentName,
+    deploymentType: "preview",
     adminKey: previewAdminKey,
     verbose: !!options.verbose,
     dryRun: false,
@@ -333,6 +351,17 @@ async function deployToNewPreviewDeployment(
     liveComponentSources: false,
     pushAllModules: !!options.pushAllModules,
     largeIndexDeletionCheck: "no verification", // fine for preview deployments
+    // A freshly claimed preview has no data to backfill. A reused one keeps
+    // the data earlier pushes wrote, so an unstaged index there can block
+    // the deploy like on any other deployment.
+    largeIndexBackfillCheck: data.isNewDeployment
+      ? "no verification"
+      : options.skipLargeIndexesCheck
+        ? "has confirmation"
+        : "ask for confirmation",
+    // Staged validation on throwaway preview data is cheap to redo, so
+    // discarding it needs no confirmation.
+    stagedValidatorDeletionCheck: "no verification",
     warnOnSlowSchemaValidation: true,
     message: options.message,
   };
@@ -378,7 +407,9 @@ async function deployToExistingDeployment(
     liveComponentSources?: boolean | undefined;
     envFile?: string | undefined;
     skipWorkosCheck?: boolean | undefined;
+    skipLargeIndexesCheck: boolean;
     allowDeletingLargeIndexes: boolean;
+    allowUnstagingSchema: boolean;
     message: string | null;
   },
 ) {

package/src/cli/deploymentSelect.ts

@@ -12,7 +12,10 @@ import {
   parseDeploymentSelector,
   ParsedDeploymentSelector,
 } from "./lib/deploymentSelector.js";
-import { updateEnvAndConfigForDeploymentSelection } from "./configure.js";
+import {
+  deploymentUrlForApplicationEnv,
+  updateEnvAndConfigForDeploymentSelection,
+} from "./configure.js";
 import { fetchDeploymentCanonicalUrls } from "./lib/deploy2.js";
 import {
   loadProjectLocalConfig,
@@ -267,9 +270,9 @@ export async function saveSelectedDeployment(
     });
   }
 
-  const { convexSiteUrl: siteUrl } =
+  const { convexCloudUrl, convexSiteUrl: siteUrl } =
     deployment.deploymentFields.deploymentType === "local"
-      ? { convexSiteUrl: null }
+      ? { convexCloudUrl: deployment.url, convexSiteUrl: null }
       : await fetchDeploymentCanonicalUrls(ctx, {
           adminKey: deployment.adminKey,
           deploymentUrl: deployment.url,
@@ -278,7 +281,7 @@ export async function saveSelectedDeployment(
   await updateEnvAndConfigForDeploymentSelection(
     ctx,
     {
-      url: deployment.url,
+      url: deploymentUrlForApplicationEnv(deployment, convexCloudUrl),
       siteUrl,
       deploymentName: deployment.deploymentFields.deploymentName,
       teamSlug: deployment.deploymentFields.teamSlug,

package/src/cli/dev.ts

@@ -143,6 +143,15 @@ export const dev = new Command("dev")
       .default(false)
       .hideHelp(),
   )
+  .addOption(
+    // Only `--prod` pushes check this; hidden alongside it.
+    new Option(
+      "--allow-unstaging-schema",
+      "Skip the confirmation prompt when a push to production discards a staged table validator that is validating or already validated.",
+    )
+      .default(false)
+      .hideHelp(),
+  )
   .addOption(
     new Option(
       "--env-file <envFile>",

package/src/cli/env.ts

@@ -1,4 +1,4 @@
-import { Command } from "@commander-js/extra-typings";
+import { Command, Option } from "@commander-js/extra-typings";
 import { chalkStderr } from "chalk";
 import { Context, oneoffContext } from "../bundler/context.js";
 import {
@@ -43,7 +43,7 @@ const envSetCmd = new Command("set")
       "• `pbpaste | npx convex env set API_KEY` (macOS)",
       "• `Get-Clipboard | npx convex env set API_KEY` (Windows PowerShell)",
       "",
-      "To update many variables at once, save them with `npx convex env list > .env.convex`, edit the file, then reapply the changes with `npx convex env set --force < .env.convex`.",
+      "To update many variables at once, save them with `npx convex env list --include-values > .env.convex`, edit the file, then reapply the changes with `npx convex env set --force < .env.convex`.",
     ].join("\n"),
   )
   .option(
@@ -167,16 +167,22 @@ const envListCmd = new Command("list")
   .summary("List all environment variables and their values")
   .description(
     [
-      "• List all variables and their values: `npx convex env list`",
       "• List only variable names (no values): `npx convex env list --names-only`",
-      "• Save all variables to a file: `npx convex env list > .env.convex`",
-      "• Append to a file: `npx convex env list >> .env.convex`",
+      "• List all variables and their values: `npx convex env list --include-values`",
+      "• Save all variables to a file: `npx convex env list --include-values > .env.convex`",
+      "• Append to a file: `npx convex env list --include-values >> .env.convex`",
     ].join("\n"),
   )
   .option(
     "--names-only",
     "List only the names of environment variables, without their values",
   )
+  .addOption(
+    new Option(
+      "--include-values",
+      "Print each variable's value alongside its name, in .env format (the default). Pass it explicitly in scripts that rely on the values.",
+    ).conflicts("namesOnly"),
+  )
   .configureHelp({ showGlobalOptions: true })
   .allowExcessArguments(false)
   .action(async (cmdOptions, cmd) => {
@@ -207,8 +213,8 @@ export const env = new Command("env")
       "• Set interactively: `npx convex env set NAME`",
       "• Set multiple from file: `npx convex env set --from-file .env`",
       "• Unset a variable: `npx convex env remove NAME`",
-      "• List all variables and their values: `npx convex env list`",
       "• List only variable names (no values): `npx convex env list --names-only`",
+      "• List all variables and their values: `npx convex env list --include-values`",
       "• Print a variable's value: `npx convex env get NAME`",
       "",
       "By default, this sets and views variables on your dev deployment.",

package/src/cli/envDefault.ts

@@ -144,8 +144,8 @@ const envDefaultList = addEnvDefaultOptions(
     .summary("List all default environment variables and their values")
     .description(
       [
-        "• List all default variables and their values: `npx convex env default list`",
         "• List only default variable names (no values): `npx convex env default list --names-only`",
+        "• List all default variables and their values: `npx convex env default list --include-values`",
         "",
         "The deployment type is determined by the current deployment (local maps to dev), or by --type if provided.",
       ].join("\n"),
@@ -154,6 +154,12 @@ const envDefaultList = addEnvDefaultOptions(
       "--names-only",
       "List only the names of environment variables, without their values",
     )
+    .addOption(
+      new Option(
+        "--include-values",
+        "Print each variable's value alongside its name, in .env format (the default). Pass it explicitly in scripts that rely on the values.",
+      ).conflicts("namesOnly"),
+    )
     .configureHelp({ showGlobalOptions: true })
     .allowExcessArguments(false),
 ).action(async (cmdOptions, cmd) => {
@@ -176,8 +182,8 @@ export const envDefault = new Command("default")
       "",
       "• Set a default variable: `npx convex env default set NAME 'value'`",
       "• Unset a default variable: `npx convex env default remove NAME`",
-      "• List all default variables and their values: `npx convex env default list`",
       "• List only default variable names (no values): `npx convex env default list --names-only`",
+      "• List all default variables and their values: `npx convex env default list --include-values`",
       "• Print a default variable's value: `npx convex env default get NAME`",
     ].join("\n"),
   )

package/src/cli/generatedApi.ts

@@ -461,7 +461,7 @@ export interface components {
         };
         ReferralCode: string;
         /** @enum {string} */
-        RegionName: "aws-us-east-1" | "aws-eu-west-1";
+        RegionName: "aws-us-east-1" | "aws-eu-west-1" | "aws-ca-central-1" | "aws-ap-southeast-2";
         /** Format: int64 */
         TeamId: number;
         TeamName: string;

package/src/cli/lib/api.ts

@@ -26,6 +26,7 @@ import {
 import { loadProjectLocalConfig } from "./localDeployment/filePaths.js";
 import { chalkStderr } from "chalk";
 import { RegionName } from "../generatedApi.js";
+import type { RegionName as PublishedRegionName } from "@convex-dev/platform/managementApi";
 export type DeploymentName = string;
 export type CloudDeploymentType = "prod" | "dev" | "preview" | "custom";
 export type AccountRequiredDeploymentType = CloudDeploymentType | "local";
@@ -66,7 +67,10 @@ export async function createProject(
       body: {
         projectName,
         deploymentType: deploymentToProvision?.deploymentType ?? null,
-        deploymentRegion: deploymentToProvision?.region ?? null,
+        // The published platform types lag the CLI's own spec by a release,
+        // so the cast is deliberate: a new region needs no change here.
+        deploymentRegion: (deploymentToProvision?.region ??
+          null) as PublishedRegionName | null,
       },
     },
   );

package/src/cli/lib/checkForLargeIndexBackfill.ts

@@ -0,0 +1,173 @@
+import { chalkStderr } from "chalk";
+import { Context } from "../../bundler/context.js";
+import {
+  changeSpinner,
+  logFinishedStep,
+  logMessage,
+  stopSpinner,
+} from "../../bundler/log.js";
+import { formatIndex } from "./indexes.js";
+import { promptYesNo } from "./utils/prompts.js";
+import { Span } from "./tracing.js";
+import {
+  EvaluateSchemaResponse,
+  IndexPrediction,
+} from "./deployApi/evaluateSchema.js";
+import { SchemaEvaluation } from "./schemaEvaluation.js";
+import { integerFromEnv } from "./utils/utils.js";
+
+// Same size as the large index deletion check, so both index guards fire at
+// the same table size. Separate env override so tests can trigger one
+// without the other.
+const MIN_DOCUMENTS_FOR_INDEX_BACKFILL_WARNING = 100_000;
+
+export type LargeIndexBackfillCheck =
+  // Don’t check whether the push creates indexes on large tables
+  | "no verification"
+  // If it does, ask for confirmation (and fail in non-interactive envs)
+  | "ask for confirmation"
+  // If it does, proceed (the user has used --skip-large-indexes-check)
+  | "has confirmation";
+
+type BlockingIndex = {
+  // The component instance path ("" for the root component), matching the
+  // keys of `componentSchemaEvaluations`.
+  componentPath: string;
+  index: IndexPrediction;
+};
+
+/**
+ * Warn — and on a real deploy require confirmation — when a push creates or
+ * enables a non-staged index on a large table. `wait_for_schema` blocks the
+ * deploy until every such index is backfilled, which can take a long time on
+ * large tables; staging the index instead backfills it in the background.
+ *
+ * Skipped silently when the deployment doesn't support `evaluate_schema`:
+ * the check protects the developer's time, not correctness.
+ */
+export async function checkForLargeIndexBackfill({
+  ctx,
+  span,
+  schemaEvaluation,
+  options,
+  mode,
+}: {
+  ctx: Context;
+  span: Span;
+  schemaEvaluation: SchemaEvaluation;
+  options: { url: string };
+  // "warn" prints the warning without prompting, for dry runs.
+  mode: "warn" | "ask for confirmation" | "has confirmation";
+}): Promise<void> {
+  changeSpinner("Verifying that the push isn’t creating large indexes...");
+  const response = await schemaEvaluation(span);
+  if (response === null) {
+    return;
+  }
+
+  const blockingIndexes = blockingIndexesFromSchemaEvaluation(
+    response,
+    minDocumentsForIndexBackfillWarning(),
+  );
+  if (blockingIndexes.length === 0) {
+    return;
+  }
+
+  const plural = blockingIndexes.length !== 1;
+  logMessage(`⚠️  This push will ${chalkStderr.bold("create")} the following ${plural ? "indexes on large tables" : "index on a large table"}
+in your deployment (${options.url}). The deploy will block until ${plural ? "they finish" : "it finishes"} backfilling:
+
+${blockingIndexes.map(formatBlockingIndex).join("\n")}
+
+Tip: stage the index (e.g. \`.index("by_field", { fields: ["field"], staged: true })\`) to backfill
+it in the background without blocking the deploy, then remove \`staged\` in a later push once it’s ready.
+Learn more at https://docs.convex.dev/database/reading-data/indexes#staged-indexes
+`);
+
+  if (mode === "warn") {
+    return;
+  }
+
+  if (mode === "has confirmation") {
+    logFinishedStep(
+      "Proceeding with push since --skip-large-indexes-check is set",
+    );
+    return;
+  }
+
+  if (!process.stdin.isTTY) {
+    return ctx.crash({
+      exitCode: 1,
+      errorType: "fatal",
+      printedMessage: `To confirm the push:
+• run the deploy command in an ${chalkStderr.bold("interactive terminal")}
+• or run the deploy command with the ${chalkStderr.bold("--skip-large-indexes-check")} flag`,
+    });
+  }
+
+  stopSpinner();
+  if (
+    !(await promptYesNo(ctx, {
+      message: `Create ${plural ? "these indexes" : "this index"} now?`,
+      default: false,
+    }))
+  ) {
+    return ctx.crash({
+      exitCode: 1,
+      errorType: "fatal",
+      printedMessage: `Canceling push`,
+    });
+  }
+
+  logFinishedStep("Proceeding with push.");
+}
+
+// An index blocks the deploy when this push starts (`added`) or waits on
+// (`enabled` before its staged backfill finished) a backfill that isn't
+// staged. Added staged indexes report `needsBackfill` too but never block,
+// and an in-progress backfill an earlier push started (`identical`) isn't
+// this push's doing.
+export function blockingIndexesFromSchemaEvaluation(
+  response: EvaluateSchemaResponse,
+  minDocuments: number,
+): BlockingIndex[] {
+  return Object.entries(response.componentSchemaEvaluations).flatMap(
+    ([componentPath, prediction]) =>
+      prediction.indexes
+        .filter(
+          (index) =>
+            index.needsBackfill &&
+            !index.staged &&
+            (index.change === "added" || index.change === "enabled") &&
+            index.numDocs >= minDocuments,
+        )
+        .map((index) => ({ componentPath, index })),
+  );
+}
+
+function formatBlockingIndex({ componentPath, index }: BlockingIndex) {
+  const componentNameFormatted =
+    componentPath !== "" ? `${chalkStderr.gray(componentPath)}:` : "";
+  const documentsCountFormatted = chalkStderr.yellowBright(
+    `⚠️  ${index.numDocs.toLocaleString()} documents`,
+  );
+  const enabledNote =
+    index.change === "enabled"
+      ? `\n   ${chalkStderr.gray("enabled before its staged backfill finished")}`
+      : "";
+  return (
+    "⛔ " +
+    componentNameFormatted +
+    formatIndex(index) +
+    "  " +
+    documentsCountFormatted +
+    enabledNote
+  );
+}
+
+function minDocumentsForIndexBackfillWarning(): number {
+  return integerFromEnv(
+    "CONVEX_MIN_DOCUMENTS_FOR_INDEX_BACKFILL_WARNING",
+    MIN_DOCUMENTS_FOR_INDEX_BACKFILL_WARNING,
+  );
+}