convex 1.46.0 → 1.48.0-alpha.0

package/src/cli/lib/profiles.ts

@@ -0,0 +1,398 @@
+import path from "path";
+import { isDeepStrictEqual } from "util";
+import { randomUUID } from "crypto";
+import { TempPath } from "../../bundler/fs.js";
+import * as dotenv from "dotenv";
+import { z } from "zod";
+import { BigBrainAuth, Context } from "../../bundler/context.js";
+import { logFinishedStep, logWarning } from "../../bundler/log.js";
+import { globalConfigPath, readGlobalConfig } from "./utils/globalConfig.js";
+import { ENV_VAR_FILE_PATH, rootDirectory } from "./utils/utils.js";
+
+import { CONVEX_ACCOUNT_ENV_VAR } from "./envvars.js";
+
+export type ProfileSelection = {
+  name: string;
+  source: string;
+};
+
+const settingsSchema = z
+  .object({
+    defaultProfile: z.string().optional(),
+    directories: z.record(z.string()).optional(),
+    revokeReplacedTokens: z.boolean().optional(),
+  })
+  .passthrough();
+
+const credentialsSchema = z
+  .object({
+    accessToken: z.string().min(1),
+    memberId: z.string().optional(),
+    email: z.string().optional(),
+  })
+  .passthrough();
+
+export type ProfileCredentials = z.infer<typeof credentialsSchema>;
+
+export async function readProfileSettings(
+  ctx: Context,
+): Promise<z.infer<typeof settingsSchema>> {
+  if (!ctx.fs.exists(globalConfigPath())) return {};
+  try {
+    return settingsSchema.parse(
+      JSON.parse(ctx.fs.readUtf8File(globalConfigPath())),
+    );
+  } catch {
+    return await ctx.crash({
+      exitCode: 1,
+      errorType: "invalid filesystem data",
+      printedMessage: `Unable to read account settings from ${globalConfigPath()}. Check defaultProfile, directories, and revokeReplacedTokens.`,
+    });
+  }
+}
+
+function writeProfileSettings(ctx: Context, settings: Record<string, unknown>) {
+  if (Object.keys(settings).length === 0) {
+    if (ctx.fs.exists(globalConfigPath())) ctx.fs.unlink(globalConfigPath());
+    return;
+  }
+  const temporary = `${globalConfigPath()}.${randomUUID()}.tmp` as TempPath;
+  try {
+    ctx.fs.writeUtf8File(temporary, JSON.stringify(settings, null, 2), 0o600);
+    ctx.fs.swapTmpFile(temporary, globalConfigPath());
+  } finally {
+    if (ctx.fs.exists(temporary)) ctx.fs.unlink(temporary);
+  }
+}
+
+export async function updateProfileSettings<T>(
+  ctx: Context,
+  update: (settings: z.infer<typeof settingsSchema>) => T | Promise<T>,
+  afterWrite?: () => void,
+): Promise<T> {
+  const lockPath = `${globalConfigPath()}.lock`;
+  ctx.fs.mkdir(rootDirectory(), { allowExisting: true });
+  const deadline = Date.now() + 5000;
+  for (;;) {
+    try {
+      ctx.fs.mkdir(lockPath);
+      break;
+    } catch (err: any) {
+      // Only the process that creates the directory owns the update lock.
+      // eslint-disable-next-line no-restricted-syntax
+      if (err.code !== "EEXIST") throw err;
+      if (Date.now() >= deadline) {
+        return await ctx.crash({
+          exitCode: 1,
+          errorType: "fatal",
+          printedMessage: `Another command is updating Convex accounts. Retry after it finishes. If it was interrupted, remove ${lockPath}.`,
+        });
+      }
+      await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+  }
+  let released = false;
+  const release = () => {
+    if (released) return;
+    ctx.fs.rmdir(lockPath);
+    released = true;
+  };
+  const cleanup = ctx.registerCleanup(async () => release());
+  try {
+    const settings = await readProfileSettings(ctx);
+    const result = await update(settings);
+    writeProfileSettings(ctx, settings);
+    // Credential deletion follows persistence while sharing the lock with renewals.
+    afterWrite?.();
+    return result;
+  } finally {
+    ctx.removeCleanup(cleanup);
+    release();
+  }
+}
+
+export function profileConfigPath(name: string): string {
+  return path.join(
+    rootDirectory(),
+    "profiles",
+    `${encodeURIComponent(name)}.json`,
+  );
+}
+
+export async function validateProfileName(
+  ctx: Context,
+  name: string,
+): Promise<void> {
+  if (!/^[a-z0-9][a-z0-9_-]*$/.test(name)) {
+    await ctx.crash({
+      exitCode: 1,
+      errorType: "invalid filesystem or env vars",
+      printedMessage:
+        "Account names must start with a lowercase letter or number and contain only lowercase letters, numbers, hyphens, and underscores.",
+    });
+  }
+}
+
+export async function getProfileSelection(
+  ctx: Context,
+  options: { name?: string | undefined; envFile?: string | undefined } = {},
+): Promise<ProfileSelection | null> {
+  let name = options.name;
+  let source = "--account";
+  if (name === undefined && process.env[CONVEX_ACCOUNT_ENV_VAR] !== undefined) {
+    name = process.env[CONVEX_ACCOUNT_ENV_VAR];
+    source = `${CONVEX_ACCOUNT_ENV_VAR} environment variable`;
+  }
+  if (name === undefined) {
+    const files = options.envFile
+      ? [options.envFile]
+      : [ENV_VAR_FILE_PATH, ".env"];
+    for (const file of files) {
+      if (!ctx.fs.exists(file)) continue;
+      name = dotenv.parse(ctx.fs.readUtf8File(file))[CONVEX_ACCOUNT_ENV_VAR];
+      if (name !== undefined) {
+        source = `${CONVEX_ACCOUNT_ENV_VAR} in ${file}`;
+        break;
+      }
+    }
+  }
+  if (name === undefined) {
+    const config = await readProfileSettings(ctx);
+    const binding = findProfileDirectory(
+      config.directories ?? {},
+      process.cwd(),
+    );
+    name = binding?.name ?? config.defaultProfile;
+    source = binding
+      ? `directory selection for ${binding.directory}`
+      : `default account in ${globalConfigPath()}`;
+  }
+  if (name === undefined) return null;
+  await validateProfileName(ctx, name);
+  return { name, source };
+}
+
+export async function readProfileAuth(
+  ctx: Context,
+  profile: ProfileSelection,
+  allowMissing = false,
+): Promise<Extract<BigBrainAuth, { kind: "accessToken" }> | null> {
+  const credentials = await readProfileCredentials(ctx, profile, allowMissing);
+  return credentials === null
+    ? null
+    : {
+        kind: "accessToken",
+        accessToken: credentials.accessToken,
+        header: `Bearer ${credentials.accessToken}`,
+        profile,
+      };
+}
+
+export async function readProfileCredentials(
+  ctx: Context,
+  profile: ProfileSelection,
+  allowMissing = false,
+): Promise<ProfileCredentials | null> {
+  await validateProfileName(ctx, profile.name);
+  const configPath = profileConfigPath(profile.name);
+  if (!ctx.fs.exists(configPath)) {
+    if (allowMissing) return null;
+    return await ctx.crash({
+      exitCode: 1,
+      errorType: "invalid filesystem data",
+      printedMessage: `Account "${profile.name}" selected by ${profile.source} is not logged in. Run \`npx convex login --account ${profile.name}\`.`,
+    });
+  }
+  try {
+    return credentialsSchema.parse(JSON.parse(ctx.fs.readUtf8File(configPath)));
+  } catch {
+    return await ctx.crash({
+      exitCode: 1,
+      errorType: "invalid filesystem data",
+      printedMessage: `Unable to read credentials for account "${profile.name}" from ${configPath}.`,
+    });
+  }
+}
+
+export function readProfileToken(ctx: Context, name: string): string | null {
+  try {
+    const token = JSON.parse(
+      ctx.fs.readUtf8File(profileConfigPath(name)),
+    ).accessToken;
+    return typeof token === "string" && token.length > 0 ? token : null;
+  } catch {
+    return null;
+  }
+}
+
+async function writeCredentialsFile(
+  ctx: Context,
+  name: string,
+  credentials: ProfileCredentials,
+) {
+  const configPath = profileConfigPath(name);
+  const directory = path.dirname(configPath);
+  ctx.fs.mkdir(directory, { allowExisting: true });
+  for (const dir of [rootDirectory(), directory]) {
+    if (!ctx.fs.lstat(dir).isDirectory()) {
+      return await ctx.crash({
+        exitCode: 1,
+        errorType: "invalid filesystem data",
+        printedMessage: `Cannot save account credentials through a symlink or non-directory: ${dir}.`,
+      });
+    }
+  }
+  const temporary = `${configPath}.${randomUUID()}.tmp` as TempPath;
+  try {
+    ctx.fs.writeUtf8File(
+      temporary,
+      JSON.stringify(credentials, null, 2),
+      0o600,
+    );
+    ctx.fs.swapTmpFile(temporary, configPath);
+  } finally {
+    if (ctx.fs.exists(temporary)) ctx.fs.unlink(temporary);
+  }
+}
+
+export async function writeProfileCredentials(
+  ctx: Context,
+  name: string,
+  accessToken: string,
+  metadata: Omit<ProfileCredentials, "accessToken"> = {},
+  expected?: ProfileCredentials | null,
+): Promise<void> {
+  await validateProfileName(ctx, name);
+  await updateProfileSettings(ctx, async (settings) => {
+    if (expected !== undefined) {
+      const current = await readProfileCredentials(
+        ctx,
+        { name, source: "login" },
+        true,
+      );
+      if (!isDeepStrictEqual(current, expected)) {
+        return await ctx.crash({
+          exitCode: 1,
+          errorType: "fatal",
+          printedMessage: `Account "${name}" changed during login. Retry the login.`,
+        });
+      }
+    }
+    await writeCredentialsFile(ctx, name, { ...metadata, accessToken });
+    if (settings.defaultProfile === name) settings.accessToken = accessToken;
+  });
+}
+
+export async function legacyAuthForFirstProfile(
+  ctx: Context,
+  profile: ProfileSelection,
+): Promise<Extract<BigBrainAuth, { kind: "accessToken" }> | null> {
+  const settings = await readProfileSettings(ctx);
+  const profilesDir = path.dirname(profileConfigPath(profile.name));
+  if (
+    settings.defaultProfile !== undefined ||
+    (ctx.fs.exists(profilesDir) &&
+      ctx.fs.listDir(profilesDir).some((entry) => entry.name.endsWith(".json")))
+  ) {
+    return null;
+  }
+  const config = readGlobalConfig(ctx);
+  return config === null
+    ? null
+    : {
+        kind: "accessToken",
+        accessToken: config.accessToken,
+        header: `Bearer ${config.accessToken}`,
+        profile,
+      };
+}
+
+export async function initializeDefaultProfile(
+  ctx: Context,
+  profile: ProfileSelection,
+  namingLegacyAccount: boolean,
+  select = true,
+) {
+  const defaultProfile = await updateProfileSettings(ctx, async (settings) => {
+    if (settings.defaultProfile !== undefined) return undefined;
+    const credentials = await readProfileCredentials(ctx, profile);
+    const accessToken = credentials!.accessToken;
+    const legacyToken = settings.accessToken;
+    if (
+      !select &&
+      (savedProfileNames(ctx).some((name) => name !== profile.name) ||
+        (!namingLegacyAccount &&
+          typeof legacyToken === "string" &&
+          legacyToken.length > 0 &&
+          legacyToken !== accessToken))
+    )
+      return undefined;
+    if (
+      !namingLegacyAccount &&
+      typeof legacyToken === "string" &&
+      legacyToken.length > 0 &&
+      legacyToken !== accessToken
+    ) {
+      // An explicitly new login keeps the previous account as the default for
+      // directories that haven't selected an account.
+      let legacyName = "default";
+      for (
+        let suffix = 2;
+        ctx.fs.exists(profileConfigPath(legacyName));
+        suffix++
+      ) {
+        legacyName = `default-${suffix}`;
+      }
+      await writeCredentialsFile(ctx, legacyName, { accessToken: legacyToken });
+      settings.defaultProfile = legacyName;
+    } else {
+      settings.defaultProfile = profile.name;
+      settings.accessToken = accessToken;
+    }
+
+    return settings.defaultProfile;
+  });
+  if (defaultProfile !== undefined)
+    logFinishedStep(`Default account: ${defaultProfile}`);
+}
+
+export function findProfileDirectory(
+  directories: Record<string, string>,
+  cwd: string,
+): { directory: string; name: string } | null {
+  let directory = path.resolve(cwd);
+  for (;;) {
+    if (Object.hasOwn(directories, directory)) {
+      return { directory, name: directories[directory] };
+    }
+    const parent = path.dirname(directory);
+    if (parent === directory) return null;
+    directory = parent;
+  }
+}
+
+export function savedProfileNames(ctx: Context): string[] {
+  const directory = path.join(rootDirectory(), "profiles");
+  if (!ctx.fs.exists(directory)) return [];
+  return ctx.fs
+    .listDir(directory)
+    .filter((entry) => entry.name.endsWith(".json"))
+    .map((entry) => entry.name.slice(0, -5))
+    .sort();
+}
+
+export async function writeProfileDirectory(ctx: Context, name: string) {
+  await validateProfileName(ctx, name);
+  await updateProfileSettings(ctx, (settings) => {
+    settings.directories = { ...settings.directories, [process.cwd()]: name };
+  });
+  logFinishedStep(
+    `Saved directory selection: ${name} (applies to this directory and its subdirectories).`,
+  );
+  const selected = await getProfileSelection(ctx);
+  if (selected !== null && selected.name !== name) {
+    logWarning(
+      `Account "${selected.name}" selected by ${selected.source} overrides this directory selection. Remove that override to use account "${name}".`,
+    );
+  }
+}

package/src/cli/lib/run.ts

@@ -535,6 +535,8 @@ export async function pushToDeployment(
       liveComponentSources: args.liveComponentSources,
       pushAllModules: false,
       largeIndexDeletionCheck: "no verification", // `convex run` can’t push to prod
+      largeIndexBackfillCheck: "no verification", // ditto
+      stagedValidatorDeletionCheck: "no verification", // ditto
       warnOnSlowSchemaValidation: true,
       message: null,
     },

package/src/cli/lib/schemaEvaluation.ts

@@ -0,0 +1,88 @@
+import { Context } from "../../bundler/context.js";
+import { logVerbose } from "../../bundler/log.js";
+import { EvaluateSchemaResponse } from "./deployApi/evaluateSchema.js";
+import { StartPushRequest } from "./deployApi/startPush.js";
+import { evaluateSchema } from "./deploy2.js";
+import { Span } from "./tracing.js";
+
+type EvaluateOptions = {
+  url: string;
+  deploymentName: string | null;
+  adminKey: string;
+};
+
+/**
+ * A schema evaluation shared by the pre-push checks. Resolves to `null` when
+ * the deployment doesn't support `evaluate_schema` or the request failed, in
+ * which case each check skips itself.
+ */
+export type SchemaEvaluation = (
+  span: Span,
+) => Promise<EvaluateSchemaResponse | null>;
+
+/**
+ * Evaluate the schema at most once per push however many checks consume it:
+ * the first check to call sends the request under its own span, and the rest
+ * await the same promise. Each request makes the backend evaluate the schema
+ * module in an isolate and predict against a fresh snapshot, so repeating it
+ * per check would add that work and a round trip to the deploy's critical
+ * path for every check.
+ */
+/**
+ * Whether a schema module in the push calls `.staged(`. Bundles aren't
+ * minified, so the call survives bundling. A false positive from a comment or
+ * string costs one evaluation, while a push without staged validators skips
+ * the evaluation round trip on its critical path.
+ */
+export function pushStagesValidators(request: StartPushRequest): boolean {
+  return [
+    request.appDefinition.schema,
+    ...request.componentDefinitions.map((definition) => definition.schema),
+  ].some((schema) => schema?.source.includes(".staged(") ?? false);
+}
+
+export function sharedSchemaEvaluation(
+  ctx: Context,
+  request: StartPushRequest,
+  options: EvaluateOptions,
+): SchemaEvaluation {
+  let evaluation: Promise<EvaluateSchemaResponse | null> | undefined;
+  return (span) => {
+    evaluation ??= evaluateSchemaBestEffort({ ctx, span, request, options });
+    return evaluation;
+  };
+}
+
+// The checks are advisory: if evaluation fails, fall back to skipping them
+// rather than aborting the push over a check that isn't required for
+// correctness. Plausibly transient failures — network errors and 5xx
+// responses (e.g. table summaries still bootstrapping) — are retried with
+// backoff by the shared fetch wrapper (see `pushCode`); anything else is
+// deterministic, including a 2xx whose shape the CLI doesn't recognize, so
+// it skips the checks immediately.
+async function evaluateSchemaBestEffort({
+  ctx,
+  span,
+  request,
+  options,
+}: {
+  ctx: Context;
+  span: Span;
+  request: StartPushRequest;
+  options: EvaluateOptions;
+}): Promise<EvaluateSchemaResponse | null> {
+  try {
+    return await evaluateSchema(
+      ctx,
+      span,
+      request,
+      options,
+      /* bestEffort */ true,
+    );
+  } catch (error: unknown) {
+    logVerbose(
+      `Skipping schema evaluation checks: ${error instanceof Error ? error.message : String(error)}`,
+    );
+    return null;
+  }
+}

package/src/cli/lib/usageLimits.ts

@@ -21,6 +21,7 @@ export const USAGE_LIMIT_METRICS = [
   "databaseIoGb",
   "searchQueryGb",
   "dataEgressGb",
+  "aiGatewayCostDollars",
 ] as const satisfies readonly UsageLimitMetric[];
 
 // Fails to compile if the backend's `UsageLimitMetric` gains a metric not listed
@@ -47,6 +48,7 @@ export const METRIC_LABELS: Record<UsageLimitMetric, string> = {
   databaseIoGb: "Database I/O",
   searchQueryGb: "Search queries",
   dataEgressGb: "Data egress",
+  aiGatewayCostDollars: "AI Gateway",
 };
 
 export function metricLabel(metric: string): string {

package/src/cli/lib/utils/mutex.ts

@@ -1,32 +1,14 @@
 export class Mutex {
-  currentlyRunning: Promise<void> | null = null;
-  waiting: Array<() => Promise<void>> = [];
+  private tail: Promise<void> = Promise.resolve();
 
   async runExclusive<T>(fn: () => Promise<T>): Promise<T> {
-    const outerPromise = new Promise<T>((resolve, reject) => {
-      const wrappedCallback: () => Promise<void> = () => {
-        return fn()
-          .then((v: T) => resolve(v))
-          .catch((e: any) => reject(e));
-      };
-      this.enqueueCallbackForMutex(wrappedCallback);
-    });
-    return outerPromise;
-  }
-
-  private enqueueCallbackForMutex(callback: () => Promise<void>) {
-    if (this.currentlyRunning === null) {
-      this.currentlyRunning = callback().finally(() => {
-        const nextCb = this.waiting.shift();
-        if (nextCb === undefined) {
-          this.currentlyRunning = null;
-        } else {
-          this.enqueueCallbackForMutex(nextCb);
-        }
-      });
-      this.waiting.length = 0;
-    } else {
-      this.waiting.push(callback);
-    }
+    const result = this.tail.then(fn);
+    // Reserve the next position before running fn, and allow the queue to
+    // continue even if fn rejects or throws synchronously.
+    this.tail = result.then(
+      () => {},
+      () => {},
+    );
+    return result;
   }
 }