convex 1.48.0 → 1.49.0-alpha.0

package/src/cli/lib/plugins/codex.ts

@@ -0,0 +1,177 @@
+import os from "os";
+import path from "path";
+import { z } from "zod";
+import { nodeFs } from "../../../bundler/fs.js";
+import type { Cli, Places } from "./harness.js";
+import {
+  convexMarketplaces,
+  type Marketplace,
+  type Plugin,
+} from "./marketplaces.js";
+import {
+  fetchGitHubJson,
+  onPath,
+  parseList,
+  subfolders,
+  versionOf,
+} from "./utils.js";
+
+/** Builds `getCodexLocations` for the given places, so tests can point it at fixtures. */
+export const createGetCodexLocations =
+  (places: Places) => async (): Promise<Cli[]> => {
+    // The ChatGPT app also has `codex-cli/bin/codex`, a script that runs this
+    // binary. Running that script from another app sets off macOS's App
+    // Management prompt, and running the binary doesn't.
+    const desktop = path.join(
+      places.applications,
+      "ChatGPT.app",
+      "Contents",
+      "Resources",
+      "codex-cli",
+      "CodexCLI.app",
+      "Contents",
+      "MacOS",
+      "codex",
+    );
+    const found: Omit<Cli, "version">[] = [
+      ...onPath("codex", places.path).map((p) => ({
+        source: "terminal" as const,
+        path: p,
+      })),
+      ...(nodeFs.exists(desktop)
+        ? [{ source: "desktop" as const, path: desktop }]
+        : []),
+      // The extension keeps its binary in a folder named for the platform, like
+      // `bin/macos-aarch64`.
+      ...subfolders(path.join(places.home, ".vscode", "extensions"))
+        .filter((dir) => path.basename(dir).startsWith("openai.chatgpt-"))
+        .flatMap((dir) => subfolders(path.join(dir, "bin")))
+        .map((dir) => ({
+          source: "vscode" as const,
+          path: path.join(dir, "codex"),
+        })),
+    ];
+    return Promise.all(
+      found.map(async (cli) => ({
+        ...cli,
+        version: await versionOf(cli.path),
+      })),
+    );
+  };
+
+/** Every copy of Codex on this machine: the terminal CLI, the ChatGPT app's copy and the VS Code extension's copies. */
+export const getCodexLocations = createGetCodexLocations({
+  home: os.homedir(),
+  path: process.env.PATH ?? "",
+  applications: "/Applications",
+});
+
+/** The newest Convex plugin a Codex user can install, which is main of our repo. */
+async function codexLatest(): Promise<Record<string, string>> {
+  const plugin = await fetchGitHubJson<{ version: string }>(
+    "convex/convex-codex-plugin/main/plugins/convex/.codex-plugin/plugin.json",
+  );
+  return plugin === null ? {} : { convex: plugin.version };
+}
+
+/** `codex plugin marketplace list --json` */
+const codexMarketplaceList = z.object({
+  marketplaces: z.array(
+    z.object({
+      name: z.string(),
+      root: z.string(),
+      marketplaceSource: z
+        .object({ sourceType: z.string(), source: z.string() })
+        .optional(),
+    }),
+  ),
+});
+
+/** `codex plugin list --json` */
+const codexPluginList = z.object({
+  installed: z.array(
+    z.object({
+      name: z.string(),
+      marketplaceName: z.string(),
+      version: z.string(),
+      enabled: z.boolean(),
+    }),
+  ),
+});
+
+/** Reads the plugins in `codex plugin list --json`, or returns null when it doesn't match. */
+function codexPlugins(pluginList: string): Plugin[] | null {
+  const plugins = parseList(
+    codexPluginList,
+    pluginList,
+    "codex plugin list --json",
+  );
+  if (plugins === null) return null;
+  return plugins.installed.map((p) => ({
+    name: p.name,
+    marketplace: p.marketplaceName,
+    version: p.version,
+    enabled: p.enabled,
+  }));
+}
+
+/** Reads Codex's Convex marketplaces from `codex plugin marketplace list --json` and `codex plugin list --json`, or returns null when either doesn't match. */
+export function codexMarketplaces(
+  marketplaceList: string,
+  pluginList: string,
+): Marketplace[] | null {
+  const marketplaces = parseList(
+    codexMarketplaceList,
+    marketplaceList,
+    "codex plugin marketplace list --json",
+  );
+  const plugins = codexPlugins(pluginList);
+  if (marketplaces === null || plugins === null) return null;
+  return convexMarketplaces(
+    marketplaces.marketplaces.map((m) => ({
+      name: m.name,
+      // Anything that isn't a git marketplace is shown by its folder.
+      source:
+        m.marketplaceSource?.sourceType === "git"
+          ? { kind: "git", url: m.marketplaceSource.source }
+          : { kind: "folder", path: m.root },
+    })),
+    plugins,
+  );
+}
+
+const pluginId = "convex@convex-codex-plugin";
+/** The install step that brings an installed plugin to the latest version. When it fails, the user keeps an old version. */
+const upgradeStep = ["plugin", "marketplace", "upgrade", "convex-codex-plugin"];
+
+/** Codex as `npx convex plugins` installs and reads it. */
+export const codex = {
+  id: "codex" as const,
+  name: "Codex",
+  plugin: pluginId,
+  upgradeStep,
+  installSteps: [
+    // Notes:
+    //
+    // 1. The marketplace is added from the `get-convex` org, the org's name
+    //    before the rename to `convex`. Codex refuses to re-add a marketplace
+    //    from a different source, so switching would make install fail for
+    //    anyone who installed before. Switch once install handles an existing
+    //    marketplace.
+    //
+    // 2. Codex installs plugins from its own local copy of each marketplace, so
+    //    it doesn't see the new plugin versions we publish until that copy is
+    //    updated.
+
+    // Add the marketplace
+    ["plugin", "marketplace", "add", "get-convex/convex-codex-plugin"],
+    // Update the local copy from GitHub, so the latest plugin version is
+    // available to install - this is important
+    upgradeStep,
+    // Install the latest plugin from the updated marketplace
+    ["plugin", "add", pluginId],
+  ],
+  plugins: codexPlugins,
+  marketplaces: codexMarketplaces,
+  latest: codexLatest,
+};

package/src/cli/lib/plugins/harness.ts

@@ -0,0 +1,49 @@
+/**
+ * The Claude Code and Codex CLIs found on the user's machine. Users of the
+ * desktop apps or the VS Code extensions often have no CLI on PATH, but each
+ * app ships its own copy, which takes the same commands and shares the same
+ * plugin state as the terminal CLI.
+ */
+import * as semver from "semver";
+import { getClaudeCodeLocations } from "./claudeCode.js";
+import { getCodexLocations } from "./codex.js";
+
+/** Where a copy of a harness CLI was found. */
+type CliSource = "terminal" | "desktop" | "vscode";
+
+/** A copy of a harness CLI and the first line it prints for `--version`. */
+export type Cli = { source: CliSource; path: string; version: string };
+
+/** Where to look for harness CLIs. */
+export type Places = {
+  home: string;
+  /** Directories separated like `$PATH`. */
+  path: string;
+  /** Where macOS apps are installed, `/Applications`. */
+  applications: string;
+};
+
+/** The copy with the highest version. Versions compare as semver, where a release beats its prereleases. */
+function newest(clis: Cli[]): Cli | undefined {
+  // Every copy printed its version when the finder ran it.
+  const version = (cli: Cli) =>
+    semver.coerce(cli.version, { includePrerelease: true })!;
+  return [...clis].sort((a, b) => semver.rcompare(version(a), version(b)))[0];
+}
+
+/** Builds `getNewestHarnessLocations` from the given location getters, so tests can pass fixed locations. */
+export const createGetNewestHarnessLocations =
+  (getters: {
+    getCodexLocations: () => Promise<Cli[]>;
+    getClaudeCodeLocations: () => Promise<Cli[]>;
+  }) =>
+  async () => ({
+    codex: newest(await getters.getCodexLocations()),
+    claudeCode: newest(await getters.getClaudeCodeLocations()),
+  });
+
+/** The newest copy of each harness's CLI on this machine, or `undefined` for a harness that isn't installed. */
+export const getNewestHarnessLocations = createGetNewestHarnessLocations({
+  getCodexLocations,
+  getClaudeCodeLocations,
+});

package/src/cli/lib/plugins/marketplaces.ts

@@ -0,0 +1,94 @@
+import { logOutput } from "../../../bundler/log.js";
+import { formatRows } from "./utils.js";
+
+/** Names of the Convex plugins, including the old internal beta. */
+const pluginNames = ["convex", "convex-beta"];
+
+/** A Convex plugin as its harness reports it. */
+export type Plugin = {
+  name: string;
+  marketplace: string;
+  /** The version that runs. */
+  version: string;
+  /** Claude Code only. */
+  scope?: string;
+  enabled: boolean;
+  /** Claude Code folder installs only. The copy cached at install time, when it differs from the folder. */
+  cachedVersion?: string | undefined;
+};
+
+/** Where a marketplace comes from. `other` is a source type we haven't seen, by its type name. */
+export type Source =
+  | { kind: "github"; repo: string }
+  | { kind: "git"; url: string }
+  | { kind: "folder"; path: string }
+  | { kind: "other"; type: string };
+
+/** A marketplace named after Convex or holding a Convex plugin, with the Convex plugins installed from it. */
+export type Marketplace = { name: string; source: Source; plugins: Plugin[] };
+
+/** Keeps the marketplaces named after Convex or holding a Convex plugin, each with its Convex plugins. */
+export function convexMarketplaces(
+  marketplaces: { name: string; source: Source }[],
+  plugins: Plugin[],
+): Marketplace[] {
+  const convex = plugins.filter((p) => pluginNames.includes(p.name));
+  return marketplaces
+    .map((m) => ({
+      ...m,
+      plugins: convex.filter((p) => p.marketplace === m.name),
+    }))
+    .filter((m) => m.name.startsWith("convex") || m.plugins.length > 0);
+}
+
+/** Describes a marketplace's source for people. */
+function sourceText(source: Source): string {
+  switch (source.kind) {
+    case "github":
+      return `github.com/${source.repo}`;
+    case "git":
+      return source.url.replace(/^https?:\/\//, "").replace(/\.git$/, "");
+    case "folder":
+      return source.path;
+    case "other":
+      return source.type;
+  }
+}
+
+/** Prints each Convex marketplace with its Convex plugins indented under it. */
+export function logMarketplaces(
+  title: string,
+  marketplaces: Marketplace[],
+  latest: Record<string, string>,
+) {
+  logOutput(title);
+  if (marketplaces.length === 0) {
+    logOutput("  No Convex plugin installed");
+    return;
+  }
+  const marketplaceLines = formatRows(
+    marketplaces.map((m) => [m.name, sourceText(m.source)]),
+  );
+  // Plugin rows are formatted together so their columns line up across marketplaces.
+  const pluginLines = formatRows(
+    marketplaces
+      .flatMap((m) => m.plugins)
+      .map((p) => [
+        p.name,
+        p.version,
+        latest[p.name] === undefined ? "" : `latest ${latest[p.name]}`,
+        p.scope ?? "-",
+        p.enabled ? "enabled" : "disabled",
+        p.cachedVersion === undefined ? "" : `cached copy ${p.cachedVersion}`,
+      ]),
+    "    ",
+  );
+  let next = 0;
+  marketplaces.forEach((m, i) => {
+    logOutput(marketplaceLines[i]);
+    for (const line of pluginLines.slice(next, next + m.plugins.length)) {
+      logOutput(line);
+    }
+    next += m.plugins.length;
+  });
+}

package/src/cli/lib/plugins/skills.ts

@@ -0,0 +1,101 @@
+import os from "os";
+import path from "path";
+import { nodeFs } from "../../../bundler/fs.js";
+import { logOutput } from "../../../bundler/log.js";
+import { fetchAgentSkillsCatalog } from "../versionApi.js";
+
+/** Finds folders named `skills` in a folder's children and grandchildren, without following links. */
+function skillsFolders(dir: string, depth: number): string[] {
+  let entries;
+  try {
+    entries = nodeFs.listDir(dir);
+  } catch {
+    return [];
+  }
+  return entries
+    .filter((entry) => entry.isDirectory())
+    .flatMap((entry) => {
+      const child = path.join(dir, entry.name);
+      if (entry.name === "skills") return [child];
+      return depth > 1 ? skillsFolders(child, depth - 1) : [];
+    });
+}
+
+/** Convex skill names from the catalog on version.convex.dev, split into current and deleted ones. */
+type SkillNames = { current: Set<string>; deleted: Set<string> };
+
+/** A skills folder holding Convex skills. */
+type SkillsFolder = {
+  path: string;
+  current: number;
+  deleted: number;
+  brokenLinks: number;
+};
+
+/** Finds the skills folders in a root's hidden folders that hold Convex skills, counting deleted skills and broken links. */
+export function convexSkillsFolders(
+  root: string,
+  names: SkillNames,
+): SkillsFolder[] {
+  return nodeFs
+    .listDir(root)
+    .filter((entry) => entry.isDirectory() && entry.name.startsWith("."))
+    .flatMap((entry) => skillsFolders(path.join(root, entry.name), 2))
+    .map((folder) => {
+      const entries = nodeFs.listDir(folder).map((entry) => entry.name);
+      const current = entries.filter((name) => names.current.has(name));
+      const deleted = entries.filter((name) => names.deleted.has(name));
+      // exists follows links, so it's false for a link whose target is gone.
+      const brokenLinks = [...current, ...deleted].filter(
+        (name) => !nodeFs.exists(path.join(folder, name)),
+      ).length;
+      return {
+        path: folder,
+        current: current.length,
+        deleted: deleted.length,
+        brokenLinks,
+      };
+    })
+    .filter((folder) => folder.current + folder.deleted > 0);
+}
+
+/** Prints each skills folder in a root's hidden folders that holds Convex skills, counting deleted skills and broken links. */
+function logSkills(
+  title: string,
+  root: string,
+  label: string,
+  names: SkillNames,
+) {
+  const folders = convexSkillsFolders(root, names);
+  logOutput(title);
+  if (folders.length === 0) {
+    logOutput("  No Convex skills found");
+    return;
+  }
+  for (const folder of folders) {
+    logOutput(
+      `  ${folder.path.replace(root, label)}  ${folder.current} Convex skills, ${folder.deleted} deleted, ${folder.brokenLinks} broken links`,
+    );
+  }
+}
+
+/** Prints the Convex skills in the hidden folders of the home folder and the current folder. */
+export async function logConvexSkills() {
+  const catalog = await fetchAgentSkillsCatalog();
+  if (catalog.kind === "error") {
+    logOutput("Skills: couldn't fetch the list of Convex skills");
+  } else {
+    const names = (kind: "active" | "deleted") =>
+      new Set(
+        catalog.data.skills
+          .filter((skill) => skill.status.kind === kind)
+          .map((skill) => skill.skillName),
+      );
+    const skillNames = {
+      current: names("active"),
+      deleted: names("deleted"),
+    };
+    logSkills("Global skills", os.homedir(), "~", skillNames);
+    logSkills("Project skills", process.cwd(), ".", skillNames);
+  }
+}

package/src/cli/lib/plugins/utils.ts

@@ -0,0 +1,121 @@
+import child_process from "child_process";
+import path from "path";
+import { promisify } from "util";
+import * as Sentry from "@sentry/node";
+import { z } from "zod";
+import { nodeFs } from "../../../bundler/fs.js";
+
+/** The file the shell runs for `name`, like `which`. */
+export function onPath(name: string, pathDirs: string): string[] {
+  const dir = pathDirs
+    .split(path.delimiter)
+    .find((d) => nodeFs.exists(path.join(d, name)));
+  return dir === undefined ? [] : [path.join(dir, name)];
+}
+
+/** The folders directly inside `dir`, sorted by name, or none when `dir` doesn't exist. */
+export function subfolders(dir: string): string[] {
+  let entries;
+  try {
+    entries = nodeFs.listDir(dir);
+  } catch {
+    return [];
+  }
+  return entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => path.join(dir, entry.name))
+    .sort();
+}
+
+/** The first line a CLI prints for `--version`. */
+export async function versionOf(cli: string): Promise<string> {
+  const { stdout } = await promisify(child_process.execFile)(cli, [
+    "--version",
+  ]);
+  return stdout.split("\n")[0];
+}
+
+/** Runs a command and resolves to whether it succeeded and everything it printed. */
+export function run(
+  command: string,
+  args: string[],
+): Promise<{ ok: boolean; output: string }> {
+  return new Promise((resolve) => {
+    let output = "";
+
+    const proc = child_process.spawn(command, args, {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    proc.stdout.on("data", (chunk) => (output += chunk));
+    proc.stderr.on("data", (chunk) => (output += chunk));
+    proc.on("error", () => resolve({ ok: false, output }));
+    proc.on("close", (code) => resolve({ ok: code === 0, output }));
+  });
+}
+
+/** Runs a command and resolves to its stdout, or null when it can't be started. */
+export function capture(
+  command: string,
+  args: string[],
+): Promise<string | null> {
+  return new Promise((resolve) => {
+    let stdout = "";
+
+    const proc = child_process.spawn(command, args, {
+      stdio: ["ignore", "pipe", "inherit"],
+    });
+
+    proc.stdout.on("data", (chunk) => (stdout += chunk));
+    proc.on("error", () => resolve(null));
+    proc.on("close", () => resolve(stdout));
+  });
+}
+
+/** Fetches a JSON file from GitHub, or resolves to null when it can't, reporting why to Sentry. */
+export async function fetchGitHubJson<T>(path: string): Promise<T | null> {
+  try {
+    const response = await fetch(`https://raw.githubusercontent.com/${path}`);
+    if (!response.ok) {
+      Sentry.captureException(
+        new Error(`Failed to fetch ${path}: status = ${response.status}`),
+      );
+      return null;
+    }
+    return (await response.json()) as T;
+  } catch (error) {
+    Sentry.captureException(error);
+    return null;
+  }
+}
+
+/** Parses a JSON list from a harness CLI, or returns null and reports to Sentry when it isn't JSON or doesn't match. */
+export function parseList<T>(
+  schema: z.ZodType<T, z.ZodTypeDef, unknown>,
+  list: string,
+  command: string,
+): T | null {
+  let result;
+  try {
+    result = schema.safeParse(JSON.parse(list));
+  } catch {
+    // A failing CLI can print nothing, which isn't JSON.
+    result = undefined;
+  }
+  if (!result?.success) {
+    Sentry.captureMessage(`Unexpected output from \`${command}\``, "error");
+    return null;
+  }
+  return result.data;
+}
+
+/** Pads each column to its widest cell and indents every row. */
+export function formatRows(rows: string[][], indent = "  "): string[] {
+  if (rows.length === 0) return [];
+  const widths = rows[0].map((_, i) =>
+    Math.max(...rows.map((row) => row[i].length)),
+  );
+  return rows.map((row) =>
+    `${indent}${row.map((cell, i) => cell.padEnd(widths[i])).join("  ")}`.trimEnd(),
+  );
+}

package/src/cli/lib/utils/utils.ts

@@ -45,6 +45,7 @@ export const CONVEX_DEPLOYMENT_ENV_VAR_NAME = "CONVEX_DEPLOYMENT";
 export const CONVEX_SELF_HOSTED_URL_VAR_NAME = "CONVEX_SELF_HOSTED_URL";
 export const CONVEX_SELF_HOSTED_ADMIN_KEY_VAR_NAME =
   "CONVEX_SELF_HOSTED_ADMIN_KEY";
+export const IGNORED_DEPLOY_KEY = "<ignore_deploy_key>";
 const MAX_RETRIES = 6;
 // After 3 retries, log a progress message that we're retrying the request
 const RETRY_LOG_THRESHOLD = 3;
@@ -69,7 +70,7 @@ export async function processDeployKeyValue(
   }
 
   // Check for <ignore_deploy_key> sentinel
-  if (deployKey === "<ignore_deploy_key>") {
+  if (deployKey === IGNORED_DEPLOY_KEY) {
     return undefined;
   }
 
@@ -96,9 +97,20 @@ export async function processDeployKeyValue(
 export function readDeployKeyFromEnv(
   getEnv: (name: string) => string | undefined | null,
 ): string | undefined {
-  const fromDeployKey = getEnv(CONVEX_DEPLOY_KEY_ENV_VAR_NAME);
-  const fromToken = getEnv(CONVEX_DEPLOYMENT_TOKEN_ENV_VAR_NAME);
-  return fromDeployKey || fromToken || undefined;
+  return readDeployKeyEnvVar(getEnv)?.value;
+}
+
+export function readDeployKeyEnvVar(
+  getEnv: (name: string) => string | undefined | null,
+): { name: string; value: string } | undefined {
+  for (const name of [
+    CONVEX_DEPLOY_KEY_ENV_VAR_NAME,
+    CONVEX_DEPLOYMENT_TOKEN_ENV_VAR_NAME,
+  ]) {
+    const value = getEnv(name);
+    if (value) return { name, value };
+  }
+  return undefined;
 }
 
 // The prefix the backend's `require_operation` checks use when an identity
@@ -108,17 +120,9 @@ const PERMISSION_DENIED_MESSAGE_PREFIX =
   "You do not have permission to perform this operation";
 
 function deployKeyPermissionAdvice(): string | null {
-  // Name whichever env var is actually set. `CONVEX_DEPLOY_KEY` takes
-  // precedence over `CONVEX_DEPLOYMENT_TOKEN` when both are present.
-  const envVarName = process.env[CONVEX_DEPLOY_KEY_ENV_VAR_NAME]
-    ? CONVEX_DEPLOY_KEY_ENV_VAR_NAME
-    : process.env[CONVEX_DEPLOYMENT_TOKEN_ENV_VAR_NAME]
-      ? CONVEX_DEPLOYMENT_TOKEN_ENV_VAR_NAME
-      : null;
-  if (envVarName === null) {
-    return null;
-  }
-  return `This is determined by the permissions granted to ${envVarName}.`;
+  const envVar = readDeployKeyEnvVar((name) => process.env[name]);
+  if (envVar === undefined || envVar.value === IGNORED_DEPLOY_KEY) return null;
+  return `This is determined by the permissions granted to ${envVar.name}.`;
 }
 
 /**