convex-rust 0.0.0-stage → 0.1.0

package/dist/cli/rustApi.js

@@ -0,0 +1,609 @@
+// Typed `src/_generated/api.rs`: Rust references to Convex functions for ctx.run_query & co.
+// Port of the fork CLI's codegen_templates/rust_api.ts. Instead of the fork's bundler, the TS
+// functions are analysed locally with esbuild + a short-lived Node subprocess (no backend).
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+import { convexModuleName } from "./manifest.js";
+const compareStrings = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+export const RUST_API_HEADER = `// @generated by \`convex-rust\`. Do not edit; changes are overwritten.
+//
+// Typed references to your Convex functions for \`ctx.run_query\`,
+// \`ctx.run_mutation\` and \`ctx.run_action\`. Include it from \`lib.rs\` with
+//
+//     #[path = "_generated/api.rs"]
+//     pub mod api;
+#![allow(
+    non_upper_case_globals,
+    non_snake_case,
+    non_camel_case_types,
+    dead_code,
+    unused_imports,
+    clippy::all
+)]
+`;
+/** Placeholder `api.rs` until the first codegen fills it in. */
+export const RUST_API_STUB = `${RUST_API_HEADER}
+// No functions yet: this file is filled in by the next \`convex-rust build\` or \`convex-rust codegen\`.
+`;
+const RUST_KEYWORDS = new Set([
+    "as",
+    "break",
+    "const",
+    "continue",
+    "else",
+    "enum",
+    "extern",
+    "false",
+    "fn",
+    "for",
+    "if",
+    "impl",
+    "in",
+    "let",
+    "loop",
+    "match",
+    "mod",
+    "move",
+    "mut",
+    "pub",
+    "ref",
+    "return",
+    "static",
+    "struct",
+    "trait",
+    "true",
+    "type",
+    "unsafe",
+    "use",
+    "where",
+    "while",
+    "async",
+    "await",
+    "dyn",
+    "abstract",
+    "become",
+    "box",
+    "do",
+    "final",
+    "macro",
+    "override",
+    "priv",
+    "typeof",
+    "unsized",
+    "virtual",
+    "yield",
+    "try",
+    "gen",
+]);
+const NON_RAW_KEYWORDS = new Set(["self", "Self", "super", "crate", "_"]);
+/** A Rust identifier for `name` (a Convex module, function or field name). */
+export function rustIdent(name) {
+    let id = name.replace(/[^A-Za-z0-9_]/g, "_");
+    if (id === "" || /^[0-9]/.test(id))
+        id = `_${id}`;
+    if (NON_RAW_KEYWORDS.has(id))
+        return `${id}_`;
+    if (RUST_KEYWORDS.has(id))
+        return `r#${id}`;
+    return id;
+}
+/** `list_messages` / `listMessages` -> `ListMessages`. */
+export function pascalCase(name) {
+    const parts = name.split(/[^A-Za-z0-9]+/).filter((p) => p !== "");
+    const out = parts.map((p) => p[0].toUpperCase() + p.slice(1)).join("");
+    return /^[0-9]/.test(out) || out === "" ? `T${out}` : out;
+}
+function parseValidatorJson(json) {
+    if (!json)
+        return null;
+    return JSON.parse(json);
+}
+function derive() {
+    return `#[derive(Clone, Debug, Serialize, Deserialize)]\n#[serde(crate = "convex_fn::serde")]`;
+}
+/**
+ * Rust type for a validator. Objects become structs named after `hint`
+ * (pushed to `gen.structs`); anything the Rust types can't express exactly
+ * (general unions, mixed literals) becomes `serde_json::Value`.
+ */
+export function rustType(v, hint, gen) {
+    switch (v.type) {
+        case "null":
+            return "()";
+        case "number":
+            return "f64";
+        case "bigint":
+            return "Int64";
+        case "boolean":
+            return "bool";
+        case "string":
+        case "id":
+            return "String";
+        case "bytes":
+            return "Bytes";
+        case "commitTs":
+            return "Int64";
+        case "any":
+            return "Value";
+        case "literal": {
+            const value = v.value;
+            if (typeof value === "string")
+                return "String";
+            if (typeof value === "boolean")
+                return "bool";
+            if (typeof value === "number")
+                return "f64";
+            return "Value";
+        }
+        case "array":
+            return `Vec<${rustType(v.value, `${hint}Item`, gen)}>`;
+        case "record":
+            return `BTreeMap<String, ${rustType(v.values.fieldType, `${hint}Value`, gen)}>`;
+        case "union": {
+            const members = v.value;
+            const nonNull = members.filter((m) => m.type !== "null");
+            if (members.length === 2 && nonNull.length === 1) {
+                return `Option<${rustType(nonNull[0], hint, gen)}>`;
+            }
+            if (members.every((m) => m.type === "literal" && typeof m.value === "string")) {
+                return "String";
+            }
+            return "Value";
+        }
+        case "object": {
+            gen.structs.push(structDef(v, hint, gen));
+            return hint;
+        }
+        default:
+            return "Value";
+    }
+}
+function structDef(v, name, gen) {
+    const lines = [derive(), `pub struct ${name} {`];
+    const fields = Object.entries(v.value).sort(([a], [b]) => compareStrings(a, b));
+    for (const [fieldName, field] of fields) {
+        const ident = rustIdent(fieldName);
+        const bare = ident.replace(/^r#/, "");
+        const ty = rustType(field.fieldType, `${name}${pascalCase(fieldName)}`, gen);
+        if (field.fieldType.type === "id") {
+            lines.push(`    /// \`Id<"${field.fieldType.tableName}">\``);
+        }
+        const attrs = [];
+        if (bare !== fieldName)
+            attrs.push(`rename = ${JSON.stringify(fieldName)}`);
+        if (field.optional) {
+            attrs.push("default", 'skip_serializing_if = "Option::is_none"');
+        }
+        if (attrs.length > 0)
+            lines.push(`    #[serde(${attrs.join(", ")})]`);
+        lines.push(`    pub ${ident}: ${field.optional && !ty.startsWith("Option<") ? `Option<${ty}>` : ty},`);
+    }
+    lines.push("}");
+    return lines.join("\n");
+}
+function newNode() {
+    return { fns: [], structs: [], kids: new Map() };
+}
+function descend(root, modulePath) {
+    let node = root;
+    for (const part of modulePath.split("/")) {
+        const key = rustIdent(part);
+        if (!node.kids.has(key))
+            node.kids.set(key, newNode());
+        node = node.kids.get(key);
+    }
+    return node;
+}
+function renderFunction(node, fnPath, name, argsJson, returnsJson) {
+    const gen = { structs: [] };
+    const base = pascalCase(name);
+    let argsType = "Value";
+    try {
+        const args = parseValidatorJson(argsJson);
+        if (args && args.type === "object") {
+            argsType = rustType(args, `${base}Args`, gen);
+        }
+    }
+    catch {
+        // Keep `Value`.
+    }
+    let returnsType = "Value";
+    try {
+        const returns = parseValidatorJson(returnsJson);
+        if (returns)
+            returnsType = rustType(returns, `${base}Returns`, gen);
+    }
+    catch {
+        // Keep `Value`.
+    }
+    node.structs.push(...gen.structs);
+    node.fns.push(`pub const ${rustIdent(name)}: FunctionRef<${argsType}, ${returnsType}> =\n    FunctionRef::new(${JSON.stringify(fnPath)});`);
+}
+function indent(text, depth) {
+    const pad = "    ".repeat(depth);
+    return text
+        .split("\n")
+        .map((l) => (l === "" ? l : pad + l))
+        .join("\n");
+}
+function renderNode(node, depth) {
+    const parts = [];
+    parts.push(indent("use convex_fn::serde::{Deserialize, Serialize};\nuse convex_fn::{Bytes, FunctionRef, Int64};\nuse serde_json::Value;\nuse std::collections::BTreeMap;", depth));
+    for (const s of node.structs)
+        parts.push(indent(s, depth));
+    for (const f of node.fns)
+        parts.push(indent(f, depth));
+    for (const [name, kid] of [...node.kids.entries()].sort(([a], [b]) => compareStrings(a, b))) {
+        parts.push(`${indent(`pub mod ${name} {`, depth)}\n${renderNode(kid, depth + 1)}\n${indent("}", depth)}`);
+    }
+    return parts.join("\n\n");
+}
+/** Contents of `api.rs` for the analyzed modules (`messages.js`, ...). */
+export function rustApiCodegen(modules) {
+    const pub = newNode();
+    const internal = newNode();
+    const paths = Object.keys(modules).sort(compareStrings);
+    for (const modulePath of paths) {
+        const moduleName = modulePath.replace(/\.js$/, "");
+        for (const f of modules[modulePath].functions) {
+            const visibility = f.visibility?.kind;
+            if (visibility !== "public" && visibility !== "internal")
+                continue;
+            const target = visibility === "public" ? pub : internal;
+            renderFunction(descend(target, moduleName), `${moduleName}:${f.name}`, f.name, f.args, f.returns);
+        }
+    }
+    const out = [RUST_API_HEADER, renderNode(pub, 0)];
+    if (internal.kids.size > 0) {
+        if (pub.kids.has("internal")) {
+            out.push("// `internal` functions are omitted: a module is named `internal`.");
+        }
+        else {
+            out.push(`/// \`internal\` functions (\`internalQuery\`, \`internalMutation\`, \`internalAction\`).\npub mod internal {\n${renderNode(internal, 1)}\n}`);
+        }
+    }
+    return out.join("\n") + "\n";
+}
+// ---------------------------------------------------------------------------------------------
+// Local analysis of the project's TypeScript functions
+/** Runs in the subprocess: prints one JSON line `{ [module]: fn[] }`. */
+export const ANALYZE_SCRIPT = `
+import { pathToFileURL } from "node:url";
+const [dir, ...mods] = process.argv.slice(1);
+const out = {};
+for (const m of mods) {
+  let ns;
+  try {
+    ns = await import(pathToFileURL(dir + "/" + m).href);
+  } catch (e) {
+    process.stderr.write("convex-rust: could not load " + m + ": " + (e && e.message || e) + "\\n");
+    continue;
+  }
+  const fns = [];
+  for (const [name, f] of Object.entries(ns)) {
+    if (typeof f !== "function") continue;
+    const udfType = f.isQuery ? "Query" : f.isMutation ? "Mutation" : f.isAction ? "Action" : null;
+    if (udfType === null) continue;
+    const kind = f.isPublic ? "public" : f.isInternal ? "internal" : null;
+    if (kind === null) continue;
+    let args = null, returns = null;
+    try { args = f.exportArgs(); returns = f.exportReturns(); } catch {}
+    fns.push({ name, udfType, visibility: { kind }, args, returns });
+  }
+  out[m] = fns;
+}
+process.stdout.write("\\n@@convex-analysis@@" + JSON.stringify(out) + "\\n");
+`;
+export function parseAnalysisOutput(stdout) {
+    const marker = "@@convex-analysis@@";
+    const i = stdout.lastIndexOf(marker);
+    if (i < 0)
+        return null;
+    try {
+        const raw = JSON.parse(stdout.slice(i + marker.length).split("\n")[0]);
+        const result = {};
+        for (const [mod, functions] of Object.entries(raw))
+            result[mod] = { functions };
+        return result;
+    }
+    catch {
+        return null;
+    }
+}
+const ENTRY_EXTS = [".js", ".mjs", ".cjs", ".jsx", ".ts", ".mts", ".cts", ".tsx"];
+const SKIP_DIRS = new Set(["node_modules", "_generated", "target", "src", "_wasm", ".git"]);
+/**
+ * TS/JS entry points of the functions directory, following the stock bundler's rules
+ * (skips `_generated`, dotfiles, `schema.*`, files with several dots, component directories).
+ * `skipDirs` are absolute directories to leave out (e.g. the Rust output directory).
+ */
+export function findEntryPoints(functionsDir, skipDirs = []) {
+    const out = [];
+    const skip = new Set(skipDirs.map((d) => path.resolve(d)));
+    const visit = (dir) => {
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        entries.sort((a, b) => compareStrings(a.name, b.name));
+        for (const e of entries) {
+            const p = path.join(dir, e.name);
+            if (e.isDirectory()) {
+                if (SKIP_DIRS.has(e.name) || e.name.startsWith(".") || skip.has(p))
+                    continue;
+                if (fs.existsSync(path.join(p, "convex.config.ts")))
+                    continue;
+                visit(p);
+            }
+            else if (e.isFile()) {
+                const base = e.name;
+                if (!ENTRY_EXTS.some((x) => base.toLowerCase().endsWith(x)))
+                    continue;
+                if (base.endsWith(".d.ts") || base.startsWith(".") || base.startsWith("#"))
+                    continue;
+                if (base === "schema.ts" || base === "schema.js")
+                    continue;
+                if ((base.match(/\./g) || []).length > 1)
+                    continue;
+                if (path.relative(functionsDir, p).includes(" "))
+                    continue;
+                out.push(p);
+            }
+        }
+    };
+    visit(functionsDir);
+    return out;
+}
+function runNode(script, args, cwd, timeoutMs) {
+    return new Promise((resolve) => {
+        const child = spawn(process.execPath, ["--input-type=module", "-e", script, "--", ...args], {
+            cwd,
+            stdio: ["ignore", "pipe", "pipe"],
+            env: process.env,
+        });
+        let stdout = "";
+        let stderr = "";
+        child.stdout.setEncoding("utf8");
+        child.stdout.on("data", (d) => (stdout += d));
+        child.stderr.setEncoding("utf8");
+        child.stderr.on("data", (d) => (stderr += d));
+        const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+        child.on("error", () => {
+            clearTimeout(timer);
+            resolve({ stdout, stderr, ok: false });
+        });
+        child.on("close", (code) => {
+            clearTimeout(timer);
+            resolve({ stdout, stderr, ok: code === 0 });
+        });
+    });
+}
+async function loadEsbuild(projectRoot) {
+    // Prefer the esbuild that ships with the project's `convex` package.
+    try {
+        const req = createRequire(path.join(projectRoot, "package.json"));
+        const convexDir = path.dirname(req.resolve("convex/package.json"));
+        const esb = createRequire(path.join(convexDir, "package.json")).resolve("esbuild");
+        return (await import(pathToFileURL(esb).href)).default ?? (await import(pathToFileURL(esb).href));
+    }
+    catch {
+        // fall through
+    }
+    try {
+        return (await import("esbuild"));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Analyzes the TS functions under `functionsDir` without a backend: bundles each module with
+ * esbuild (npm packages external, so `convex/server` is the project's), imports the bundles in a
+ * Node subprocess and reads each registered function's visibility and validators. Returns null
+ * when analysis is not possible (callers keep the existing api.rs).
+ */
+export async function analyzeTsFunctions(projectRoot, functionsDir, skipDirs = [], timeoutMs = 60_000, log = () => { }) {
+    const entries = findEntryPoints(functionsDir, skipDirs);
+    if (entries.length === 0)
+        return { modules: {}, warnings: [] };
+    const esbuild = await loadEsbuild(projectRoot);
+    if (!esbuild)
+        return null;
+    // Inside the project so that external imports (`convex/server`) resolve from its node_modules.
+    const base = path.join(projectRoot, ".convex-rust");
+    fs.mkdirSync(base, { recursive: true });
+    const tmp = fs.mkdtempSync(path.join(base, "analysis-"));
+    try {
+        fs.writeFileSync(path.join(tmp, "package.json"), '{"type":"module"}\n');
+        const names = entries.map((p) => path.relative(functionsDir, p).split(path.sep).join("/").replace(/\.[^./]+$/, ""));
+        await esbuild.build({
+            entryPoints: Object.fromEntries(entries.map((p, i) => [names[i], p])),
+            outdir: tmp,
+            outExtension: { ".js": ".mjs" },
+            bundle: true,
+            format: "esm",
+            platform: "node",
+            packages: "external",
+            logLevel: "silent",
+            write: true,
+            loader: { ".wasm": "empty" },
+            absWorkingDir: projectRoot,
+            // Per-module failures should not sink the whole run.
+            metafile: false,
+        }).catch(async (e) => {
+            // Retry module by module so one broken file does not hide the rest.
+            const failed = [];
+            for (let i = 0; i < entries.length; i++) {
+                try {
+                    await esbuild.build({
+                        entryPoints: { [names[i]]: entries[i] },
+                        outdir: tmp,
+                        outExtension: { ".js": ".mjs" },
+                        bundle: true,
+                        format: "esm",
+                        platform: "node",
+                        packages: "external",
+                        logLevel: "silent",
+                        loader: { ".wasm": "empty" },
+                        absWorkingDir: projectRoot,
+                    });
+                }
+                catch {
+                    failed.push(names[i]);
+                }
+            }
+            if (failed.length === entries.length)
+                throw e;
+            log(`warning: could not bundle ${failed.join(", ")}: ${String(e?.errors?.[0]?.text ?? e?.message ?? e)}`);
+        });
+        const present = names.filter((n) => fs.existsSync(path.join(tmp, `${n}.mjs`)));
+        const { stdout, stderr, ok } = await runNode(ANALYZE_SCRIPT, [tmp, ...present.map((n) => `${n}.mjs`)], projectRoot, timeoutMs);
+        if (!ok) {
+            log(`warning: the analysis process failed: ${stderr.trim().split("\n").slice(-3).join(" | ")}`);
+            return null;
+        }
+        const raw = parseAnalysisOutput(stdout);
+        if (raw === null)
+            return null;
+        const modules = {};
+        for (const [k, v] of Object.entries(raw))
+            modules[k.replace(/\.mjs$/, ".js")] = v;
+        const warnings = stderr
+            .split("\n")
+            .filter((l) => l.startsWith("convex-rust: could not load"))
+            .map((l) => l.replace(/^convex-rust: /, ""));
+        for (const n of names)
+            if (!present.includes(n))
+                warnings.push(`could not bundle ${n}`);
+        return { modules, warnings };
+    }
+    catch (e) {
+        const first = e?.errors?.[0];
+        const why = first ? `${first.text}${first.location ? ` (${first.location.file}:${first.location.line})` : ""}` : String(e?.message ?? e);
+        log(`warning: could not bundle the TypeScript functions: ${why}` +
+            (/_generated/.test(why) ? " (run `npx convex dev` or `npx convex codegen` once so convex/_generated exists)" : ""));
+        return null;
+    }
+    finally {
+        fs.rmSync(tmp, { recursive: true, force: true });
+    }
+}
+/** Rust functions (from the wasm manifest) as analysed modules under `<outRel>/<module>.js`. */
+export function rustEntriesAsModules(entries, outRel) {
+    const mods = {};
+    for (const e of entries) {
+        // HTTP actions are routed in http.ts, not called through `api`.
+        if (e.kind === "http")
+            continue;
+        const key = `${outRel ? outRel + "/" : ""}${convexModuleName(e.modulePath)}.js`;
+        (mods[key] ??= { functions: [] }).functions.push({
+            name: e.name,
+            udfType: "Action",
+            visibility: { kind: e.visibility },
+            args: e.args == null ? null : JSON.stringify(e.args),
+            returns: e.returns == null ? null : JSON.stringify(e.returns),
+        });
+    }
+    return mods;
+}
+// ---------------------------------------------------------------------------------------------
+// Orchestration used by build / codegen
+const RELATIVE_API = path.join("src", "_generated", "api.rs");
+export function apiRsPath(crateDir) {
+    return path.join(crateDir, RELATIVE_API);
+}
+/** Function paths (`messages:list`) present in an `api.rs`. */
+export function functionPathsInApiRs(contents) {
+    const out = new Set();
+    for (const m of contents.matchAll(/FunctionRef::new\("([^"]+)"\)/g))
+        out.add(m[1]);
+    return out;
+}
+export function rustSourcesReferenceApi(crateDir) {
+    const visit = (dir) => {
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return false;
+        }
+        for (const e of entries) {
+            const p = path.join(dir, e.name);
+            if (e.isDirectory()) {
+                if (e.name !== "_generated" && visit(p))
+                    return true;
+            }
+            else if (e.name.endsWith(".rs")) {
+                try {
+                    if (/\bapi\s*::/.test(fs.readFileSync(p, "utf8")))
+                        return true;
+                }
+                catch {
+                    // ignore
+                }
+            }
+        }
+        return false;
+    };
+    return visit(path.join(crateDir, "src"));
+}
+function writeIfChanged(file, text) {
+    try {
+        if (fs.readFileSync(file, "utf8") === text)
+            return false;
+    }
+    catch {
+        // new
+    }
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.writeFileSync(file, text);
+    return true;
+}
+/**
+ * Regenerates `<crate>/src/_generated/api.rs` from the TS functions plus the Rust ones. Only
+ * runs when the file exists or the Rust sources use `api::`. A failed analysis leaves the file
+ * alone. Returns true when the file changed.
+ */
+export async function updateApiRs(o) {
+    const file = apiRsPath(o.crateDir);
+    const exists = fs.existsSync(file);
+    if (!exists && !rustSourcesReferenceApi(o.crateDir))
+        return false;
+    const res = await analyzeTsFunctions(o.projectRoot, o.functionsDir, [o.outDir], 60_000, (m) => o.log?.(m));
+    if (res === null) {
+        o.log?.("warning: could not analyze the TypeScript functions; leaving src/_generated/api.rs as is");
+        if (!exists)
+            writeIfChanged(file, RUST_API_STUB);
+        return false;
+    }
+    for (const w of res.warnings)
+        o.log?.(`warning: ${w}`);
+    const outRel = path.relative(o.functionsDir, o.outDir).split(path.sep).join("/");
+    const modules = { ...res.modules, ...rustEntriesAsModules(o.rustEntries ?? [], outRel) };
+    return writeIfChanged(file, rustApiCodegen(modules));
+}
+/** Rust functions of the last build, kept in the cargo target dir so the first compile sees them. */
+export function readRustCache(cacheFile) {
+    try {
+        return JSON.parse(fs.readFileSync(cacheFile, "utf8"));
+    }
+    catch {
+        return [];
+    }
+}
+export function writeRustCache(cacheFile, entries) {
+    try {
+        fs.mkdirSync(path.dirname(cacheFile), { recursive: true });
+        fs.writeFileSync(cacheFile, JSON.stringify(entries));
+    }
+    catch {
+        // best effort
+    }
+}

package/dist/cli/scaffold.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Stable Rust is enough for the guest crate (verified with stable 1.99.1: build, run and HTTP action on a
+ * stock backend); nothing in convex_fn or the macros needs nightly features. Override with CONVEX_RUST_CHANNEL.
+ */
+export declare const DEFAULT_RUST_CHANNEL = "stable";
+export interface ScaffoldOptions {
+    /** Path to the `convex_fn` crate (env CONVEX_FN_PATH). Unset: registry placeholder. */
+    convexFnPath?: string | undefined;
+    /** Rust toolchain channel (env CONVEX_RUST_CHANNEL). */
+    channel?: string | undefined;
+    /** `[build] target-dir` for the generated `.cargo/config.toml`, relative to the crate. */
+    targetDir?: string | undefined;
+}
+/** Cargo target directory kept outside the Convex functions directory (relative to the crate). */
+export declare function defaultTargetDir(projectRoot: string, crateDir: string): string;
+export declare function scaffoldFiles(rustDir: string, opts?: ScaffoldOptions): Record<string, string>;
+export interface ScaffoldResult {
+    created: string[];
+    skipped: string[];
+}
+/** Writes the scaffold; never overwrites existing files. */
+export declare function scaffoldRust(rustDir: string, opts?: ScaffoldOptions): ScaffoldResult;