convex-rust 0.0.0-stage → 0.1.1

package/dist/cli/wasisdk.js

@@ -0,0 +1,220 @@
+/**
+ * wasi-sdk discovery / install, so crates that compile C (zstd-sys, libsqlite3-sys, ...) build for
+ * wasm32-wasip1. cc-rs reads `CC_wasm32_wasip1`, `AR_wasm32_wasip1` and `CFLAGS_wasm32_wasip1`
+ * (https://docs.rs/cc/latest/cc/#external-configuration-via-environment-variables).
+ */
+import path from "node:path";
+import fs from "node:fs";
+import os from "node:os";
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { RUST_TARGET } from "./toolchain.js";
+export const WASI_SDK_VERSION = "34.0";
+const RELEASE_BASE = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-34";
+/** Pinned release archives (sha256 as published by GitHub for the release assets). */
+export const WASI_SDK_ASSETS = {
+    "darwin-arm64": {
+        file: "wasi-sdk-34.0-arm64-macos.tar.gz",
+        sha256: "9c59398106b417f8f14913380fdf0097a8cc0ff4af9eb3ce0065a859e88d49e9",
+        size: 180322002,
+    },
+    "darwin-x64": {
+        file: "wasi-sdk-34.0-x86_64-macos.tar.gz",
+        sha256: "87d27fa8adc68dee59bfbf2e22a6d34ef717c34d6bf1d8af2a56fc929d9ce0eb",
+        size: 182668326,
+    },
+    "linux-arm64": {
+        file: "wasi-sdk-34.0-arm64-linux.tar.gz",
+        sha256: "f7e243dff54d60bcc576e94d6166b69f410f2500ae4a9ceef34315be10e77971",
+        size: 192255247,
+    },
+    "linux-x64": {
+        file: "wasi-sdk-34.0-x86_64-linux.tar.gz",
+        sha256: "b761e3a0721dbae9c09a0059e5fdb2bf917d1b4a8a7b430fb3b5aafb0984b2c4",
+        size: 192383077,
+    },
+    "win32-x64": {
+        file: "wasi-sdk-34.0-x86_64-windows.tar.gz",
+        sha256: "cccb5c323a9b34f0349a9b09e8804a0a7632c68c3310f4b5f437ed57d7e71d8f",
+        size: 619003408,
+    },
+    "win32-arm64": {
+        file: "wasi-sdk-34.0-arm64-windows.tar.gz",
+        sha256: "45e1c71f3e965621e7b98ebe1d37b0e4b1f77f3e8072113ffb4534e67b1a4b7c",
+        size: 620307231,
+    },
+};
+/** `~/.cache/convex-rust` (override: `CONVEX_RUST_CACHE_DIR`; honours `XDG_CACHE_HOME`). */
+export function cacheRoot(env = process.env) {
+    if (env.CONVEX_RUST_CACHE_DIR)
+        return env.CONVEX_RUST_CACHE_DIR;
+    const base = env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
+    return path.join(base, "convex-rust");
+}
+export function cachedSdkDir(env = process.env) {
+    return path.join(cacheRoot(env), `wasi-sdk-${WASI_SDK_VERSION}`);
+}
+const exe = process.platform === "win32" ? ".exe" : "";
+/** True when `dir` looks like a wasi-sdk (clang + sysroot). */
+export function isWasiSdk(dir) {
+    return (fs.existsSync(path.join(dir, "bin", `clang${exe}`)) &&
+        fs.existsSync(path.join(dir, "share", "wasi-sysroot")));
+}
+/** `WASI_SDK_PATH` first, then the cache dir. Null when neither holds a usable SDK. */
+export function findWasiSdk(env = process.env) {
+    if (env.WASI_SDK_PATH && isWasiSdk(env.WASI_SDK_PATH))
+        return { dir: env.WASI_SDK_PATH, source: "env" };
+    const c = cachedSdkDir(env);
+    if (isWasiSdk(c))
+        return { dir: c, source: "cache" };
+    return null;
+}
+/**
+ * Environment that points cc-rs at the SDK. Variables the user already set are left alone.
+ * `clang` is used as `ar` fallback only when `llvm-ar` is missing.
+ */
+export function wasiSdkEnv(sdkDir, env = process.env) {
+    const t = RUST_TARGET.replace(/-/g, "_");
+    const out = {};
+    const bin = (n) => path.join(sdkDir, "bin", n + exe);
+    const sysroot = path.join(sdkDir, "share", "wasi-sysroot");
+    const set = (k, v) => {
+        if (!env[k] && !env[k.replace(t, RUST_TARGET)])
+            out[k] = v;
+    };
+    set(`CC_${t}`, bin("clang"));
+    set(`CXX_${t}`, bin("clang++"));
+    set(`AR_${t}`, bin("llvm-ar"));
+    set(`CFLAGS_${t}`, `--sysroot=${sysroot}`);
+    set(`CXXFLAGS_${t}`, `--sysroot=${sysroot}`);
+    return out;
+}
+/**
+ * Crates in the resolved (target-filtered) graph whose build script uses a C compiler driver
+ * (`cc`, `cmake`, `bindgen` is not counted). Their names, sorted.
+ */
+export function crateNamesWithCBuild(meta) {
+    const name = new Map(meta.packages.map((p) => [p.id, p.name]));
+    const out = new Set();
+    for (const node of meta.resolve?.nodes ?? []) {
+        for (const d of node.deps) {
+            const dep = name.get(d.pkg);
+            if (dep !== "cc" && dep !== "cmake")
+                continue;
+            if (!(d.dep_kinds ?? []).some((k) => k.kind === "build"))
+                continue;
+            const n = name.get(node.id);
+            if (n)
+                out.add(n);
+        }
+    }
+    return [...out].sort();
+}
+export function assetForPlatform(platform = process.platform, arch = process.arch) {
+    return WASI_SDK_ASSETS[`${platform}-${arch}`] ?? null;
+}
+async function sha256File(file) {
+    const h = createHash("sha256");
+    await pipeline(fs.createReadStream(file), h);
+    return h.digest("hex");
+}
+/**
+ * Downloads the pinned wasi-sdk into the cache dir (sha256 checked) and returns its directory.
+ * An archive already present in `<cache>/dl` with the right hash is reused.
+ */
+export async function installWasiSdk(opts = {}) {
+    const env = opts.env ?? process.env;
+    const log = opts.log ?? ((m) => console.error(m));
+    const dest = cachedSdkDir(env);
+    if (isWasiSdk(dest))
+        return dest;
+    const asset = assetForPlatform();
+    if (!asset) {
+        throw new Error(`No pinned wasi-sdk build for ${process.platform}/${process.arch}. Download one from https://github.com/WebAssembly/wasi-sdk/releases and set WASI_SDK_PATH.`);
+    }
+    const dlDir = path.join(cacheRoot(env), "dl");
+    fs.mkdirSync(dlDir, { recursive: true });
+    const archive = path.join(dlDir, asset.file);
+    let have = fs.existsSync(archive) && (await sha256File(archive)) === asset.sha256;
+    if (!have) {
+        const url = `${RELEASE_BASE}/${asset.file}`;
+        log(`Downloading wasi-sdk ${WASI_SDK_VERSION} (${(asset.size / 1024 / 1024).toFixed(0)} MB) from ${url} ...`);
+        const res = await (opts.fetchImpl ?? fetch)(url);
+        if (!res.ok || !res.body)
+            throw new Error(`download failed: HTTP ${res.status} for ${url}`);
+        const tmp = `${archive}.${process.pid}.part`;
+        try {
+            await pipeline(Readable.fromWeb(res.body), fs.createWriteStream(tmp));
+            const got = await sha256File(tmp);
+            if (got !== asset.sha256)
+                throw new Error(`checksum mismatch for ${asset.file}: expected ${asset.sha256}, got ${got}`);
+            fs.renameSync(tmp, archive);
+        }
+        finally {
+            fs.rmSync(tmp, { force: true });
+        }
+        have = true;
+    }
+    const stage = fs.mkdtempSync(path.join(cacheRoot(env), ".wasi-sdk-"));
+    try {
+        log(`Extracting wasi-sdk to ${dest} ...`);
+        const r = spawnSync("tar", ["-xzf", archive, "-C", stage], { encoding: "utf8" });
+        if (r.status !== 0)
+            throw new Error(`tar failed: ${r.stderr || r.error?.message || r.status}`);
+        const top = fs.readdirSync(stage).find((n) => n.startsWith("wasi-sdk"));
+        if (!top || !isWasiSdk(path.join(stage, top)))
+            throw new Error("the archive does not contain a wasi-sdk");
+        fs.rmSync(dest, { recursive: true, force: true });
+        fs.renameSync(path.join(stage, top), dest);
+    }
+    finally {
+        fs.rmSync(stage, { recursive: true, force: true });
+    }
+    fs.rmSync(archive, { force: true });
+    return dest;
+}
+/** `--wasi-sdk` / `--no-wasi-sdk` / `CONVEX_RUST_WASI_SDK=1|0`. Default: auto (download only if a C build is detected). */
+export function wasiSdkMode(flags, env = process.env) {
+    if (flags.noWasiSdk || env.CONVEX_RUST_WASI_SDK === "0")
+        return "never";
+    if (flags.wasiSdk || env.CONVEX_RUST_WASI_SDK === "1")
+        return "always";
+    return "auto";
+}
+/**
+ * Decides the cargo environment for C compilation. Uses an SDK that is already there (always, it is free);
+ * downloads one only when `mode` is "always", or "auto" and `cNames` (crates with a C build script) is non-empty.
+ */
+export async function prepareWasiSdk(args) {
+    const env = args.env ?? process.env;
+    if (args.mode === "never")
+        return {};
+    let sdk = findWasiSdk(env);
+    if (!sdk) {
+        if (args.mode === "auto") {
+            let names = [];
+            try {
+                names = await args.cNames();
+            }
+            catch {
+                // detection is best effort
+            }
+            if (names.length === 0)
+                return {};
+            args.log(`C build scripts detected (${names.join(", ")}) and no wasi-sdk found; installing wasi-sdk ${WASI_SDK_VERSION} (disable with --no-wasi-sdk).`);
+        }
+        try {
+            sdk = { dir: await (args.install ?? installWasiSdk)({ env, log: args.log }), source: "cache" };
+        }
+        catch (e) {
+            args.log(`warning: could not install wasi-sdk: ${e?.message ?? e}`);
+            return {};
+        }
+    }
+    const out = wasiSdkEnv(sdk.dir, env);
+    if (Object.keys(out).length > 0)
+        args.log(`using wasi-sdk at ${sdk.dir} for C dependencies`);
+    return out;
+}

package/dist/cli/wasmHints.d.ts

@@ -0,0 +1,29 @@
+export interface WasmImport {
+    module: string;
+    name: string;
+    kind: "func" | "table" | "memory" | "global" | "tag";
+    /** memory/table imports: limits flags (bit 1 shared, bit 2 64-bit). */
+    flags?: number;
+}
+export interface WasmMemory {
+    flags: number;
+    imported: boolean;
+}
+export interface WasmInfo {
+    imports: WasmImport[];
+    memories: WasmMemory[];
+    exports: string[];
+}
+export declare function scanWasm(wasm: Uint8Array): WasmInfo;
+export interface Hint {
+    id: string;
+    title: string;
+    /** One-line fix. */
+    fix: string;
+}
+/** WASI functions the host implements (everything else answers ENOSYS), read from the runtime itself. */
+export declare function hostWasiFunctions(): Set<string>;
+/** Functions of the `convex` import module the host provides (src/runtime/host.ts). */
+export declare const HOST_CONVEX_IMPORTS: Set<string>;
+export declare function wasmHints(wasm: Uint8Array, implemented?: Set<string>): Hint[];
+export declare function formatWasmHints(hints: Hint[], label?: string): string[];

package/dist/cli/wasmHints.js

@@ -0,0 +1,172 @@
+/**
+ * Build-time scan of the compiled wasm's imports and memories for things that fail (or silently
+ * answer ENOSYS) on the Convex host. Only reads the binary format; nothing is instantiated.
+ */
+import { readLeb } from "./size.js";
+import { makeWasi, LineBuffer } from "../runtime/wasi.js";
+import { TmpFs } from "../runtime/tmpfs.js";
+const KINDS = ["func", "table", "memory", "global", "tag"];
+const dec = new TextDecoder();
+function readName(b, pos) {
+    let len;
+    [len, pos] = readLeb(b, pos);
+    return [dec.decode(b.subarray(pos, pos + len)), pos + len];
+}
+/** Reads a `limits` entry; returns [flags, newPos]. */
+function readLimits(b, pos) {
+    let flags;
+    [flags, pos] = readLeb(b, pos);
+    [, pos] = readLeb(b, pos);
+    if (flags & 1)
+        [, pos] = readLeb(b, pos);
+    return [flags, pos];
+}
+export function scanWasm(wasm) {
+    const info = { imports: [], memories: [], exports: [] };
+    let pos = 8;
+    while (pos < wasm.length) {
+        const id = wasm[pos++];
+        let size;
+        [size, pos] = readLeb(wasm, pos);
+        const end = pos + size;
+        let p = pos;
+        let count;
+        if (id === 2) {
+            [count, p] = readLeb(wasm, p);
+            for (let i = 0; i < count; i++) {
+                let module;
+                let name;
+                [module, p] = readName(wasm, p);
+                [name, p] = readName(wasm, p);
+                const kind = KINDS[wasm[p++]] ?? "func";
+                const imp = { module, name, kind };
+                if (kind === "func")
+                    [, p] = readLeb(wasm, p);
+                else if (kind === "tag") {
+                    p++;
+                    [, p] = readLeb(wasm, p);
+                }
+                else if (kind === "table") {
+                    p++; // reftype
+                    [imp.flags, p] = readLimits(wasm, p);
+                }
+                else if (kind === "memory") {
+                    [imp.flags, p] = readLimits(wasm, p);
+                    info.memories.push({ flags: imp.flags, imported: true });
+                }
+                else
+                    p += 2; // global: valtype, mutability
+                info.imports.push(imp);
+            }
+        }
+        else if (id === 5) {
+            [count, p] = readLeb(wasm, p);
+            for (let i = 0; i < count; i++) {
+                let flags;
+                [flags, p] = readLimits(wasm, p);
+                info.memories.push({ flags, imported: false });
+            }
+        }
+        else if (id === 7) {
+            [count, p] = readLeb(wasm, p);
+            for (let i = 0; i < count; i++) {
+                let name;
+                [name, p] = readName(wasm, p);
+                p++;
+                [, p] = readLeb(wasm, p);
+                info.exports.push(name);
+            }
+        }
+        pos = end;
+    }
+    return info;
+}
+/** WASI functions the host implements (everything else answers ENOSYS), read from the runtime itself. */
+export function hostWasiFunctions() {
+    const lb = new LineBuffer(() => { });
+    // The in-memory /tmp is part of the host's table (path_open, fd_prestat_*, fd_readdir ...).
+    const memory = () => undefined;
+    const fns = makeWasi({ memory, environ: [], line: () => { }, tmp: new TmpFs({ memory }) }, { 1: lb, 2: lb });
+    return new Set(Object.keys(fns));
+}
+/** Functions of the `convex` import module the host provides (src/runtime/host.ts). */
+export const HOST_CONVEX_IMPORTS = new Set(["task_submit", "task_poll", "sync_syscall", "log"]);
+const SOCKETS = /^sock_/;
+// Only calls that open or change paths: wasi-libc/std import fd_close, fd_fdstat_get, fd_seek ... even for stdio.
+const FS = /^(path_|fd_readdir$|fd_prestat_)/;
+export function wasmHints(wasm, implemented = hostWasiFunctions()) {
+    const info = scanWasm(wasm);
+    const hints = [];
+    const wasi = info.imports.filter((i) => i.module === "wasi_snapshot_preview1" && i.kind === "func");
+    const missing = (re) => wasi.filter((i) => re.test(i.name) && !implemented.has(i.name)).map((i) => i.name);
+    const sock = missing(SOCKETS);
+    if (sock.length) {
+        hints.push({
+            id: "wasi-sockets",
+            title: `imports ${sock.join(", ")}: raw sockets (TCP/UDP/DNS) return ENOSYS on the Convex host`,
+            fix: 'use HTTP through reqwest/ctx.fetch instead of a socket driver, or run that part in a "use node" action (ctx.run_action).',
+        });
+    }
+    const fs = missing(FS);
+    if (fs.length) {
+        hints.push({
+            id: "wasi-fs",
+            title: `imports ${fs.join(", ")}: std::fs and opening paths return ENOSYS (the host has no filesystem or VFS for them)`,
+            fix: "keep data in memory or in Convex tables/storage (ctx.storage); check dependencies that read config or temp files at runtime.",
+        });
+    }
+    if (wasi.some((i) => i.name === "proc_raise") && !implemented.has("proc_raise")) {
+        hints.push({
+            id: "wasi-proc-raise",
+            title: "imports proc_raise: raise()/abort-by-signal returns ENOSYS and the call keeps running",
+            fix: 'build with `panic = "abort"` and avoid C code that calls abort()/raise(); report errors as Result instead.',
+        });
+    }
+    const threads = info.imports.filter((i) => i.module === "wasi" && /thread/.test(i.name));
+    const shared = info.memories.some((m) => (m.flags & 2) !== 0);
+    if (threads.length || shared) {
+        hints.push({
+            id: "threads",
+            title: shared
+                ? "uses shared memory (threads / atomics build): the host cannot start threads"
+                : `imports ${threads.map((t) => `${t.module}.${t.name}`).join(", ")}: the host cannot start threads`,
+            fix: 'remove `-C target-feature=+atomics` / wasi-threads settings and replace rayon/std::thread::spawn with sequential code, or use a "use node" action.',
+        });
+    }
+    if (info.memories.some((m) => (m.flags & 4) !== 0)) {
+        hints.push({
+            id: "memory64",
+            title: "uses a 64-bit memory (memory64), which the Convex runtime does not support",
+            fix: "build for wasm32-wasip1 only (no memory64 target features).",
+        });
+    }
+    const badHost = info.imports.filter((i) => i.module === "convex" && !HOST_CONVEX_IMPORTS.has(i.name)).map((i) => i.name);
+    if (badHost.length) {
+        hints.push({
+            id: "host-imports",
+            title: `needs host import(s) convex.${badHost.join(", convex.")} that the installed convex-rust runtime does not provide`,
+            fix: "the convex_fn crate is newer than the npm package: update convex-rust (npm i convex-rust@latest) or pin the crate to the matching version.",
+        });
+    }
+    const foreign = info.imports.filter((i) => i.kind !== "memory" && i.module !== "convex" && i.module !== "wasi_snapshot_preview1" && i.module !== "wasi");
+    if (foreign.length) {
+        const names = foreign.slice(0, 6).map((i) => `${i.module}.${i.name}`);
+        hints.push({
+            id: "foreign-imports",
+            title: `imports ${names.join(", ")}${foreign.length > names.length ? ` (+${foreign.length - names.length} more)` : ""}, which the host does not provide: the module will fail to instantiate`,
+            fix: "these come from wasm-bindgen / emscripten / wasm32-unknown-unknown style crates: build only for wasm32-wasip1 and drop the dependency or its `js`/`wasm-bindgen` feature (`cargo tree -i <crate> --target wasm32-wasip1`).",
+        });
+    }
+    if (info.imports.some((i) => i.kind === "memory")) {
+        hints.push({
+            id: "imported-memory",
+            title: "imports its linear memory (env.memory), which the host does not provide",
+            fix: "do not pass `--import-memory` to the linker (check RUSTFLAGS / .cargo/config.toml).",
+        });
+    }
+    return hints;
+}
+export function formatWasmHints(hints, label = "") {
+    const l = label.trim().replace(/^\[|\]$/g, "");
+    return hints.map((h) => `warning:${l ? ` [${l}]` : ""} the wasm ${h.title}.\n  fix: ${h.fix}`);
+}

package/dist/runtime/base64.d.ts

@@ -0,0 +1,4 @@
+export declare function encodeBase64(bytes: Uint8Array): string;
+export declare function encodeBase64Js(bytes: Uint8Array): string;
+export declare function decodeBase64(text: string): Uint8Array;
+export declare function decodeBase64Js(text: string): Uint8Array;

package/dist/runtime/base64.js

@@ -0,0 +1,77 @@
+// Dependency-free base64 (the default Convex runtime has no Buffer).
+const ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+const LOOKUP = new Int16Array(128).fill(-1);
+for (let i = 0; i < ALPHABET.length; i++)
+    LOOKUP[ALPHABET.charCodeAt(i)] = i;
+// V8's built-in Uint8Array base64 (ES2026) when present; the JS code below is the fallback.
+const U8 = typeof Uint8Array !== "undefined" ? Uint8Array : undefined;
+const HAS_TO_BASE64 = typeof U8?.prototype?.toBase64 === "function";
+const HAS_FROM_BASE64 = typeof U8?.fromBase64 === "function";
+export function encodeBase64(bytes) {
+    if (HAS_TO_BASE64)
+        return bytes.toBase64();
+    return encodeBase64Js(bytes);
+}
+export function encodeBase64Js(bytes) {
+    const parts = [];
+    const CHUNK = 3 * 8192;
+    for (let start = 0; start < bytes.length; start += CHUNK) {
+        const end = Math.min(start + CHUNK, bytes.length);
+        let out = "";
+        let i = start;
+        for (; i + 2 < end; i += 3) {
+            const n = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2];
+            out +=
+                ALPHABET[(n >> 18) & 63] +
+                    ALPHABET[(n >> 12) & 63] +
+                    ALPHABET[(n >> 6) & 63] +
+                    ALPHABET[n & 63];
+        }
+        if (end - i === 1) {
+            const n = bytes[i] << 16;
+            out += ALPHABET[(n >> 18) & 63] + ALPHABET[(n >> 12) & 63] + "==";
+        }
+        else if (end - i === 2) {
+            const n = (bytes[i] << 16) | (bytes[i + 1] << 8);
+            out +=
+                ALPHABET[(n >> 18) & 63] +
+                    ALPHABET[(n >> 12) & 63] +
+                    ALPHABET[(n >> 6) & 63] +
+                    "=";
+        }
+        parts.push(out);
+    }
+    return parts.join("");
+}
+export function decodeBase64(text) {
+    if (HAS_FROM_BASE64) {
+        try {
+            // Same leniency as the fallback: ASCII whitespace and trailing padding are ignored.
+            return U8.fromBase64(text.replace(/[\s=]+$/g, ""), { lastChunkHandling: "loose" });
+        }
+        catch {
+            // fall through: the JS path reports the offending character
+        }
+    }
+    return decodeBase64Js(text);
+}
+export function decodeBase64Js(text) {
+    const s = text.replace(/[\s=]+$/g, "");
+    const out = new Uint8Array(Math.floor((s.length * 3) / 4));
+    let acc = 0;
+    let bits = 0;
+    let o = 0;
+    for (let i = 0; i < s.length; i++) {
+        const c = s.charCodeAt(i);
+        const v = c < 128 ? LOOKUP[c] : -1;
+        if (v < 0)
+            throw new Error(`invalid base64 character at ${i}`);
+        acc = (acc << 6) | v;
+        bits += 6;
+        if (bits >= 8) {
+            bits -= 8;
+            out[o++] = (acc >> bits) & 0xff;
+        }
+    }
+    return out.subarray(0, o);
+}

package/dist/runtime/crypto.d.ts

@@ -0,0 +1,7 @@
+type Json = any;
+export declare const CRYPTO_VERSION = 2;
+type Mem = () => WebAssembly.Memory;
+/** Serves one `crypto/*` sync op. `mem` returns the guest's linear memory (for `{ptr, len}` byte arguments). */
+/** Answers synchronously (no suspend) for small SHA-256 digest / HMAC, else with a promise. */
+export declare function cryptoOp(op: string, args: Json, mem: Mem): Json | Promise<Json>;
+export {};