convex-rust 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +137 -2
- package/dist/cli/bin.d.ts +2 -0
- package/dist/cli/bin.js +200 -0
- package/dist/cli/build.d.ts +146 -0
- package/dist/cli/build.js +902 -0
- package/dist/cli/demangle.d.ts +7 -0
- package/dist/cli/demangle.js +328 -0
- package/dist/cli/diagnostics.d.ts +34 -0
- package/dist/cli/diagnostics.js +296 -0
- package/dist/cli/generate.d.ts +29 -0
- package/dist/cli/generate.js +82 -0
- package/dist/cli/manifest.d.ts +24 -0
- package/dist/cli/manifest.js +75 -0
- package/dist/cli/rustApi.d.ts +78 -0
- package/dist/cli/rustApi.js +609 -0
- package/dist/cli/scaffold.d.ts +22 -0
- package/dist/cli/scaffold.js +114 -0
- package/dist/cli/shim.d.ts +30 -0
- package/dist/cli/shim.js +140 -0
- package/dist/cli/size.d.ts +17 -0
- package/dist/cli/size.js +73 -0
- package/dist/cli/split.d.ts +20 -0
- package/dist/cli/split.js +140 -0
- package/dist/cli/symbols.d.ts +30 -0
- package/dist/cli/symbols.js +133 -0
- package/dist/cli/toolchain.d.ts +53 -0
- package/dist/cli/toolchain.js +167 -0
- package/dist/cli/validators.d.ts +6 -0
- package/dist/cli/validators.js +66 -0
- package/dist/cli/wasisdk.d.ts +82 -0
- package/dist/cli/wasisdk.js +220 -0
- package/dist/cli/wasmHints.d.ts +29 -0
- package/dist/cli/wasmHints.js +172 -0
- package/dist/runtime/base64.d.ts +4 -0
- package/dist/runtime/base64.js +77 -0
- package/dist/runtime/crypto.d.ts +7 -0
- package/dist/runtime/crypto.js +421 -0
- package/dist/runtime/fastjson.d.ts +3 -0
- package/dist/runtime/fastjson.js +170 -0
- package/dist/runtime/host.d.ts +73 -0
- package/dist/runtime/host.js +1230 -0
- package/dist/runtime/http.d.ts +40 -0
- package/dist/runtime/http.js +157 -0
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.js +1 -0
- package/dist/runtime/intl.d.ts +5 -0
- package/dist/runtime/intl.js +69 -0
- package/dist/runtime/tmpfs.d.ts +129 -0
- package/dist/runtime/tmpfs.js +624 -0
- package/dist/runtime/wasi.d.ts +33 -0
- package/dist/runtime/wasi.js +216 -0
- package/package.json +57 -4
- package/rust/convex_fn/Cargo.toml +32 -0
- package/rust/convex_fn/LICENSE +201 -0
- package/rust/convex_fn/build.rs +3 -0
- package/rust/convex_fn/src/args.rs +416 -0
- package/rust/convex_fn/src/crypto.rs +237 -0
- package/rust/convex_fn/src/ctx.rs +1044 -0
- package/rust/convex_fn/src/dns.rs +356 -0
- package/rust/convex_fn/src/error.rs +179 -0
- package/rust/convex_fn/src/executor.rs +577 -0
- package/rust/convex_fn/src/fanout.rs +106 -0
- package/rust/convex_fn/src/fs.rs +19 -0
- package/rust/convex_fn/src/function_ref.rs +219 -0
- package/rust/convex_fn/src/http.rs +441 -0
- package/rust/convex_fn/src/intl.rs +75 -0
- package/rust/convex_fn/src/lib.rs +242 -0
- package/rust/convex_fn/src/logging.rs +222 -0
- package/rust/convex_fn/src/native.rs +360 -0
- package/rust/convex_fn/src/net.rs +122 -0
- package/rust/convex_fn/src/sys.rs +178 -0
- package/rust/convex_fn/src/values.rs +121 -0
- package/rust/convex_fn/src/wire.rs +579 -0
- package/rust/convex_fn_macros/Cargo.toml +13 -0
- package/rust/convex_fn_macros/LICENSE +201 -0
- package/rust/convex_fn_macros/src/derive.rs +632 -0
- package/rust/convex_fn_macros/src/lib.rs +338 -0
- package/rust/convex_reqwest/Cargo.toml +69 -0
- package/rust/convex_reqwest/LICENSE +201 -0
- package/rust/convex_reqwest/src/blocking.rs +287 -0
- package/rust/convex_reqwest/src/body.rs +152 -0
- package/rust/convex_reqwest/src/client.rs +359 -0
- package/rust/convex_reqwest/src/config_types.rs +150 -0
- package/rust/convex_reqwest/src/error.rs +241 -0
- package/rust/convex_reqwest/src/into_url.rs +57 -0
- package/rust/convex_reqwest/src/lib.rs +42 -0
- package/rust/convex_reqwest/src/multipart.rs +140 -0
- package/rust/convex_reqwest/src/request.rs +358 -0
- package/rust/convex_reqwest/src/response.rs +310 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { RUST_TARGET } from "./toolchain.js";
|
|
4
|
+
import { RUST_API_STUB } from "./rustApi.js";
|
|
5
|
+
import { managedConfigToml, resolveConvexFnPath } from "./shim.js";
|
|
6
|
+
/**
|
|
7
|
+
* Stable Rust is enough for the guest crate (verified with stable 1.99.1: build, run and HTTP action on a
|
|
8
|
+
* stock backend); nothing in convex_fn or the macros needs nightly features. Override with CONVEX_RUST_CHANNEL.
|
|
9
|
+
*/
|
|
10
|
+
export const DEFAULT_RUST_CHANNEL = "stable";
|
|
11
|
+
/** Cargo target directory kept outside the Convex functions directory (relative to the crate). */
|
|
12
|
+
export function defaultTargetDir(projectRoot, crateDir) {
|
|
13
|
+
const rel = path.relative(path.resolve(crateDir), path.join(path.resolve(projectRoot), ".convex-rust", "target"));
|
|
14
|
+
return rel.split(path.sep).join("/");
|
|
15
|
+
}
|
|
16
|
+
function convexFnDependency(rustDir, convexFnPath) {
|
|
17
|
+
if (convexFnPath) {
|
|
18
|
+
let p = path.resolve(convexFnPath);
|
|
19
|
+
const rel = path.relative(path.resolve(rustDir), p);
|
|
20
|
+
if (!path.isAbsolute(rel))
|
|
21
|
+
p = rel;
|
|
22
|
+
return `convex_fn = { path = ${JSON.stringify(p.split(path.sep).join("/"))} }`;
|
|
23
|
+
}
|
|
24
|
+
return [
|
|
25
|
+
"# TODO: `convex_fn` is not published yet and was not found in node_modules/convex-rust/rust.",
|
|
26
|
+
"# Set CONVEX_FN_PATH (or pass --convex-fn-path) when running `convex-rust init`, or point this at",
|
|
27
|
+
"# a path/git dependency.",
|
|
28
|
+
'convex_fn = "0.1"',
|
|
29
|
+
].join("\n");
|
|
30
|
+
}
|
|
31
|
+
export function scaffoldFiles(rustDir, opts = {}) {
|
|
32
|
+
const channel = opts.channel ?? DEFAULT_RUST_CHANNEL;
|
|
33
|
+
return {
|
|
34
|
+
"Cargo.toml": `[package]
|
|
35
|
+
name = "convex_rust"
|
|
36
|
+
edition = "2021"
|
|
37
|
+
version = "0.1.0"
|
|
38
|
+
publish = false
|
|
39
|
+
|
|
40
|
+
[lib]
|
|
41
|
+
crate-type = ["cdylib"]
|
|
42
|
+
|
|
43
|
+
[dependencies]
|
|
44
|
+
${convexFnDependency(rustDir, opts.convexFnPath ?? resolveConvexFnPath(rustDir))}
|
|
45
|
+
serde = { version = "1", features = ["derive"] }
|
|
46
|
+
serde_json = "1"
|
|
47
|
+
|
|
48
|
+
# One wasm per module (smaller cold load on Convex cloud): gate each module with a cargo feature named
|
|
49
|
+
# m-<module> and convex-rust build builds one wasm per module. Heavy dependencies go behind it:
|
|
50
|
+
# [features]
|
|
51
|
+
# default = ["m-example", "m-images"] # default = all, so cargo check sees everything
|
|
52
|
+
# m-example = []
|
|
53
|
+
# m-images = ["dep:image"] # with image = { version = "0.25", optional = true }
|
|
54
|
+
# and in src/lib.rs: #[cfg(feature = "m-images")] pub mod images;
|
|
55
|
+
# See docs/rust/cloud.md.
|
|
56
|
+
|
|
57
|
+
# Keeps this crate standalone even when the project root is a Cargo workspace.
|
|
58
|
+
[workspace]
|
|
59
|
+
`,
|
|
60
|
+
"src/lib.rs": `// Each module in this crate becomes a Convex module under \`rust/\`:
|
|
61
|
+
// \`src/example.rs\` is called as \`api.rust.example.greet\`.
|
|
62
|
+
pub mod example;
|
|
63
|
+
|
|
64
|
+
// Typed references to your Convex functions, generated by \`convex-rust build\` / \`convex-rust codegen\`.
|
|
65
|
+
#[path = "_generated/api.rs"]
|
|
66
|
+
pub mod api;
|
|
67
|
+
`,
|
|
68
|
+
"src/_generated/api.rs": RUST_API_STUB,
|
|
69
|
+
"src/example.rs": `use convex_fn::{action, ConvexArgs, ConvexValue, Ctx, Result};
|
|
70
|
+
use serde::{Deserialize, Serialize};
|
|
71
|
+
|
|
72
|
+
#[derive(Deserialize, ConvexArgs)]
|
|
73
|
+
pub struct GreetArgs {
|
|
74
|
+
pub name: String,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
#[derive(Serialize, ConvexValue)]
|
|
78
|
+
pub struct Greeting {
|
|
79
|
+
pub text: String,
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/// An example action. Call it as \`api.rust.example.greet({ name: "you" })\`.
|
|
83
|
+
/// \`ctx.run_query("module:name", args)\` calls your TypeScript functions.
|
|
84
|
+
#[action]
|
|
85
|
+
pub async fn greet(_ctx: &Ctx, args: GreetArgs) -> Result<Greeting> {
|
|
86
|
+
Ok(Greeting { text: format!("Hello, {}!", args.name) })
|
|
87
|
+
}
|
|
88
|
+
`,
|
|
89
|
+
"rust-toolchain.toml": `[toolchain]
|
|
90
|
+
channel = ${JSON.stringify(channel)}
|
|
91
|
+
targets = ["${RUST_TARGET}"]
|
|
92
|
+
components = ["rustfmt", "clippy"]
|
|
93
|
+
`,
|
|
94
|
+
".cargo/config.toml": managedConfigToml({ targetDir: opts.targetDir }),
|
|
95
|
+
// Build outputs. The generated wrappers (*.ts) are small and fine to commit; the wasm is rebuilt by
|
|
96
|
+
// \`convex-rust build\`, so CI must run that before \`convex deploy\`.
|
|
97
|
+
".gitignore": "# cargo output (only used when target-dir in .cargo/config.toml is removed)\ntarget/\n# compiled wasm, regenerated by `convex-rust build`\n_wasm/\n",
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
/** Writes the scaffold; never overwrites existing files. */
|
|
101
|
+
export function scaffoldRust(rustDir, opts = {}) {
|
|
102
|
+
const result = { created: [], skipped: [] };
|
|
103
|
+
for (const [rel, contents] of Object.entries(scaffoldFiles(rustDir, opts))) {
|
|
104
|
+
const full = path.join(rustDir, rel);
|
|
105
|
+
if (fs.existsSync(full)) {
|
|
106
|
+
result.skipped.push(rel);
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
fs.mkdirSync(path.dirname(full), { recursive: true });
|
|
110
|
+
fs.writeFileSync(full, contents);
|
|
111
|
+
result.created.push(rel);
|
|
112
|
+
}
|
|
113
|
+
return result;
|
|
114
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export declare const MANAGED_BEGIN = "# >>> convex-managed:patch (begin)";
|
|
2
|
+
export declare const MANAGED_END = "# <<< convex-managed:patch (end)";
|
|
3
|
+
/**
|
|
4
|
+
* Where the reqwest shim lives: `CONVEX_REQWEST_PATH`, else `node_modules/convex-rust/rust/convex_reqwest`
|
|
5
|
+
* (or the fork's `node_modules/convex/rust/convex_reqwest`) above the crate, else this package's own copy.
|
|
6
|
+
*/
|
|
7
|
+
export declare function resolveReqwestShimPath(rustDir?: string, env?: Record<string, string | undefined>): string | undefined;
|
|
8
|
+
/** Where the `convex_fn` crate lives (`CONVEX_FN_PATH`, else shipped copy), if anywhere. */
|
|
9
|
+
export declare function resolveConvexFnPath(rustDir?: string, env?: Record<string, string | undefined>): string | undefined;
|
|
10
|
+
/** Portable when inside node_modules (relative), absolute otherwise. */
|
|
11
|
+
export declare function displayPath(rustDir: string, p: string): string;
|
|
12
|
+
export declare function patchBlockBody(shimPath?: string, convexFnPath?: string, withReqwest?: boolean): string;
|
|
13
|
+
/** Absolute path of the user's `convex_fn` path dependency in `<rustDir>/Cargo.toml`. */
|
|
14
|
+
export declare function userConvexFnPath(rustDir: string): string | undefined;
|
|
15
|
+
export interface ConfigOptions {
|
|
16
|
+
shimPath?: string | undefined;
|
|
17
|
+
convexFnPath?: string | undefined;
|
|
18
|
+
withReqwest?: boolean;
|
|
19
|
+
/** `[build] target-dir`, relative to the crate directory (the parent of `.cargo`). */
|
|
20
|
+
targetDir?: string | undefined;
|
|
21
|
+
}
|
|
22
|
+
export declare function managedConfigToml(o?: ConfigOptions): string;
|
|
23
|
+
/** Does the crate depend on reqwest (directly or transitively)? */
|
|
24
|
+
export declare function usesReqwest(rustDir: string): boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Rewrites the managed block of `<rustDir>/.cargo/config.toml`; everything outside the
|
|
27
|
+
* markers is left alone. Creates the file when missing (and `Cargo.toml` exists); a file
|
|
28
|
+
* without markers is left alone. Returns true if the file changed.
|
|
29
|
+
*/
|
|
30
|
+
export declare function syncManagedPatch(rustDir: string, shimPath?: string | undefined, withReqwest?: boolean): boolean;
|
package/dist/cli/shim.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// Managed `.cargo/config.toml` patch block: swaps `reqwest` for the built-in shim
|
|
2
|
+
// (crates/convex_reqwest), which sends requests through the host fetch. Port of the fork
|
|
3
|
+
// CLI's lib/rust/scaffold.ts.
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
export const MANAGED_BEGIN = "# >>> convex-managed:patch (begin)";
|
|
7
|
+
export const MANAGED_END = "# <<< convex-managed:patch (end)";
|
|
8
|
+
const toToml = (x) => JSON.stringify(x.split(path.sep).join("/"));
|
|
9
|
+
/** `node_modules/<pkg>/rust/<crate>` found by walking up from `fromDir`. */
|
|
10
|
+
function findInNodeModules(fromDir, pkg, crate) {
|
|
11
|
+
let dir = path.resolve(fromDir);
|
|
12
|
+
for (;;) {
|
|
13
|
+
const p = path.join(dir, "node_modules", pkg, "rust", crate);
|
|
14
|
+
if (fs.existsSync(path.join(p, "Cargo.toml")))
|
|
15
|
+
return p;
|
|
16
|
+
const up = path.dirname(dir);
|
|
17
|
+
if (up === dir)
|
|
18
|
+
return undefined;
|
|
19
|
+
dir = up;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/** The package's own copy (`<package>/rust/<crate>`, created by `npm run vendor-crates`). */
|
|
23
|
+
function ownCopy(crate) {
|
|
24
|
+
const here = path.dirname(new URL(import.meta.url).pathname);
|
|
25
|
+
const p = path.resolve(here, "..", "..", "rust", crate);
|
|
26
|
+
return fs.existsSync(path.join(p, "Cargo.toml")) ? p : undefined;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Where the reqwest shim lives: `CONVEX_REQWEST_PATH`, else `node_modules/convex-rust/rust/convex_reqwest`
|
|
30
|
+
* (or the fork's `node_modules/convex/rust/convex_reqwest`) above the crate, else this package's own copy.
|
|
31
|
+
*/
|
|
32
|
+
export function resolveReqwestShimPath(rustDir, env = process.env) {
|
|
33
|
+
if (env.CONVEX_REQWEST_PATH)
|
|
34
|
+
return path.resolve(env.CONVEX_REQWEST_PATH);
|
|
35
|
+
if (rustDir) {
|
|
36
|
+
for (const pkg of ["convex-rust", "convex"]) {
|
|
37
|
+
const p = findInNodeModules(rustDir, pkg, "convex_reqwest");
|
|
38
|
+
if (p)
|
|
39
|
+
return p;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return ownCopy("convex_reqwest");
|
|
43
|
+
}
|
|
44
|
+
/** Where the `convex_fn` crate lives (`CONVEX_FN_PATH`, else shipped copy), if anywhere. */
|
|
45
|
+
export function resolveConvexFnPath(rustDir, env = process.env) {
|
|
46
|
+
if (env.CONVEX_FN_PATH)
|
|
47
|
+
return path.resolve(env.CONVEX_FN_PATH);
|
|
48
|
+
if (rustDir) {
|
|
49
|
+
for (const pkg of ["convex-rust", "convex"]) {
|
|
50
|
+
const p = findInNodeModules(rustDir, pkg, "convex_fn");
|
|
51
|
+
if (p)
|
|
52
|
+
return p;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return ownCopy("convex_fn");
|
|
56
|
+
}
|
|
57
|
+
/** Portable when inside node_modules (relative), absolute otherwise. */
|
|
58
|
+
export function displayPath(rustDir, p) {
|
|
59
|
+
if (!p.split(path.sep).includes("node_modules"))
|
|
60
|
+
return p;
|
|
61
|
+
const rel = path.relative(path.resolve(rustDir), p);
|
|
62
|
+
return path.isAbsolute(rel) ? p : rel;
|
|
63
|
+
}
|
|
64
|
+
export function patchBlockBody(shimPath, convexFnPath, withReqwest = true) {
|
|
65
|
+
if (!withReqwest) {
|
|
66
|
+
return "# Nothing to patch: no dependency uses reqwest. When one is added (cargo add reqwest),\n# the next `convex-rust build` fills in the built-in reqwest replacement here.\n";
|
|
67
|
+
}
|
|
68
|
+
if (!shimPath) {
|
|
69
|
+
return "# The built-in reqwest shim was not found (set CONVEX_REQWEST_PATH);\n# crates that depend on reqwest will not be able to make requests.\n";
|
|
70
|
+
}
|
|
71
|
+
// The shim depends on the registry `convex_fn`; with a path dependency, patch the
|
|
72
|
+
// registry one to the same path so both resolve to one crate.
|
|
73
|
+
const fnPatch = convexFnPath ? `convex_fn = { path = ${toToml(convexFnPath)} }\n` : "";
|
|
74
|
+
return `[patch.crates-io]\nreqwest = { path = ${toToml(shimPath)} }\n${fnPatch}`;
|
|
75
|
+
}
|
|
76
|
+
/** Absolute path of the user's `convex_fn` path dependency in `<rustDir>/Cargo.toml`. */
|
|
77
|
+
export function userConvexFnPath(rustDir) {
|
|
78
|
+
let toml;
|
|
79
|
+
try {
|
|
80
|
+
toml = fs.readFileSync(path.join(rustDir, "Cargo.toml"), "utf8");
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
const m = /^convex_fn\s*=\s*\{[^}\n]*\bpath\s*=\s*"([^"]+)"/m.exec(toml);
|
|
86
|
+
return m ? path.resolve(rustDir, m[1]) : undefined;
|
|
87
|
+
}
|
|
88
|
+
export function managedConfigToml(o = {}) {
|
|
89
|
+
const build = o.targetDir
|
|
90
|
+
? `# Keeps cargo's output (target/) out of the Convex functions directory, whose bundler\n# would otherwise walk it.\n[build]\ntarget-dir = ${toToml(o.targetDir)}\n\n`
|
|
91
|
+
: "";
|
|
92
|
+
// No `[build] target`: plain `cargo test` / `cargo check` run on the host; convex-rust passes --target.
|
|
93
|
+
return `# Managed by \`convex-rust\`: edit with care, the block below is regenerated.\n${build}${MANAGED_BEGIN}\n${patchBlockBody(o.shimPath, o.convexFnPath, o.withReqwest ?? false)}${MANAGED_END}\n`;
|
|
94
|
+
}
|
|
95
|
+
/** Does the crate depend on reqwest (directly or transitively)? */
|
|
96
|
+
export function usesReqwest(rustDir) {
|
|
97
|
+
const read = (f) => {
|
|
98
|
+
try {
|
|
99
|
+
return fs.readFileSync(path.join(rustDir, f), "utf8");
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return "";
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
if (/^\[\[package\]\]\s*\nname\s*=\s*"reqwest"\s*$/m.test(read("Cargo.lock")))
|
|
106
|
+
return true;
|
|
107
|
+
const toml = read("Cargo.toml");
|
|
108
|
+
return /^\s*reqwest\s*=/m.test(toml) || /^\s*\[[^\]]*\bdependencies\.reqwest\]/m.test(toml);
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Rewrites the managed block of `<rustDir>/.cargo/config.toml`; everything outside the
|
|
112
|
+
* markers is left alone. Creates the file when missing (and `Cargo.toml` exists); a file
|
|
113
|
+
* without markers is left alone. Returns true if the file changed.
|
|
114
|
+
*/
|
|
115
|
+
export function syncManagedPatch(rustDir, shimPath = resolveReqwestShimPath(rustDir), withReqwest = usesReqwest(rustDir)) {
|
|
116
|
+
const shim = shimPath ? displayPath(rustDir, shimPath) : undefined;
|
|
117
|
+
const rawFn = shim ? userConvexFnPath(rustDir) : undefined;
|
|
118
|
+
const fnPath = rawFn ? displayPath(rustDir, rawFn) : undefined;
|
|
119
|
+
const file = path.join(rustDir, ".cargo", "config.toml");
|
|
120
|
+
let cur;
|
|
121
|
+
try {
|
|
122
|
+
cur = fs.readFileSync(file, "utf8");
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
if (!fs.existsSync(path.join(rustDir, "Cargo.toml")))
|
|
126
|
+
return false;
|
|
127
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
128
|
+
fs.writeFileSync(file, managedConfigToml({ shimPath: shim, convexFnPath: fnPath, withReqwest }));
|
|
129
|
+
return true;
|
|
130
|
+
}
|
|
131
|
+
const b = cur.indexOf(MANAGED_BEGIN);
|
|
132
|
+
const e = cur.indexOf(MANAGED_END);
|
|
133
|
+
if (b < 0 || e < b)
|
|
134
|
+
return false;
|
|
135
|
+
const next = cur.slice(0, b) + MANAGED_BEGIN + "\n" + patchBlockBody(shim, fnPath, withReqwest) + cur.slice(e);
|
|
136
|
+
if (next === cur)
|
|
137
|
+
return false;
|
|
138
|
+
fs.writeFileSync(file, next);
|
|
139
|
+
return true;
|
|
140
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** Size of the compiled wasm and its biggest sections. */
|
|
2
|
+
export declare const WASM_SIZE_WARN_BYTES: number;
|
|
3
|
+
export interface WasmSizes {
|
|
4
|
+
total: number;
|
|
5
|
+
code: number;
|
|
6
|
+
data: number;
|
|
7
|
+
names: number;
|
|
8
|
+
other: number;
|
|
9
|
+
}
|
|
10
|
+
export declare function readLeb(bytes: Uint8Array, pos: number): [number, number];
|
|
11
|
+
/** Section sizes (payload bytes) from the binary format; no instantiation. */
|
|
12
|
+
export declare function wasmSectionSizes(wasm: Uint8Array): WasmSizes;
|
|
13
|
+
export declare function formatBytes(n: number): string;
|
|
14
|
+
export declare function describeWasmSize(wasm: Uint8Array, warnBytes?: number): {
|
|
15
|
+
message: string;
|
|
16
|
+
warning?: string;
|
|
17
|
+
};
|
package/dist/cli/size.js
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/** Size of the compiled wasm and its biggest sections. */
|
|
2
|
+
export const WASM_SIZE_WARN_BYTES = 20 * 1024 * 1024;
|
|
3
|
+
export function readLeb(bytes, pos) {
|
|
4
|
+
let result = 0;
|
|
5
|
+
let shift = 0;
|
|
6
|
+
while (pos < bytes.length) {
|
|
7
|
+
const b = bytes[pos++];
|
|
8
|
+
result += (b & 0x7f) * 2 ** shift;
|
|
9
|
+
if ((b & 0x80) === 0)
|
|
10
|
+
return [result, pos];
|
|
11
|
+
shift += 7;
|
|
12
|
+
}
|
|
13
|
+
return [result, pos]; // truncated input
|
|
14
|
+
}
|
|
15
|
+
/** Section sizes (payload bytes) from the binary format; no instantiation. */
|
|
16
|
+
export function wasmSectionSizes(wasm) {
|
|
17
|
+
const sizes = {
|
|
18
|
+
total: wasm.length,
|
|
19
|
+
code: 0,
|
|
20
|
+
data: 0,
|
|
21
|
+
names: 0,
|
|
22
|
+
other: 0,
|
|
23
|
+
};
|
|
24
|
+
let pos = 8;
|
|
25
|
+
const decoder = new TextDecoder();
|
|
26
|
+
while (pos < wasm.length) {
|
|
27
|
+
const id = wasm[pos++];
|
|
28
|
+
let size;
|
|
29
|
+
[size, pos] = readLeb(wasm, pos);
|
|
30
|
+
const end = pos + size;
|
|
31
|
+
if (id === 10)
|
|
32
|
+
sizes.code += size;
|
|
33
|
+
else if (id === 11)
|
|
34
|
+
sizes.data += size;
|
|
35
|
+
else if (id === 0) {
|
|
36
|
+
const [nameLen, namePos] = readLeb(wasm, pos);
|
|
37
|
+
const name = decoder.decode(wasm.subarray(namePos, namePos + nameLen));
|
|
38
|
+
if (name === "name")
|
|
39
|
+
sizes.names += size;
|
|
40
|
+
else
|
|
41
|
+
sizes.other += size;
|
|
42
|
+
}
|
|
43
|
+
else
|
|
44
|
+
sizes.other += size;
|
|
45
|
+
pos = end;
|
|
46
|
+
}
|
|
47
|
+
return sizes;
|
|
48
|
+
}
|
|
49
|
+
export function formatBytes(n) {
|
|
50
|
+
if (n >= 1024 * 1024)
|
|
51
|
+
return `${(n / 1024 / 1024).toFixed(1)} MB`;
|
|
52
|
+
if (n >= 1024)
|
|
53
|
+
return `${(n / 1024).toFixed(1)} KB`;
|
|
54
|
+
return `${n} B`;
|
|
55
|
+
}
|
|
56
|
+
export function describeWasmSize(wasm, warnBytes = WASM_SIZE_WARN_BYTES) {
|
|
57
|
+
const s = wasmSectionSizes(wasm);
|
|
58
|
+
const message = `Rust wasm: ${formatBytes(s.total)} ` +
|
|
59
|
+
`(code ${formatBytes(s.code)}, data ${formatBytes(s.data)}, ` +
|
|
60
|
+
`names ${formatBytes(s.names)}, other ${formatBytes(s.other)})`;
|
|
61
|
+
// Measured on the Convex cloud: the platform re-evaluates the module (and compiles its wasm) on every call.
|
|
62
|
+
const perCall = s.total >= 1024 * 1024
|
|
63
|
+
? `Loading it costs about ${Math.round((s.total / 1024 / 1024) * 0.9)} ms on every call (cloud); gate modules with m-<module> cargo features to ship one smaller wasm per module (docs/rust/cloud.md).`
|
|
64
|
+
: undefined;
|
|
65
|
+
if (s.total <= warnBytes)
|
|
66
|
+
return { message: perCall ? `${message}\n${perCall}` : message };
|
|
67
|
+
return {
|
|
68
|
+
message: perCall ? `${message}\n${perCall}` : message,
|
|
69
|
+
warning: `The Rust wasm is ${formatBytes(s.total)}, above the ${formatBytes(warnBytes)} threshold. ` +
|
|
70
|
+
`Largest section: ${s.code >= s.data ? "code" : "data"}. ` +
|
|
71
|
+
`Consider fewer/lighter dependencies, \`opt-level = "z"\`, \`lto = true\`, or installing \`wasm-opt\` (binaryen).`,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** Rewrites the export section, keeping only the exports for which `keep(name)` is true. */
|
|
2
|
+
export declare function filterExports(wasm: Uint8Array, keep: (name: string) => boolean): Uint8Array;
|
|
3
|
+
/**
|
|
4
|
+
* Whether an export stays in a module's wasm. `prefixes` are `convex_run:<module_path>:`. The match is
|
|
5
|
+
* exact: what follows the prefix is the function name (no `:`), so `convex_run:c::a::b:f` is not kept in
|
|
6
|
+
* the wasm of module `c::a`.
|
|
7
|
+
*/
|
|
8
|
+
export declare function keepsExport(exportName: string, prefixes: string[]): boolean;
|
|
9
|
+
export interface SplitOptions {
|
|
10
|
+
/** wasm-opt flags (same as the whole-module optimization). */
|
|
11
|
+
wasmOptArgs: string[];
|
|
12
|
+
cacheDir: string;
|
|
13
|
+
log?: (m: string) => void;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* One wasm per Convex module name (`cpu`, `a/b`), keyed by that name. `modulePaths` are the Rust
|
|
17
|
+
* `module_path!()` values of the functions. Returns null when `wasm-opt` is unavailable or fails
|
|
18
|
+
* (callers fall back to the single shared wasm).
|
|
19
|
+
*/
|
|
20
|
+
export declare function splitWasmByModule(wasm: Uint8Array, modulePaths: string[], opts: SplitOptions): Map<string, Uint8Array> | null;
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// Per-module wasm: one copy of the module per Convex module, with only that module's `convex_run:*`
|
|
2
|
+
// exports, so wasm-opt can drop the code (and its dependencies) the other modules need. A stock Convex
|
|
3
|
+
// deployment compiles every wasm a module imports on EVERY call (module scope is not kept between calls),
|
|
4
|
+
// at roughly 0.8 ms per MB, so a small module should not pay for the whole crate.
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import fs from "node:fs";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { spawnSync } from "node:child_process";
|
|
9
|
+
import { convexModuleName } from "./manifest.js";
|
|
10
|
+
function readLeb(b, pos) {
|
|
11
|
+
let r = 0;
|
|
12
|
+
let shift = 0;
|
|
13
|
+
while (pos < b.length) {
|
|
14
|
+
const x = b[pos++];
|
|
15
|
+
r += (x & 0x7f) * 2 ** shift;
|
|
16
|
+
if ((x & 0x80) === 0)
|
|
17
|
+
return [r, pos];
|
|
18
|
+
shift += 7;
|
|
19
|
+
}
|
|
20
|
+
throw new Error("truncated LEB128");
|
|
21
|
+
}
|
|
22
|
+
function lebBytes(n) {
|
|
23
|
+
const out = [];
|
|
24
|
+
do {
|
|
25
|
+
let b = n % 128;
|
|
26
|
+
n = Math.floor(n / 128);
|
|
27
|
+
if (n > 0)
|
|
28
|
+
b |= 0x80;
|
|
29
|
+
out.push(b);
|
|
30
|
+
} while (n > 0);
|
|
31
|
+
return out;
|
|
32
|
+
}
|
|
33
|
+
/** Rewrites the export section, keeping only the exports for which `keep(name)` is true. */
|
|
34
|
+
export function filterExports(wasm, keep) {
|
|
35
|
+
const dec = new TextDecoder();
|
|
36
|
+
let pos = 8;
|
|
37
|
+
while (pos < wasm.length) {
|
|
38
|
+
const start = pos;
|
|
39
|
+
const id = wasm[pos++];
|
|
40
|
+
let size;
|
|
41
|
+
[size, pos] = readLeb(wasm, pos);
|
|
42
|
+
const payload = pos;
|
|
43
|
+
const end = pos + size;
|
|
44
|
+
if (id === 7) {
|
|
45
|
+
let count;
|
|
46
|
+
let p;
|
|
47
|
+
[count, p] = readLeb(wasm, payload);
|
|
48
|
+
const kept = [];
|
|
49
|
+
for (let i = 0; i < count; i++) {
|
|
50
|
+
const s = p;
|
|
51
|
+
let len;
|
|
52
|
+
[len, p] = readLeb(wasm, p);
|
|
53
|
+
const name = dec.decode(wasm.subarray(p, p + len));
|
|
54
|
+
p += len + 1; // kind
|
|
55
|
+
[, p] = readLeb(wasm, p); // index
|
|
56
|
+
if (keep(name))
|
|
57
|
+
kept.push(Array.from(wasm.subarray(s, p)));
|
|
58
|
+
}
|
|
59
|
+
const body = [...lebBytes(kept.length), ...kept.flat()];
|
|
60
|
+
const out = new Uint8Array(start + 1 + lebBytes(body.length).length + body.length + (wasm.length - end));
|
|
61
|
+
out.set(wasm.subarray(0, start), 0);
|
|
62
|
+
let o = start;
|
|
63
|
+
out[o++] = 7;
|
|
64
|
+
for (const x of lebBytes(body.length))
|
|
65
|
+
out[o++] = x;
|
|
66
|
+
out.set(body, o);
|
|
67
|
+
o += body.length;
|
|
68
|
+
out.set(wasm.subarray(end), o);
|
|
69
|
+
return out;
|
|
70
|
+
}
|
|
71
|
+
pos = end;
|
|
72
|
+
}
|
|
73
|
+
return wasm;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Whether an export stays in a module's wasm. `prefixes` are `convex_run:<module_path>:`. The match is
|
|
77
|
+
* exact: what follows the prefix is the function name (no `:`), so `convex_run:c::a::b:f` is not kept in
|
|
78
|
+
* the wasm of module `c::a`.
|
|
79
|
+
*/
|
|
80
|
+
export function keepsExport(exportName, prefixes) {
|
|
81
|
+
return (!exportName.startsWith("convex_run:") ||
|
|
82
|
+
prefixes.some((p) => exportName.startsWith(p) && !exportName.includes(":", p.length)));
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* One wasm per Convex module name (`cpu`, `a/b`), keyed by that name. `modulePaths` are the Rust
|
|
86
|
+
* `module_path!()` values of the functions. Returns null when `wasm-opt` is unavailable or fails
|
|
87
|
+
* (callers fall back to the single shared wasm).
|
|
88
|
+
*/
|
|
89
|
+
export function splitWasmByModule(wasm, modulePaths, opts) {
|
|
90
|
+
const log = opts.log ?? (() => { });
|
|
91
|
+
const ver = spawnSync("wasm-opt", ["--version"], { encoding: "utf8" });
|
|
92
|
+
if (ver.status !== 0) {
|
|
93
|
+
log("--split needs wasm-opt (binaryen) on PATH; using one shared wasm");
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
const modules = new Map();
|
|
97
|
+
for (const mp of modulePaths) {
|
|
98
|
+
const m = convexModuleName(mp);
|
|
99
|
+
if (!modules.has(m))
|
|
100
|
+
modules.set(m, new Set());
|
|
101
|
+
modules.get(m).add(mp);
|
|
102
|
+
}
|
|
103
|
+
const prefix = (mp) => `convex_run:${mp}:`;
|
|
104
|
+
const out = new Map();
|
|
105
|
+
fs.mkdirSync(opts.cacheDir, { recursive: true });
|
|
106
|
+
for (const [name, mps] of modules) {
|
|
107
|
+
const prefixes = [...mps].map(prefix);
|
|
108
|
+
const filtered = filterExports(wasm, (e) => keepsExport(e, prefixes));
|
|
109
|
+
const key = createHash("sha256")
|
|
110
|
+
.update(ver.stdout + "\0" + opts.wasmOptArgs.join(" ") + "\0split")
|
|
111
|
+
.update(filtered)
|
|
112
|
+
.digest("hex");
|
|
113
|
+
const cached = path.join(opts.cacheDir, `${key}.wasm`);
|
|
114
|
+
if (fs.existsSync(cached)) {
|
|
115
|
+
out.set(name, new Uint8Array(fs.readFileSync(cached)));
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
const src = path.join(opts.cacheDir, `${key}.${process.pid}.in`);
|
|
119
|
+
const dst = path.join(opts.cacheDir, `${key}.${process.pid}.tmp`);
|
|
120
|
+
fs.writeFileSync(src, filtered);
|
|
121
|
+
try {
|
|
122
|
+
const r = spawnSync("wasm-opt", [...opts.wasmOptArgs, "--remove-unused-module-elements", src, "-o", dst], {
|
|
123
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
124
|
+
encoding: "utf8",
|
|
125
|
+
});
|
|
126
|
+
if (r.status !== 0) {
|
|
127
|
+
log(`wasm-opt failed splitting ${name} (${r.status}); using one shared wasm`);
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
const bytes = new Uint8Array(fs.readFileSync(dst));
|
|
131
|
+
fs.renameSync(dst, cached);
|
|
132
|
+
out.set(name, bytes);
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
fs.rmSync(src, { force: true });
|
|
136
|
+
fs.rmSync(dst, { force: true });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return out;
|
|
140
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export interface SymbolMap {
|
|
2
|
+
version: 1;
|
|
3
|
+
/** File name under `_wasm/`. */
|
|
4
|
+
wasm: string;
|
|
5
|
+
/** Convex modules (`cpu`, `a/b`) whose functions this wasm exports. */
|
|
6
|
+
modules: string[];
|
|
7
|
+
/** Function index (imports first, as in `wasm-function[N]`) to demangled name. */
|
|
8
|
+
names: Record<string, string>;
|
|
9
|
+
}
|
|
10
|
+
/** Function names (subsection 1) of the "name" custom section, by function index. */
|
|
11
|
+
export declare function parseNameSection(wasm: Uint8Array): Map<number, string>;
|
|
12
|
+
/** Parses `wasm-opt --print-function-map` output (`index:name` per line). */
|
|
13
|
+
export declare function parseFunctionMap(text: string): Map<number, string>;
|
|
14
|
+
export declare function buildSymbolMap(names: Map<number, string>, info: {
|
|
15
|
+
wasm: string;
|
|
16
|
+
modules: string[];
|
|
17
|
+
}): SymbolMap;
|
|
18
|
+
export declare function symbolsDir(targetDir: string): string;
|
|
19
|
+
/** Replaces the maps in `dir` with `maps` (stale ones removed). */
|
|
20
|
+
export declare function writeSymbolMaps(dir: string, maps: SymbolMap[]): string[];
|
|
21
|
+
export declare function readSymbolMaps(dir: string): SymbolMap[];
|
|
22
|
+
/**
|
|
23
|
+
* Rewrites `wasm-function[N]:0xOFF` into `name (wasm-function[N]:0xOFF)`. With several maps, `module` picks one;
|
|
24
|
+
* otherwise a module named like `rust/<module>` in the text does, if exactly one does.
|
|
25
|
+
*/
|
|
26
|
+
export declare function symbolize(text: string, maps: SymbolMap[], module?: string): {
|
|
27
|
+
text: string;
|
|
28
|
+
replaced: number;
|
|
29
|
+
note?: string;
|
|
30
|
+
};
|