@amritk/nish-aarch64-darwin 0.35.0 → 0.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/INSTALL.md +73 -59
  2. package/bin/nish +0 -0
  3. package/docs/AI.md +50 -4
  4. package/package.json +1 -1
  5. package/runtime/async.d.mts +2 -0
  6. package/runtime/async.mjs +7 -0
  7. package/runtime/nish.d.ts +4849 -430
  8. package/runtime/nish.h +25 -0
  9. package/runtime/page.d.ts +53 -0
  10. package/runtime/page.mjs +146 -0
  11. package/runtime/runtime-async.c +557 -0
  12. package/runtime/runtime-wasm.c +63 -7
  13. package/runtime/vite.d.ts +30 -0
  14. package/runtime/vite.mjs +150 -0
  15. package/scripts/build.sh +252 -15
  16. package/std/README.md +107 -82
  17. package/std/base64.ts +34 -5
  18. package/std/bytes.ts +279 -59
  19. package/std/cli-args.ts +296 -70
  20. package/std/cli-flags.ts +184 -19
  21. package/std/cli.ts +475 -54
  22. package/std/collections.ts +121 -23
  23. package/std/compress/checksum.ts +44 -14
  24. package/std/compress/deflate.ts +224 -111
  25. package/std/compress/gzip.ts +217 -53
  26. package/std/compress/zlib.ts +63 -7
  27. package/std/crypto/aes.ts +128 -52
  28. package/std/crypto/base64url.ts +29 -5
  29. package/std/crypto/bcrypt-batch.ts +101 -34
  30. package/std/crypto/bcrypt.ts +93 -27
  31. package/std/crypto/chacha20poly1305.ts +69 -14
  32. package/std/crypto/ct.ts +32 -13
  33. package/std/crypto/ed25519.ts +66 -16
  34. package/std/crypto/hkdf.ts +198 -13
  35. package/std/crypto/hmac.ts +242 -37
  36. package/std/crypto/jwt.ts +204 -72
  37. package/std/crypto/p256.ts +222 -72
  38. package/std/crypto/p384.ts +51 -23
  39. package/std/crypto/pki-roots.ts +32 -2
  40. package/std/crypto/pki.ts +99 -30
  41. package/std/crypto/rsa.ts +56 -22
  42. package/std/crypto/sha1.ts +3 -0
  43. package/std/crypto/sha256.ts +24 -1
  44. package/std/crypto/sha512.ts +99 -14
  45. package/std/crypto/x25519.ts +163 -24
  46. package/std/crypto/x509-random.ts +10 -0
  47. package/std/crypto/x509.ts +133 -66
  48. package/std/host/ring.ts +124 -0
  49. package/std/json-equal.ts +31 -11
  50. package/std/json-format.ts +112 -13
  51. package/std/json-number.ts +76 -26
  52. package/std/json-pointer.ts +23 -1
  53. package/std/json-tokenizer.ts +126 -51
  54. package/std/json-typed.ts +757 -96
  55. package/std/json-value.ts +438 -157
  56. package/std/json.ts +170 -38
  57. package/std/map.ts +25 -7
  58. package/std/math.ts +36 -4
  59. package/std/net/buffer-pool.ts +42 -11
  60. package/std/net/dns-config.ts +109 -23
  61. package/std/net/dns-message.ts +62 -7
  62. package/std/net/dns-resolver.ts +247 -53
  63. package/std/net/dns.ts +203 -35
  64. package/std/net/fetch.ts +1267 -290
  65. package/std/net/handler.ts +296 -80
  66. package/std/net/hpack.ts +290 -77
  67. package/std/net/http-compress.ts +152 -36
  68. package/std/net/http-fields.ts +85 -9
  69. package/std/net/http1-client-codec.ts +129 -44
  70. package/std/net/http1-codec.ts +279 -114
  71. package/std/net/http1-server.ts +409 -139
  72. package/std/net/http2-frame.ts +289 -51
  73. package/std/net/http2-tls.ts +152 -49
  74. package/std/net/http2.ts +314 -160
  75. package/std/net/http3-frame.ts +165 -27
  76. package/std/net/http3-server.ts +282 -131
  77. package/std/net/http3.ts +377 -173
  78. package/std/net/outbound.ts +37 -3
  79. package/std/net/postgres-carrier.ts +144 -8
  80. package/std/net/postgres.ts +898 -260
  81. package/std/net/prefork.ts +202 -65
  82. package/std/net/qpack.ts +120 -32
  83. package/std/net/quic-conn-ack.ts +60 -12
  84. package/std/net/quic-conn-cid.ts +135 -27
  85. package/std/net/quic-conn-params.ts +109 -22
  86. package/std/net/quic-datagram.ts +50 -3
  87. package/std/net/quic-frame.ts +333 -23
  88. package/std/net/quic-listener.ts +161 -75
  89. package/std/net/quic-packet.ts +373 -73
  90. package/std/net/quic-recovery.ts +287 -65
  91. package/std/net/quic-stream.ts +436 -146
  92. package/std/net/quic.ts +693 -323
  93. package/std/net/router.ts +87 -40
  94. package/std/net/tls/client.ts +167 -51
  95. package/std/net/tls/codec.ts +494 -68
  96. package/std/net/tls/record-client.ts +140 -73
  97. package/std/net/tls/record-server.ts +233 -105
  98. package/std/net/tls/record.ts +142 -36
  99. package/std/net/tls/schedule.ts +235 -26
  100. package/std/net/tls/ticket.ts +161 -40
  101. package/std/net/tls/verify.ts +57 -10
  102. package/std/net/tls-tcp.ts +308 -95
  103. package/std/net/tls.ts +276 -102
  104. package/std/net/upstream.ts +81 -45
  105. package/std/net/url.ts +109 -22
  106. package/std/net/websocket.ts +140 -26
  107. package/std/net/webtransport.ts +243 -106
  108. package/std/pair.ts +10 -2
  109. package/std/random.ts +42 -15
  110. package/std/regex.ts +422 -257
  111. package/std/region.ts +70 -15
  112. package/std/secret.ts +52 -11
  113. package/std/sort.ts +17 -2
  114. package/std/sqlite.ts +256 -35
  115. package/std/sync/ring.ts +229 -56
  116. package/std/sync/shm.ts +203 -23
  117. package/std/term-io.ts +154 -78
  118. package/std/term-pixels.ts +144 -19
  119. package/std/term-probe.ts +77 -35
  120. package/std/term-rate.ts +199 -118
  121. package/std/term.ts +400 -175
  122. package/std/testing.ts +130 -32
  123. package/std/text.ts +84 -17
  124. package/std/threads.ts +248 -56
  125. package/std/yaml-json.ts +111 -97
  126. package/std/yaml-lines.ts +47 -15
  127. package/std/yaml-write.ts +45 -30
  128. package/std/yaml.ts +764 -384
@@ -0,0 +1,30 @@
1
+ /**
2
+ * `nish/vite`: the Vite plugin that builds a Nish module imported by a page
3
+ * into wasm, inlined and loaded without a fetch (docs/guide/using-nish/interop.md).
4
+ */
5
+
6
+ export interface NishViteOptions {
7
+ /** Modules to build to wasm, paths from the root. Default: none. */
8
+ entries?: string[];
9
+ /**
10
+ * Host modules, paths from the root, each passed to the compiler as
11
+ * `--host`; a root `package.json` `"nish": { "host": [...] }` also names
12
+ * them. The ones an entry imports are passed to its loader. Default: none.
13
+ */
14
+ host?: string[];
15
+ /** The `nish` command. Default: this package's `bin/nish`. */
16
+ compiler?: string;
17
+ /** `scripts/build.sh`. Default: this package's. */
18
+ buildScript?: string;
19
+ }
20
+
21
+ /** The hooks the plugin uses, which Vite and Rollup share. */
22
+ export interface NishVitePlugin {
23
+ name: "nish";
24
+ enforce: "pre";
25
+ configResolved(config: { root?: string; command?: "build" | "serve"; cacheDir?: string }): void;
26
+ resolveId(source: string, importer?: string): Promise<string | null>;
27
+ load(id: string): Promise<string | null>;
28
+ }
29
+
30
+ export default function nish(options?: NishViteOptions): NishVitePlugin;
@@ -0,0 +1,150 @@
1
+ /**
2
+ * `nish/vite` (WP50 stage 1): a Vite plugin that lets a page import a Nish
3
+ * module as the TypeScript it is and get, in a build, the wasm build of it.
4
+ *
5
+ * The entries are the modules compiled to wasm, `nish({ entries })`, paths
6
+ * from the root. (The root package.json's `nish` field cannot name them yet:
7
+ * the compiler refuses a key there other than `noPanic`, `capabilities` and
8
+ * `host`.) In
9
+ * `vite build` an import that resolves to one is answered with a generated
10
+ * module. The plugin compiles the entry with `nish --target wasm32 --emit-dts`,
11
+ * links the IR with `scripts/build.sh --profile wasm-host --inline`, and the
12
+ * generated module calls the loader's `loadSync` on the inlined bytes and
13
+ * re-exports each export by name. The page fetches nothing and awaits nothing.
14
+ * In `vite serve` the import is left to Vite, which serves the TypeScript.
15
+ *
16
+ * An entry that imports host modules (WP50 stage 2) is instantiated with
17
+ * them: the generated module imports each one the `.d.ts` names in `Imports`
18
+ * and passes them as `host`, so the page passes nothing. One seam is left for
19
+ * a later stage of docs/wp50-wasm-host.md: the `.nish` client specifier in
20
+ * `resolveId`.
21
+ *
22
+ * The plugin drives only the hooks Vite and Rollup share (`configResolved`,
23
+ * `resolveId`, `load`), so `tests/run.js` calls them directly, with no Vite.
24
+ */
25
+ import { execFile } from "node:child_process";
26
+ import { mkdirSync, readFileSync } from "node:fs";
27
+ import path from "node:path";
28
+ import { fileURLToPath } from "node:url";
29
+ import { promisify } from "node:util";
30
+
31
+ const run = promisify(execFile);
32
+ const packageDir = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
33
+ const PREFIX = "\0nish-entry:";
34
+
35
+ /**
36
+ * The host modules an entry imports, as absolute paths: the
37
+ * `typeof import("...")` members of the `.d.ts`'s `Imports` type, each a
38
+ * specifier from the `.d.ts`'s directory written without its `.ts`, which is
39
+ * put back so the import names the file. None when there is no `Imports`.
40
+ */
41
+ function hostModules(dts, dir) {
42
+ const line = dts.split("\n").find((l) => l.startsWith("export type Imports = "));
43
+ if (line === undefined) return [];
44
+ return [...line.matchAll(/typeof import\("([^"]+)"\)/g)].map((m) => `${path.resolve(dir, m[1])}.ts`);
45
+ }
46
+
47
+ /**
48
+ * The import lines and the import object an entry is instantiated with: each
49
+ * host module imported whole and passed under `host`, or `{}` with none.
50
+ */
51
+ function importsFor(hosts) {
52
+ const lines = hosts.map((h, i) => `import * as __nishHost${i} from ${JSON.stringify(h)};`);
53
+ if (hosts.length === 0) return { lines, object: "{}" };
54
+ const host = hosts.length === 1 ? "__nishHost0" : `{ ${hosts.map((_, i) => `...__nishHost${i}`).join(", ")} }`;
55
+ return { lines, object: `{ host: ${host} }` };
56
+ }
57
+
58
+ /**
59
+ * The names the loader returns, read from the `Exports` interface of the
60
+ * `.d.ts` the same compile wrote: one member a line, `memory` included.
61
+ */
62
+ function exportNames(dts) {
63
+ const body = dts.slice(dts.indexOf("export interface Exports {"));
64
+ const names = [];
65
+ for (const m of body.slice(0, body.indexOf("\n}")).matchAll(/^ {2}(?:readonly )?([A-Za-z_$][\w$]*)[(:]/gm)) {
66
+ names.push(m[1]);
67
+ }
68
+ return names;
69
+ }
70
+
71
+ /** The plugin. `compiler` and `buildScript` default to the ones in this package. */
72
+ export default function nish(options = {}) {
73
+ const compiler = options.compiler ?? path.join(packageDir, "bin", "nish");
74
+ const buildScript = options.buildScript ?? path.join(packageDir, "scripts", "build.sh");
75
+ const runtimeWasm = path.join(packageDir, "runtime", "runtime-wasm.c");
76
+ let root = process.cwd();
77
+ let building = false;
78
+ let cacheDir = path.join(root, "node_modules", ".vite");
79
+ let entries = new Set();
80
+
81
+ const configure = (config) => {
82
+ root = config.root ?? root;
83
+ building = config.command === "build";
84
+ cacheDir = config.cacheDir ?? path.join(root, "node_modules", ".vite");
85
+ entries = new Set((options.entries ?? []).map((e) => path.resolve(root, e)));
86
+ };
87
+ configure({ root });
88
+
89
+ /** Compile and link one entry; resolves to the paths the generated module imports. */
90
+ const buildEntry = async (entry) => {
91
+ const stem = path.basename(entry, path.extname(entry));
92
+ const dir = path.join(cacheDir, "nish", path.relative(root, entry).replace(/[\\/]/g, "_"));
93
+ mkdirSync(dir, { recursive: true });
94
+ const ll = path.join(dir, `${stem}.ll`);
95
+ const dts = path.join(dir, `${stem}.d.ts`);
96
+ const host = (options.host ?? []).flatMap((h) => ["--host", path.resolve(root, h)]);
97
+ await run(compiler, [entry, "--target", "wasm32", ...host, "--emit-dts", dts, "-o", ll], { cwd: root });
98
+ const loader = path.join(dir, `${stem}.mjs`);
99
+ const ir = readFileSync(ll, "utf8");
100
+ // runtime-wasm.c is linked when the IR needs a runtime function (a weak
101
+ // `extern_weak` one, a panic stub wasm-ld writes itself, does not count)
102
+ // or the loader calls the arena.
103
+ const needsRuntime =
104
+ /^declare (?!extern_weak)[^"\n]*@nish_/m.test(ir) ||
105
+ readFileSync(loader, "utf8").includes("raw.nish_arena_mark(");
106
+ const inline = path.join(dir, `${stem}.bytes.mjs`);
107
+ const inputs = needsRuntime ? [ll, runtimeWasm] : [ll];
108
+ await run("bash", [buildScript, ...inputs, "-o", path.join(dir, `${stem}.wasm`), "--profile", "wasm-host", "--inline", inline], {
109
+ cwd: root,
110
+ });
111
+ const types = readFileSync(dts, "utf8");
112
+ return { loader, inline, names: exportNames(types), hosts: hostModules(types, dir) };
113
+ };
114
+
115
+ return {
116
+ name: "nish",
117
+ enforce: "pre",
118
+ configResolved: configure,
119
+ async resolveId(source, importer) {
120
+ if (!building) return null;
121
+ // The seam for a `./client.nish` specifier (the page's client module of a
122
+ // later stage) is here; stage 1 resolves only the entries.
123
+ const base = importer ? path.dirname(importer) : root;
124
+ const candidates = source.startsWith(".")
125
+ ? [path.resolve(base, source)]
126
+ : path.isAbsolute(source)
127
+ ? [source, path.join(root, source)]
128
+ : [];
129
+ const hit = candidates.find((c) => entries.has(c));
130
+ return hit === undefined ? null : PREFIX + hit;
131
+ },
132
+ async load(id) {
133
+ if (!id.startsWith(PREFIX)) return null;
134
+ const entry = id.slice(PREFIX.length);
135
+ const { loader, inline, names, hosts } = await buildEntry(entry);
136
+ const imports = importsFor(hosts);
137
+ // The module's own bindings are prefixed, so an export named `bytes`,
138
+ // `loadSync` or `exports` does not redeclare one of them.
139
+ const lines = [
140
+ `import { loadSync as __nishLoadSync } from ${JSON.stringify(loader)};`,
141
+ `import { bytes as __nishBytes } from ${JSON.stringify(inline)};`,
142
+ ...imports.lines,
143
+ `const __nishExports = __nishLoadSync(__nishBytes, ${imports.object});`,
144
+ "export default __nishExports;",
145
+ ];
146
+ if (names.length > 0) lines.push(`export const { ${names.join(", ")} } = __nishExports;`);
147
+ return `${lines.join("\n")}\n`;
148
+ },
149
+ };
150
+ }
package/scripts/build.sh CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env bash
2
2
  # Build an Nish .ll module (plus optional C sources) into a native binary.
3
3
  #
4
- # scripts/build.sh <module.ll> [more .ll/.c files...] -o <out> [--profile debug|speed|size|wasm|wasi|napi]
4
+ # scripts/build.sh <module.ll> [more .ll/.c files...] -o <out> [--profile debug|speed|size|wasm|wasm-host|wasi|napi]
5
5
  #
6
6
  # The C runtime is sixteen translation units and is named as one: an input
7
7
  # <dir>/runtime.c also compiles <dir>/runtime-os.c, the half that wraps the
@@ -20,7 +20,9 @@
20
20
  # <dir>/runtime-bytes.c, the byte-class runs of `nish/bytes`,
21
21
  # <dir>/runtime-sandbox.c, the confinement `--sandbox` applies,
22
22
  # <dir>/runtime-shm.c, the shared-memory regions of `nish/sync/shm`, and
23
- # <dir>/runtime-region.c, the arenas a program makes and frees one at a time. Each of those
23
+ # <dir>/runtime-region.c, the arenas a program makes and frees one at a time, and,
24
+ # only for a program whose IR declares one of its functions,
25
+ # <dir>/runtime-async.c, the executor `spawn` and `nish:async`'s waits park tasks on. Each of those
24
26
  # files says why they are compiled and measured apart.
25
27
  #
26
28
  # Profiles:
@@ -33,6 +35,28 @@
33
35
  # load it from Node. Add runtime/runtime-wasm.c to the inputs when a
34
36
  # function uses arrays (the arena and the array cold paths, no libc);
35
37
  # strings and I/O still need a WASI runtime and are not available.
38
+ # wasm-host the wasm profile's module for a page (WP50 stage 1): the same
39
+ # inputs, linked with an explicit export list instead of exporting
40
+ # everything -- the program's exported functions, `memory`, and,
41
+ # when runtime-wasm.c is linked, the arena entry points the
42
+ # `--emit-dts` loader calls (nish_arena_mark, nish_arena_release,
43
+ # nish_alloc_array, nish_reset_arena, nish_free_arena). The stack is
44
+ # a fixed 64 KiB (-z stack-size=65536). Runs `wasm-opt -Oz` on the
45
+ # module when wasm-opt (binaryen) is on PATH; the build line says
46
+ # whether it ran. Two flags take this profile alone, and any other
47
+ # refuses them with exit 2:
48
+ # --memory <MiB> memory fixed at <MiB> MiB (1 to 4096), the
49
+ # initial size and the maximum both, so it
50
+ # never grows and `memory.buffer` is one
51
+ # ArrayBuffer for the module's life; an arena
52
+ # that runs out traps. Without it (the
53
+ # default) the memory starts at what the
54
+ # module needs and grows on demand, as under
55
+ # the wasm profile.
56
+ # --inline <file.mjs> also write <file.mjs>, an ES module whose
57
+ # `export const bytes = new Uint8Array([...])`
58
+ # holds the .wasm's bytes, so a page loads the
59
+ # module with `loadSync(bytes)` and no fetch.
36
60
  # wasi wasm32-wasi command module: runtime.c linked against wasi-libc, so
37
61
  # string programs run under any WASI host (`_start` runs `main`;
38
62
  # `node examples/wasi-host.mjs app.wasm args...`). Needs a WASI
@@ -138,6 +162,9 @@ cpu="" # --cpu x86-64-v3: the one CPU level abov
138
162
  sqlite_to="" # --sqlite-objects <dir>: compile runtime/sqlite/ alone, into <dir>
139
163
  sqlite_from="" # --sqlite-from <dir>: link runtime/sqlite/ compiled there
140
164
  modules_at="" # --module-objects <dir>: each X.ll's object is <dir>/X.o
165
+ memory_mib="" # --memory <MiB>: wasm-host's fixed memory (WP50 stage 1)
166
+ wasm_opt="" # what the build line says of wasm-opt; set only by wasm-host
167
+ inline_to="" # --inline <file.mjs>: wasm-host's bytes as an ES module
141
168
  while [ $# -gt 0 ]; do
142
169
  case "$1" in
143
170
  -o) out="$2"; shift 2 ;;
@@ -152,6 +179,12 @@ while [ $# -gt 0 ]; do
152
179
  --sqlite-objects) sqlite_to="$2"; shift 2 ;;
153
180
  --sqlite-from) sqlite_from="$2"; shift 2 ;;
154
181
  --module-objects) modules_at="$2"; shift 2 ;;
182
+ --memory)
183
+ [ -n "${2-}" ] || { echo "error: --memory needs a size in MiB" >&2; exit 2; }
184
+ memory_mib="$2"; shift 2 ;;
185
+ --inline)
186
+ [ -n "${2-}" ] || { echo "error: --inline needs a file to write" >&2; exit 2; }
187
+ inline_to="$2"; shift 2 ;;
155
188
  --pgo-generate) pgo=(-fprofile-generate); shift ;;
156
189
  --pgo-use)
157
190
  [ -f "$2" ] || { echo "error: --pgo-use: profile '$2' not found (run the instrumented binary, then llvm-profdata merge)" >&2; exit 2; }
@@ -164,9 +197,19 @@ done
164
197
  if [ -n "$cpu" ]; then
165
198
  [ "$cpu" = x86-64-v3 ] || { echo "error: --cpu takes x86-64-v3, not '$cpu'" >&2; exit 2; }
166
199
  case "$profile" in
167
- wasm|wasi) echo "error: --cpu x86-64-v3 is an x86-64 level, so the $profile profile does not take it" >&2; exit 2 ;;
200
+ wasm|wasm-host|wasi) echo "error: --cpu x86-64-v3 is an x86-64 level, so the $profile profile does not take it" >&2; exit 2 ;;
168
201
  esac
169
202
  fi
203
+ if [ -n "$memory_mib" ] || [ -n "$inline_to" ]; then
204
+ [ "$profile" = wasm-host ] || { echo "error: --memory and --inline take the wasm-host profile, not '$profile'" >&2; exit 2; }
205
+ fi
206
+ if [ -n "$memory_mib" ]; then
207
+ case "$memory_mib" in
208
+ *[!0-9]*|0*) memory_ok=0 ;;
209
+ *) if [ "${#memory_mib}" -le 4 ] && [ "$memory_mib" -le 4096 ]; then memory_ok=1; else memory_ok=0; fi ;;
210
+ esac
211
+ [ "$memory_ok" = 1 ] || { echo "error: --memory takes a whole number of MiB from 1 to 4096, not '$memory_mib'" >&2; exit 2; }
212
+ fi
170
213
  # wasm-ld exports a wasm module's linear memory as `memory`, so under the wasm
171
214
  # profile's `--export-all` a function of that name makes a module Node refuses
172
215
  # to compile ("Duplicate export name 'memory'", #643). The wasi module exports
@@ -175,7 +218,7 @@ fi
175
218
  # when it knows the build is wasm, but IR compiled for the host with `-o` alone
176
219
  # reaches these profiles without telling it, so the IR is read here too.
177
220
  case "$profile" in
178
- wasm|wasi)
221
+ wasm|wasm-host|wasi)
179
222
  for i in ${inputs[@]+"${inputs[@]}"}; do
180
223
  case "$i" in
181
224
  *.ll)
@@ -294,12 +337,40 @@ for i in ${inputs[@]+"${inputs[@]}"}; do
294
337
  *.c) if [ -n "$sqlite_dir" ] && [ "$(cd "$(dirname -- "$i")" && pwd -P)" = "$sqlite_dir" ]; then sqlite_wanted=0; break; fi ;;
295
338
  esac
296
339
  done
340
+ # The async executor (runtime-async.c) is selected by the IR the same way: a
341
+ # module that declares one of its `nish_async_*` functions -- every program
342
+ # that spawns, awaits a kernel wait or runs an async main does -- gets it
343
+ # compiled and linked beside runtime.c. Any other program links none of it, so
344
+ # its binary holds no `nish_async` symbol and its bytes stay as they were.
345
+ # `--runtime-objects` compiles it with the rest, so the runtime cache holds it,
346
+ # and `--runtime-from` links its object only when it is wanted. A caller that
347
+ # names it itself is left alone, as `halves` leaves a half.
348
+ async_wanted=0
349
+ if [ ${#lls[@]} -gt 0 ] && grep -Eq '^declare [^"]*@nish_async_' "${lls[@]}"; then
350
+ async_wanted=1
351
+ fi
352
+ if [ "$async_wanted" = 1 ]; then
353
+ for i in ${inputs[@]+"${inputs[@]}"}; do
354
+ case "$i" in
355
+ */runtime-async.c|runtime-async.c) async_wanted=0 ;;
356
+ esac
357
+ done
358
+ fi
359
+ if [ "$async_wanted" = 1 ] && [ -z "$objects_from" ]; then
360
+ for i in ${inputs[@]+"${inputs[@]}"}; do
361
+ case "$i" in
362
+ */runtime.c|runtime.c)
363
+ side="${i%runtime.c}runtime-async.c"
364
+ if [ -f "$side" ]; then inputs+=("$side"); fi ;;
365
+ esac
366
+ done
367
+ fi
297
368
  if [ "$sqlite_wanted" = 1 ] && [ -z "$sqlite_dir" ]; then
298
369
  echo "error: the module calls SQLite, and $(dirname "$0")/../runtime/sqlite/ is not there" >&2
299
370
  exit 2
300
371
  fi
301
- if [ "$sqlite_wanted" = 1 ] && [ "$profile" = wasm ]; then
302
- echo "error: the module calls SQLite, which the wasm profile cannot link: it has no libc for runtime/sqlite/sqlite3.c to build on; use --profile wasi" >&2
372
+ if [ "$sqlite_wanted" = 1 ] && { [ "$profile" = wasm ] || [ "$profile" = wasm-host ]; }; then
373
+ echo "error: the module calls SQLite, which the $profile profile cannot link: it has no libc for runtime/sqlite/sqlite3.c to build on; use --profile wasi" >&2
303
374
  exit 2
304
375
  fi
305
376
 
@@ -615,7 +686,7 @@ if [ -n "$objects_to" ]; then
615
686
  # the file `runtime.c` in the runtime's own directory, whichever directory
616
687
  # the first link that compiled it was run in.
617
688
  runtime_dir=$(dirname -- "$runtime")
618
- for unit in runtime.c $halves; do
689
+ for unit in runtime.c $halves runtime-async.c; do
619
690
  (cd -- "$runtime_dir" && exec "$CC" "${compile[@]}" -c "$unit" -o "$objects_to/${unit%.c}.o") &
620
691
  pids+=("$!")
621
692
  done
@@ -653,6 +724,10 @@ if [ -n "$objects_from" ]; then
653
724
  for unit in runtime.c $halves; do
654
725
  objects+=("$objects_from/${unit%.c}.o")
655
726
  done
727
+ if [ "$async_wanted" = 1 ]; then
728
+ [ -f "$objects_from/runtime-async.o" ] || { echo "error: --runtime-from: $objects_from/runtime-async.o is missing" >&2; exit 2; }
729
+ objects+=("$objects_from/runtime-async.o")
730
+ fi
656
731
  inputs=("${objects[@]}")
657
732
  # clang runs dsymutil itself when one command compiles and links with -g on
658
733
  # macOS; a link of objects does not, so it is run here, while the program's
@@ -672,7 +747,7 @@ case "$profile" in
672
747
  speed|size)
673
748
  "$CC" "${common[@]}" "${optimise[@]}" ${lto_parts[@]+"${lto_parts[@]}"} \
674
749
  "${gc[@]}" ${strip_flag[@]+"${strip_flag[@]}"} "${inputs[@]}" -lm -o "$out" ;;
675
- wasm)
750
+ wasm|wasm-host)
676
751
  # A region's chunks come from malloc, and the freestanding wasm runtime
677
752
  # (runtime-wasm.c) has none, so IR that makes or enters one is refused here
678
753
  # rather than failing at the link on an undefined `nish_region_*`. The wasi
@@ -680,7 +755,7 @@ case "$profile" in
680
755
  for i in ${inputs[@]+"${inputs[@]}"}; do
681
756
  case "$i" in
682
757
  *.ll) if grep -q "@nish_region_" "$i"; then
683
- echo "error: $i uses regions, which the wasm profile cannot link (it has no malloc); build it with --profile wasi" >&2
758
+ echo "error: $i uses regions, which the $profile profile cannot link (it has no malloc); build it with --profile wasi" >&2
684
759
  exit 2
685
760
  fi ;;
686
761
  esac
@@ -689,7 +764,7 @@ case "$profile" in
689
764
  # rather than probing PATH so a Homebrew llvm without a PATH entry still works.
690
765
  wasm_ld=$("$CC" -print-prog-name=wasm-ld)
691
766
  if [ ! -x "$wasm_ld" ]; then
692
- echo "error: the wasm profile needs wasm-ld (install lld; on macOS: brew install llvm@18)" >&2
767
+ echo "error: the $profile profile needs wasm-ld (install lld; on macOS: brew install llvm@18)" >&2
693
768
  exit 2
694
769
  fi
695
770
  # IR compiled for the host rather than with `--target wasm32` calls std/crypto/aes.ts's
@@ -744,15 +819,177 @@ case "$profile" in
744
819
  *.ll) if grep -q '@nish_round_init' "$i"; then round_wanted=1; fi ;;
745
820
  esac
746
821
  done
822
+ # Under wasm-host a program whose loader guards panics (one with module
823
+ # storage or a host module: the compiler ends its IR with a
824
+ # `; nish.sites <count>` line) reports each panic through the page's
825
+ # `nish.panic` import, which runtime-wasm.c makes the call to when it is
826
+ # compiled with NISH_WASM_HOST. Any other program links the bytes it always
827
+ # did, with no import. IR with a tagged site links the runtime for the
828
+ # import, or the panic would stay the silent trap of an undefined weak
829
+ # stub; linked for that, it exports the arena entry points as any link of
830
+ # runtime-wasm.c does, since a guarded loader that moves an array marks
831
+ # and releases the arena around each call.
832
+ host_panics=0
833
+ if [ "$profile" = wasm-host ]; then
834
+ for i in ${inputs[@]+"${inputs[@]}"}; do
835
+ case "$i" in
836
+ *.ll) if grep -qE '^; nish\.sites [0-9]+$' "$i"; then host_panics=1; fi ;;
837
+ esac
838
+ done
839
+ fi
840
+ if [ "$host_panics" = 1 ] && [ "$wasm_named" = 0 ] && [ "$round_wanted" = 0 ]; then
841
+ for i in ${inputs[@]+"${inputs[@]}"}; do
842
+ case "$i" in
843
+ *.ll) if grep -qE '^[[:space:]]+call void @nish_panic[A-Za-z0-9_]*\(.*, !nish\.site !\{i32 ' "$i"; then round_wanted=1; fi ;;
844
+ esac
845
+ done
846
+ fi
747
847
  if [ "$round_wanted" = 1 ] && [ "$wasm_named" = 0 ]; then
748
848
  inputs+=("$(cd "$(dirname "$0")/../runtime" && pwd)/runtime-wasm.c")
749
849
  fi
750
850
  # -mbulk-memory lowers llvm.memset/memcpy (`new Array<T>(n)`, `push` growth) to the
751
851
  # memory.fill/memory.copy instructions instead of libc calls the freestanding link lacks.
752
- "$CC" -Wno-override-module --target=wasm32-unknown-unknown -Oz -nostdlib -mbulk-memory \
753
- ${tls[@]+"${tls[@]}"} \
754
- -Wl,--no-entry -Wl,--export-all -Wl,--strip-all -Wl,--gc-sections \
755
- "${inputs[@]}" -o "$out" ;;
852
+ wasm_compile=(-Wno-override-module --target=wasm32-unknown-unknown -Oz -nostdlib -mbulk-memory)
853
+ if [ "$profile" = wasm ]; then
854
+ "$CC" "${wasm_compile[@]}" \
855
+ ${tls[@]+"${tls[@]}"} \
856
+ -Wl,--no-entry -Wl,--export-all -Wl,--strip-all -Wl,--gc-sections \
857
+ "${inputs[@]}" -o "$out"
858
+ else
859
+ # Panic sites (WP50 stage 3). For a program whose loader guards panics
860
+ # the compiler tags each call of a runtime panic `!nish.site !{i32 N}`
861
+ # and lists the sites at the end of the module as
862
+ # `; nish.site N <kind> <path:line:col>` comments; code generation drops
863
+ # both, so `--profile wasm` links what it always did. Here each tagged
864
+ # call becomes its `_at` form taking the site as a first argument,
865
+ # numbered across the inputs, runtime-wasm.c (compiled with
866
+ # NISH_WASM_HOST) hands it to the page's `nish.panic` import, and the
867
+ # sites are written to `<out>.sites.json` for the loader. A program
868
+ # without the `; nish.sites` line has no tag, no import and no sidecar,
869
+ # and its build never touches `<out>.sites.json`; a guarded build
870
+ # removes the one an earlier build left before it writes its own.
871
+ host_define=()
872
+ host_inputs=("${inputs[@]}")
873
+ if [ "$host_panics" = 1 ]; then
874
+ rm -f "$out.sites.json"
875
+ host_define=(-DNISH_WASM_HOST)
876
+ make_work
877
+ sites_dir="$work/sites"
878
+ mkdir -p "$sites_dir"
879
+ sites_base=0
880
+ sites_json="$sites_dir/sites"
881
+ : > "$sites_json"
882
+ host_inputs=()
883
+ for i in "${inputs[@]}"; do
884
+ case "$i" in
885
+ *.ll)
886
+ rewritten="$sites_dir/${#host_inputs[@]}.ll"
887
+ awk -v base="$sites_base" -v list="$sites_json" '
888
+ # One tagged call: "call void @nish_panic<x>(<args>)<attrs>, !nish.site !{i32 N}<rest>".
889
+ # (`close` is a builtin, so the brace is `shut`.)
890
+ function untag(line, at, rest, shut, n, head, open, args) {
891
+ at = index(line, ", !nish.site !{i32 ")
892
+ rest = substr(line, at + 19)
893
+ shut = index(rest, "}")
894
+ n = substr(rest, 1, shut - 1) + base
895
+ head = substr(line, 1, at - 1)
896
+ open = index(head, "(")
897
+ args = substr(head, open + 1)
898
+ return substr(head, 1, open - 1) "_at(i32 " n (substr(args, 1, 1) == ")" ? "" : ", ") args substr(rest, shut + 1)
899
+ }
900
+ /^; nish\.site [0-9]+ / {
901
+ kind = $4
902
+ where = $0
903
+ sub(/^; nish\.site [0-9]+ [a-z]+ /, "", where)
904
+ gsub(/\\/, "\\\\", where)
905
+ gsub(/"/, "\\\"", where)
906
+ printf " { \"kind\": \"%s\", \"at\": \"%s\" }\n", kind, where >> list
907
+ count++
908
+ print
909
+ next
910
+ }
911
+ /^declare [a-z_ ]*void @nish_panic[a-z_]*\(/ {
912
+ print
913
+ open = index($0, "(")
914
+ print substr($0, 1, open - 1) "_at(i32" (substr($0, open + 1, 1) == ")" ? "" : ", ") substr($0, open + 1)
915
+ next
916
+ }
917
+ # Only a call line: a string constant may spell the tag in its bytes.
918
+ /^[ \t]+call void @nish_panic[A-Za-z0-9_]*\(.*, !nish\.site !\{i32 [0-9]+\}/ { print untag($0); next }
919
+ { print }
920
+ END { print count + 0 > (list ".count") }
921
+ ' "$i" > "$rewritten"
922
+ sites_base=$((sites_base + $(cat "$sites_json.count")))
923
+ host_inputs+=("$rewritten") ;;
924
+ *) host_inputs+=("$i") ;;
925
+ esac
926
+ done
927
+ {
928
+ printf '['
929
+ if [ -s "$sites_json" ]; then
930
+ printf '\n'
931
+ sed '$!s/$/,/' "$sites_json"
932
+ fi
933
+ printf ']\n'
934
+ } > "$out.sites.json"
935
+ fi
936
+ # The export list. A module's exported functions are its `define` lines
937
+ # with external linkage and default visibility: the compiler gives
938
+ # everything else `internal` or `private`, and the other linkages are
939
+ # listed for IR it did not write. wasm-ld exports `memory` itself.
940
+ exports=()
941
+ runtime_linked=0
942
+ for i in "${inputs[@]}"; do
943
+ case "$i" in
944
+ */runtime-wasm.c|runtime-wasm.c) runtime_linked=1 ;;
945
+ *.ll)
946
+ while IFS= read -r name; do
947
+ exports+=("-Wl,--export=$name")
948
+ done < <(grep -E '^define ' "$i" \
949
+ | grep -Ev '^define (internal|private|linkonce|linkonce_odr|weak|weak_odr|available_externally|extern_weak|common|hidden|protected) ' \
950
+ | sed -nE 's/^[^@]*@"?([^"(]+)"?\(.*/\1/p')
951
+ # Module storage (WP50 stage 3): each `@<name>.data` global is
952
+ # exported, and wasm-ld exports a data symbol as a global holding
953
+ # its address, which the loader reads into `layout.<name>.at`.
954
+ while IFS= read -r name; do
955
+ exports+=("-Wl,--export=$name")
956
+ done < <(sed -nE 's/^@"?([^" =]+\.data)"? = global .*/\1/p' "$i") ;;
957
+ esac
958
+ done
959
+ if [ "$runtime_linked" = 1 ]; then
960
+ for name in nish_arena_mark nish_arena_release nish_alloc_array nish_reset_arena nish_free_arena; do
961
+ exports+=("-Wl,--export=$name")
962
+ done
963
+ fi
964
+ # shellcheck disable=SC2054 # -Wl,<flag> is one argument: the comma is the linker's
965
+ memory=(-Wl,-z,stack-size=65536)
966
+ if [ -n "$memory_mib" ]; then
967
+ bytes=$((memory_mib * 1048576))
968
+ memory+=("-Wl,--initial-memory=$bytes" "-Wl,--max-memory=$bytes")
969
+ fi
970
+ "$CC" "${wasm_compile[@]}" ${host_define[@]+"${host_define[@]}"} \
971
+ ${tls[@]+"${tls[@]}"} \
972
+ -Wl,--no-entry ${exports[@]+"${exports[@]}"} "${memory[@]}" -Wl,--strip-all -Wl,--gc-sections \
973
+ "${host_inputs[@]}" -o "$out"
974
+ # binaryen's size pass is optional: the module is complete without it.
975
+ if command -v wasm-opt >/dev/null 2>&1; then
976
+ wasm-opt -Oz --enable-bulk-memory "$out" -o "$out"
977
+ wasm_opt="wasm-opt -Oz"
978
+ else
979
+ wasm_opt="no wasm-opt"
980
+ fi
981
+ if [ -n "$inline_to" ]; then
982
+ # Decimal, one array literal: of the inline forms the smallest after
983
+ # compression (docs/wp50-wasm-host.md, Q10). od prints each byte as an
984
+ # unsigned decimal, and tr and sed join them with commas.
985
+ {
986
+ printf '// The bytes of %s, from scripts/build.sh --profile wasm-host --inline.\n' "${out##*/}"
987
+ printf 'export const bytes = new Uint8Array(['
988
+ od -An -v -tu1 "$out" | tr -s ' \n' ' ' | sed -E 's/^ //; s/ $//; s/ /,/g' | tr -d '\n'
989
+ printf ']);\n'
990
+ } > "$inline_to"
991
+ fi
992
+ fi ;;
756
993
  wasi)
757
994
  sysroot=${WASI_SYSROOT:-}
758
995
  for d in /usr/lib/wasi-sysroot /opt/wasi-sdk/share/wasi-sysroot /usr/share/wasi-sysroot; do
@@ -830,4 +1067,4 @@ esac
830
1067
  if [ "${dsym:-0}" = 1 ]; then "$("$CC" -print-prog-name=dsymutil)" "$out"; fi
831
1068
 
832
1069
  # `wc -c` pads with spaces on macOS; strip them so the number is clean.
833
- printf '%s: %s bytes (%s)\n' "$out" "$(wc -c < "$out" | tr -d ' ')" "$profile"
1070
+ printf '%s: %s bytes (%s%s)\n' "$out" "$(wc -c < "$out" | tr -d ' ')" "$profile" "${wasm_opt:+, $wasm_opt}"