@nimbus-sh/core 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +112 -0
  3. package/dist/_shared/tarball-stream.d.ts +93 -0
  4. package/dist/_shared/tarball-stream.d.ts.map +1 -0
  5. package/dist/_shared/tarball-stream.js +235 -0
  6. package/dist/_shared/tarball.d.ts +17 -0
  7. package/dist/_shared/tarball.d.ts.map +1 -0
  8. package/dist/_shared/tarball.js +39 -0
  9. package/dist/runtime/clang-runner.d.ts +38 -0
  10. package/dist/runtime/clang-runner.d.ts.map +1 -0
  11. package/dist/runtime/clang-runner.js +866 -0
  12. package/dist/runtime/facet-host.d.ts +12 -0
  13. package/dist/runtime/facet-host.d.ts.map +1 -1
  14. package/dist/runtime/local-facet-host.d.ts.map +1 -1
  15. package/dist/runtime/local-facet-host.js +29 -9
  16. package/dist/runtime/ruby-gems.d.ts +30 -0
  17. package/dist/runtime/ruby-gems.d.ts.map +1 -0
  18. package/dist/runtime/ruby-gems.js +636 -0
  19. package/dist/runtime/ruby-runner.d.ts +127 -0
  20. package/dist/runtime/ruby-runner.d.ts.map +1 -0
  21. package/dist/runtime/ruby-runner.js +1357 -0
  22. package/dist/runtime/session-process-supervisor.d.ts +15 -0
  23. package/dist/runtime/session-process-supervisor.d.ts.map +1 -1
  24. package/dist/runtime/session-process-supervisor.js +30 -0
  25. package/dist/workspace/nimbus-workspace.d.ts.map +1 -1
  26. package/dist/workspace/nimbus-workspace.js +9 -2
  27. package/package.json +4 -2
  28. package/src/_shared/tarball-stream.ts +263 -0
  29. package/src/_shared/tarball.ts +46 -0
  30. package/src/runtime/clang-runner.ts +924 -0
  31. package/src/runtime/facet-host.ts +12 -0
  32. package/src/runtime/local-facet-host.ts +28 -8
  33. package/src/runtime/ruby-gems.ts +682 -0
  34. package/src/runtime/ruby-runner.ts +1484 -0
  35. package/src/runtime/session-process-supervisor.ts +27 -0
  36. package/src/workspace/nimbus-workspace.ts +10 -1
@@ -0,0 +1,866 @@
1
+ /**
2
+ * clang-runner.ts — compile, link, and execute C programs for Nimbus WASI.
3
+ *
4
+ * Architecture (compile-link-run, two facet calls):
5
+ *
6
+ * compile : clang.wasm + sysroot subset for C includes + user .c
7
+ * → produces .o bytes (returned to supervisor).
8
+ * link : lld.wasm + sysroot subset for link (crt1.o + libc.a)
9
+ * + .o from compile → produces .wasm executable.
10
+ * write : final .wasm flushed to user VFS at the requested path.
11
+ *
12
+ * The filesystem both halves see is the one WASI layer every other
13
+ * non-node runtime uses (wasi-instance.ts), seeded and sealed.
14
+ *
15
+ * Splitting compile and link into separate facet calls keeps each
16
+ * call under the empirical payload ceiling. Each ships:
17
+ *
18
+ * - compile: 31 MiB clang.wasm + ~1.3 MiB sysroot subset (C includes).
19
+ * - link : 19 MiB lld.wasm + ~0.75 MiB libs + tiny .o.
20
+ *
21
+ * Sysroot subset extraction happens supervisor-side via a small ustar
22
+ * parser. The full sysroot.tar is parsed once when the clang runtime
23
+ * warms for a session; compile/link calls reuse the filtered subsets.
24
+ *
25
+ * Dispatch stays direct: no sleeps, no caller-side retries, and no
26
+ * catch-and-continue around loader failures.
27
+ */
28
+ import { CRED_KERNEL, requireVfsCred, WASM32_WASI_NIMBUS_ABI } from './os-contracts.js';
29
+ import { resolveVfsPath } from '../vfs/path.js';
30
+ import { hasLeadingCliFlag } from './cli-flags.js';
31
+ import { WASI_ABI_NAMESPACE, WASI_INSTANCE_PREAMBLE_SRC } from './wasi-instance.js';
32
+ const CLANG_VERSION_FLAGS = new Set(['--version', '-v']);
33
+ /** Build the runner factory. Closes over the facet host + vfs. */
34
+ export function makeClangRunnerFactory(deps) {
35
+ const runtimeVfs = deps.vfs.as(CRED_KERNEL);
36
+ return function clangRunnerFactory(manifest, installRoot, binName, binKind) {
37
+ const findFile = (rel) => {
38
+ const entry = manifest.files.find((f) => f.path === rel);
39
+ return entry ? `${installRoot}/${entry.path}` : null;
40
+ };
41
+ const clangVfsPath = findFile('bin/clang');
42
+ const lldVfsPath = findFile('bin/wasm-ld');
43
+ const sysrootVfsPath = findFile('share/clang/sysroot.tar');
44
+ let runtimePromise = null;
45
+ return async function clangBinHandler(ctx) {
46
+ const vfs = deps.vfs.as(requireVfsCred(ctx.cred, binName));
47
+ const argv = ctx.args || [];
48
+ const cwd = ctx.cwd || '/home/user';
49
+ // Fast paths — no wasm boot.
50
+ if (hasLeadingCliFlag(argv, CLANG_VERSION_FLAGS)) {
51
+ ctx.stdout.write(`Nimbus wasm-clang (binji-2020, LLVM 8.0.1)\n`);
52
+ ctx.stdout.write(`Target: ${WASM32_WASI_NIMBUS_ABI.id} (via wasm-ld linker)\n`);
53
+ return 0;
54
+ }
55
+ if (argv.includes('--help') || argv.includes('-h')) {
56
+ ctx.stdout.write(`usage: ${binName} [options] <source.c> -o <output>\n`);
57
+ ctx.stdout.write(`Wasm-compiled clang/wasm-ld bundle for Nimbus.\n`);
58
+ ctx.stdout.write(`Target: ${WASM32_WASI_NIMBUS_ABI.id}\n`);
59
+ ctx.stdout.write(`Supported: C compilation + linking to wasm.\n`);
60
+ return 0;
61
+ }
62
+ const isLinker = binKind === 'linker' || binName === 'wasm-ld';
63
+ // Resolve bundle paths.
64
+ if (!sysrootVfsPath || !runtimeVfs.exists(sysrootVfsPath)) {
65
+ ctx.stderr.write(`${binName}: sysroot.tar missing from install\n`);
66
+ return 127;
67
+ }
68
+ if (!clangVfsPath || !lldVfsPath) {
69
+ ctx.stderr.write(`${binName}: clang/wasm-ld missing from install\n`);
70
+ return 127;
71
+ }
72
+ // Parse argv: find input .c + output path.
73
+ const parsed = parseUserArgv(argv);
74
+ if (parsed.error) {
75
+ ctx.stderr.write(`${binName}: ${parsed.error}\n`);
76
+ return parsed.exitCode;
77
+ }
78
+ if (isLinker) {
79
+ // Direct wasm-ld invocation: pass argv through (advanced
80
+ // users only). Not on the hello-world path.
81
+ ctx.stderr.write(`${binName}: direct wasm-ld invocation not yet wired (v1.2)\n`);
82
+ return 2;
83
+ }
84
+ // Validate all user-supplied source inputs exist in the user's
85
+ // session VFS. Collect their bytes to seed the filesystem.
86
+ const userSourceFiles = {};
87
+ // Pre-built objects/archives the user passed (e.g. extra.o, libfoo.a)
88
+ // — shipped to the LINK step only (not compile).
89
+ const preBuiltLinkInputs = [];
90
+ const sourceInputs = [];
91
+ for (const input of parsed.inputPaths) {
92
+ const inputAbs = resolveVfsPath(input, cwd);
93
+ try {
94
+ if (!vfs.exists(inputAbs)) {
95
+ ctx.stderr.write(`${binName}: ${input}: No such file or directory\n`);
96
+ return 1;
97
+ }
98
+ userSourceFiles[input] = vfs.readFile(inputAbs);
99
+ }
100
+ catch (error) {
101
+ ctx.stderr.write(`${binName}: ${input}: ${errorMessage(error)}\n`);
102
+ return 1;
103
+ }
104
+ if (isSourceExt(input)) {
105
+ sourceInputs.push(input);
106
+ }
107
+ else {
108
+ // .o / .a — pass through to link as a pre-built input. The
109
+ // path in the seeded filesystem is the user-supplied relative path.
110
+ preBuiltLinkInputs.push(input);
111
+ }
112
+ }
113
+ if (sourceInputs.length === 0 && preBuiltLinkInputs.length === 0) {
114
+ ctx.stderr.write(`${binName}: no compilable / linkable inputs\n`);
115
+ return 1;
116
+ }
117
+ let runtime;
118
+ try {
119
+ if (!runtimePromise) {
120
+ runtimePromise = createClangFacetRuntime(deps.facets, {
121
+ clangVfsPath,
122
+ lldVfsPath,
123
+ sysrootVfsPath,
124
+ vfs: runtimeVfs,
125
+ });
126
+ }
127
+ runtime = await runtimePromise;
128
+ }
129
+ catch (e) {
130
+ runtimePromise = null;
131
+ ctx.stderr.write(`${binName}: clang runtime warm-up failed: ${errorMessage(e)}\n`);
132
+ return 1;
133
+ }
134
+ // Walk the user's cwd to gather headers (.h/.hpp/.hxx/.inc/...)
135
+ // and any sibling headers users typically expect to be visible
136
+ // to #include "..." resolution. Real clang/gcc auto-search the
137
+ // dir of the including source for quote-form includes; we ship
138
+ // those files at their relative-to-cwd paths so the seeded
139
+ // filesystem reproduces the user's working tree.
140
+ //
141
+ // Size-capped (4 MiB, 200 files, depth 8) so accidental
142
+ // huge-projects don't OOM the facet. Real C-tutorial projects
143
+ // are vastly under the cap.
144
+ const userIncludeBundle = collectIncludeBundle(vfs, cwd.replace(/^\/+/, ''));
145
+ // ── COMPILE PHASE ────────────────────────────────────────────
146
+ // Reuse the warm clang pool and ship only the C-include
147
+ // subset plus user source/header files for this invocation.
148
+ // Build -I flag list. We pass each user -I path verbatim AND add
149
+ // an implicit '.' (cwd) for quote-form lookup. wasm-clang's -cc1
150
+ // mode does NOT add cwd to the quote search list by default
151
+ // (the driver normally does that for "foo.h" includes), so we
152
+ // wire it ourselves. This is what makes
153
+ // clang main.c (main.c does #include "greet.h", greet.h
154
+ // next to main.c)
155
+ // succeed without an explicit -I from the user.
156
+ const userIncludeFlags = [];
157
+ for (const ip of parsed.includePaths) {
158
+ userIncludeFlags.push('-I', ip);
159
+ }
160
+ userIncludeFlags.push('-I', '.');
161
+ // Compile each source to its own .o. Object file naming: replace
162
+ // the source extension with .o. Collisions across cwd subdirs
163
+ // (e.g. src/foo.c and lib/foo.c both → foo.o) are avoided by
164
+ // keeping the directory component (the seed preserves user layout).
165
+ const objPaths = [];
166
+ const objBytesMap = {};
167
+ for (const src of sourceInputs) {
168
+ const objPath = src.replace(/\.(c|cc|cpp|cxx|c\+\+|C)$/, '.o');
169
+ // For C++ inputs use -x c++; default -x c.
170
+ const isCpp = /\.(cc|cpp|cxx|c\+\+|C)$/.test(src);
171
+ const compileArgv = [
172
+ 'clang', '-cc1', '-emit-obj',
173
+ '-disable-free',
174
+ '-isysroot', '/',
175
+ '-internal-isystem', '/include/c++/v1',
176
+ '-internal-isystem', '/include',
177
+ '-internal-isystem', '/lib/clang/8.0.1/include',
178
+ '-ferror-limit', '19',
179
+ '-fmessage-length', '80',
180
+ '-fcolor-diagnostics',
181
+ '-O2',
182
+ ...userIncludeFlags,
183
+ '-o', objPath,
184
+ '-x', isCpp ? 'c++' : 'c',
185
+ src,
186
+ ];
187
+ // Per compile we ship: the current source file + the user's
188
+ // header bundle. Multi-TU is handled at link time, not compile,
189
+ // so sibling sources stay out of the seed (smaller payload, no
190
+ // surface for unintended cross-TU textual inclusion via -I.).
191
+ const oneSourceFile = { [src]: userSourceFiles[src] };
192
+ const compileResult = await dispatchClangFacet(runtime.compile, {
193
+ sysrootFiles: {
194
+ ...runtime.compile.sysrootFiles,
195
+ ...oneSourceFile,
196
+ // User's headers from cwd tree (so quote-form #include
197
+ // resolves; this is the primary clang-include-fix payload).
198
+ ...userIncludeBundle,
199
+ },
200
+ argv: compileArgv,
201
+ outputPaths: [objPath],
202
+ });
203
+ if (compileResult.stdout)
204
+ ctx.stdout.write(compileResult.stdout);
205
+ if (compileResult.stderr)
206
+ ctx.stderr.write(compileResult.stderr);
207
+ if (compileResult.error) {
208
+ ctx.stderr.write(`${binName}: ${compileResult.error}\n`);
209
+ return 1;
210
+ }
211
+ if (compileResult.exitCode !== 0) {
212
+ return compileResult.exitCode;
213
+ }
214
+ const objBytes = compileResult.outputFiles[objPath];
215
+ if (!objBytes || objBytes.length === 0) {
216
+ ctx.stderr.write(`${binName}: compile produced no ${objPath} (internal error)\n`);
217
+ return 1;
218
+ }
219
+ objPaths.push(objPath);
220
+ objBytesMap[objPath] = objBytes;
221
+ }
222
+ // -c (compile-only): flush each .o to the user VFS, no link.
223
+ if (parsed.compileOnly) {
224
+ for (const objPath of objPaths) {
225
+ const objVfsPath = resolveVfsPath(objPath, cwd);
226
+ const parent = objVfsPath.replace(/\/[^/]+$/, '');
227
+ if (parent && parent !== objVfsPath && !vfs.exists(parent)) {
228
+ vfs.mkdir(parent, { recursive: true });
229
+ }
230
+ vfs.writeFile(objVfsPath, objBytesMap[objPath]);
231
+ }
232
+ // Honor user's -o for single-input compile-only: rename the one
233
+ // .o to the requested output if -o was passed.
234
+ if (sourceInputs.length === 1 && parsed.outputPath !== 'a.out') {
235
+ const fromVfs = resolveVfsPath(objPaths[0], cwd);
236
+ const toVfs = resolveVfsPath(parsed.outputPath, cwd);
237
+ if (fromVfs !== toVfs) {
238
+ try {
239
+ vfs.writeFile(toVfs, vfs.readFile(fromVfs));
240
+ vfs.unlink(fromVfs);
241
+ }
242
+ catch { /* best-effort */ }
243
+ }
244
+ }
245
+ return 0;
246
+ }
247
+ // Add pre-built .o / .a inputs (the user passed them on argv
248
+ // alongside .c sources, e.g. `clang main.c extra.o -o out`).
249
+ const preBuiltBytesMap = {};
250
+ for (const lp of preBuiltLinkInputs) {
251
+ preBuiltBytesMap[lp] = userSourceFiles[lp];
252
+ }
253
+ // ── LINK PHASE ───────────────────────────────────────────────
254
+ // Reuse the warm wasm-ld pool and ship only the link
255
+ // sysroot subset plus object/archive inputs for this invocation.
256
+ const stackSize = 1024 * 1024;
257
+ // User-supplied -L paths and -l libraries flow through. The user
258
+ // -L paths point at user-VFS dirs; we currently don't ship user
259
+ // libraries (they'd need their own collect step), so -l<name>
260
+ // works only against the sysroot's -L paths today. -L user-side
261
+ // would no-op silently in v1 — out of scope for this wave.
262
+ const userLinkFlags = [];
263
+ for (const lp of parsed.libraryPaths) {
264
+ userLinkFlags.push('-L', lp);
265
+ }
266
+ const linkArgv = [
267
+ 'wasm-ld',
268
+ '--no-threads',
269
+ '--export-dynamic',
270
+ '-z', `stack-size=${stackSize}`,
271
+ '-L/lib/wasm32-wasi',
272
+ // Stream-C: modern wasi-libc references __muloti4 / __divti3
273
+ // (128-bit math from utimensat's timespec arithmetic) — these
274
+ // live in compiler-rt's libclang_rt.builtins-wasm32.a at the
275
+ // clang resource dir. binji-2020's libc.a self-bundled them;
276
+ // modern doesn't, so we link compiler-rt explicitly. wasm-ld
277
+ // dead-strips unused builtins, so binji binaries are unaffected.
278
+ '-L/lib/clang/8.0.1/lib/wasi',
279
+ ...userLinkFlags,
280
+ '/lib/wasm32-wasi/crt1.o',
281
+ ...objPaths,
282
+ ...preBuiltLinkInputs,
283
+ '-lc',
284
+ ...parsed.libraries.map((l) => '-l' + l),
285
+ '-lclang_rt.builtins-wasm32',
286
+ '-o', parsed.outputPath,
287
+ ];
288
+ const linkResult = await dispatchClangFacet(runtime.link, {
289
+ sysrootFiles: { ...runtime.link.sysrootFiles, ...objBytesMap, ...preBuiltBytesMap },
290
+ argv: linkArgv,
291
+ outputPaths: [parsed.outputPath],
292
+ });
293
+ if (linkResult.stdout)
294
+ ctx.stdout.write(linkResult.stdout);
295
+ if (linkResult.stderr)
296
+ ctx.stderr.write(linkResult.stderr);
297
+ if (linkResult.error) {
298
+ ctx.stderr.write(`${binName}: ${linkResult.error}\n`);
299
+ return 1;
300
+ }
301
+ if (linkResult.exitCode !== 0) {
302
+ return linkResult.exitCode;
303
+ }
304
+ const wasmBytes = linkResult.outputFiles[parsed.outputPath];
305
+ if (!wasmBytes || wasmBytes.length === 0) {
306
+ ctx.stderr.write(`${binName}: link produced no ${parsed.outputPath} (internal error)\n`);
307
+ return 1;
308
+ }
309
+ // ── FLUSH OUTPUT ─────────────────────────────────────────────
310
+ const outVfsPath = resolveVfsPath(parsed.outputPath, cwd);
311
+ try {
312
+ const parent = outVfsPath.replace(/\/[^/]+$/, '');
313
+ if (parent && parent !== outVfsPath && !vfs.exists(parent)) {
314
+ vfs.mkdir(parent, { recursive: true });
315
+ }
316
+ vfs.writeFile(outVfsPath, wasmBytes);
317
+ vfs.chmod(outVfsPath, 0o755);
318
+ }
319
+ catch (error) {
320
+ ctx.stderr.write(`${binName}: ${parsed.outputPath}: ${errorMessage(error)}\n`);
321
+ return 1;
322
+ }
323
+ // Real linkers chmod their output executable (+x even after a
324
+ // prior chmod -x) — so `./a.out` runs with no manual chmod.
325
+ return 0;
326
+ };
327
+ };
328
+ }
329
+ /** Recognized C / C++ source extensions for input classification. */
330
+ function isSourceExt(p) {
331
+ return /\.(c|cc|cpp|cxx|c\+\+|C)$/.test(p);
332
+ }
333
+ /**
334
+ * Nimbus RUNS threaded wasm — see runtime/wasi-threads.ts — but this compiler
335
+ * cannot BUILD it. The bundled toolchain is LLVM 8 over a wasi-sdk-19 sysroot
336
+ * that ships one target directory, `lib/wasm32-wasi`, with no threads variant:
337
+ * no atomics-and-bulk-memory libc, no `libpthread.a`, and a fixed link line
338
+ * with no `--shared-memory`.
339
+ *
340
+ * Measured on the shipped sysroot, `-pthread` fell through parseUserArgv's
341
+ * catch-all for unrecognised flags, and what the user saw depended on their
342
+ * includes: `clang -pthread prog.c` on a program that does not include
343
+ * <pthread.h> built and ran with exit 0 and the flag quietly ignored, while a
344
+ * real threaded program died at `'pthread.h' file not found` — a diagnosis
345
+ * that names a missing header rather than a toolchain that has no threads at
346
+ * all, and points nowhere. Refuse at the front door instead, and say where the
347
+ * working path is, because Nimbus does run these programs once they are built
348
+ * correctly.
349
+ */
350
+ function threadedBuildRefusal(argv) {
351
+ const flag = argv.find((a) => a === '-pthread' || a === '-mthread-model' || a === '--pthread'
352
+ || /^(-target|--target)=.*threads$/.test(a));
353
+ const target = argv.findIndex((a) => a === '-target' || a === '--target');
354
+ const targetsThreads = target >= 0 && /threads$/.test(argv[target + 1] || '');
355
+ if (!flag && !targetsThreads)
356
+ return null;
357
+ return `${flag ?? `${argv[target]} ${argv[target + 1]}`}: this toolchain cannot build threaded wasm.\n`
358
+ + ` The bundled sysroot is wasm32-wasi only — it has no wasm32-wasip1-threads libc.\n`
359
+ + ` Nimbus RUNS pthread programs (mutex, condvar, join, TLS, barrier, semaphore),\n`
360
+ + ` but they must be built with a full wasi-sdk and linked against the futex shim:\n`
361
+ + ` clang --target=wasm32-wasip1-threads --sysroot=$WASI_SYSROOT -pthread \\\n`
362
+ + ` -Wl,--import-memory,--shared-memory,--max-memory=67108864 \\\n`
363
+ + ` -o prog.wasm prog.c nimbus-threads.c\n`
364
+ + ` See docs/wasi-threads.md for nimbus-threads.c and why the shim is required.`;
365
+ }
366
+ function parseUserArgv(argv) {
367
+ const inputPaths = [];
368
+ const includePaths = [];
369
+ const libraryPaths = [];
370
+ const libraries = [];
371
+ let outputPath = 'a.out';
372
+ let compileOnly = false;
373
+ // Flags that take a separate argv slot for their value.
374
+ const takesArg = new Set(['-o', '-x', '-isystem', '-include', '-isysroot',
375
+ '-target', '--target', '-std', '-MF', '-MT', '-MQ']);
376
+ for (let i = 0; i < argv.length; i++) {
377
+ const a = argv[i];
378
+ if (a === '-o' && i + 1 < argv.length) {
379
+ outputPath = argv[i + 1];
380
+ i++;
381
+ continue;
382
+ }
383
+ if (a === '-c') {
384
+ compileOnly = true;
385
+ continue;
386
+ }
387
+ // -I<path> or -I <path>
388
+ if (a === '-I' && i + 1 < argv.length) {
389
+ includePaths.push(argv[i + 1]);
390
+ i++;
391
+ continue;
392
+ }
393
+ if (a.startsWith('-I')) {
394
+ includePaths.push(a.substring(2));
395
+ continue;
396
+ }
397
+ // -L<path> or -L <path>
398
+ if (a === '-L' && i + 1 < argv.length) {
399
+ libraryPaths.push(argv[i + 1]);
400
+ i++;
401
+ continue;
402
+ }
403
+ if (a.startsWith('-L')) {
404
+ libraryPaths.push(a.substring(2));
405
+ continue;
406
+ }
407
+ // -l<name> or -l <name>
408
+ if (a === '-l' && i + 1 < argv.length) {
409
+ libraries.push(argv[i + 1]);
410
+ i++;
411
+ continue;
412
+ }
413
+ if (a.startsWith('-l')) {
414
+ libraries.push(a.substring(2));
415
+ continue;
416
+ }
417
+ // Skip recognised takes-arg flags we don't yet interpret.
418
+ if (takesArg.has(a) && i + 1 < argv.length) {
419
+ i++;
420
+ continue;
421
+ }
422
+ // Any other -flag is opaque to the parser; the user passes them
423
+ // through (we don't currently relay arbitrary flags to clang-cc1,
424
+ // see compileArgv construction).
425
+ if (a.startsWith('-'))
426
+ continue;
427
+ // Positional. If it looks like a source file, take it; else ignore.
428
+ if (isSourceExt(a)) {
429
+ inputPaths.push(a);
430
+ }
431
+ else if (a.endsWith('.o') || a.endsWith('.a')) {
432
+ // Pre-built objects/archives — treat as link-only inputs. We
433
+ // surface them as inputs so the link step picks them up; the
434
+ // compile step skips them (it only walks .c/.cc/.cpp).
435
+ inputPaths.push(a);
436
+ }
437
+ // else: drop silently (e.g. typos). clang would warn; we don't yet.
438
+ }
439
+ const threaded = threadedBuildRefusal(argv);
440
+ if (threaded) {
441
+ return {
442
+ inputPaths: [], includePaths, libraryPaths, libraries,
443
+ outputPath: '', compileOnly, exitCode: 1, error: threaded,
444
+ };
445
+ }
446
+ if (inputPaths.length === 0) {
447
+ return {
448
+ inputPaths: [], includePaths, libraryPaths, libraries,
449
+ outputPath: '', compileOnly, exitCode: 2, error: 'no input files',
450
+ };
451
+ }
452
+ return {
453
+ inputPaths, includePaths, libraryPaths, libraries,
454
+ outputPath, compileOnly, exitCode: 0,
455
+ };
456
+ }
457
+ /**
458
+ * Recognise headers / inline-include files. The compile facet ships
459
+ * these alongside the .c sources so `#include "foo.h"` (quote-form)
460
+ * resolves against the directory of the including source file — which
461
+ * is how clang / gcc behave on real Unix.
462
+ */
463
+ function isHeaderExt(name) {
464
+ return /\.(h|hh|hpp|hxx|H|inc|ipp|tcc)$/.test(name);
465
+ }
466
+ /**
467
+ * Walk a VFS directory recursively to collect headers + (optionally)
468
+ * source files, with bounded depth and total size cap, returning a
469
+ * map of root-relative-path → bytes.
470
+ *
471
+ * Layout convention: paths returned are RELATIVE TO `rootVfsPath`, so
472
+ * a header at `home/user/sub/foo.h` (when rootVfsPath is `home/user`)
473
+ * comes out as `sub/foo.h`. This matches the layout the user passes
474
+ * on argv (e.g. `clang sub/lib.c -o out`, with `lib.c` including
475
+ * `"helpers.h"` next to itself).
476
+ *
477
+ * `extra` extensions can be added (used to include `.c/.cpp/.o/.a` when
478
+ * looking under -L / sibling source dirs). Empty by default.
479
+ *
480
+ * Anti-DoS:
481
+ * - MAX_FILES = 200 (covers realistic user projects without ballooning
482
+ * the facet payload).
483
+ * - MAX_BYTES = 4 MiB.
484
+ * - MAX_DEPTH = 8 (deep enough for typical "src/", "include/", "lib/" trees).
485
+ * - skipDirs prunes obvious non-source directories.
486
+ */
487
+ function collectIncludeBundle(vfs, rootVfsPath, opts = {}) {
488
+ const extraExts = opts.extraExts ?? null;
489
+ const MAX_FILES = opts.maxFiles ?? 200;
490
+ const MAX_BYTES = opts.maxBytes ?? 4 * 1024 * 1024;
491
+ const MAX_DEPTH = opts.maxDepth ?? 8;
492
+ const out = {};
493
+ if (!vfs.exists(rootVfsPath) || !vfs.isDirectory(rootVfsPath))
494
+ return out;
495
+ // Directories pruned regardless of depth — these never contain user
496
+ // headers and would balloon the payload if traversed.
497
+ const skipDirs = new Set([
498
+ '.nimbus', 'node_modules', '.cache', '.npm', '.git',
499
+ 'dist', 'build', '.next', '.nuxt', '.svelte-kit', 'coverage',
500
+ ]);
501
+ const root = rootVfsPath.replace(/^\/+/, '').replace(/\/+$/, '');
502
+ let totalBytes = 0;
503
+ let fileCount = 0;
504
+ const stack = [{ dir: root, depth: 0 }];
505
+ while (stack.length > 0) {
506
+ const { dir, depth } = stack.pop();
507
+ if (depth > MAX_DEPTH)
508
+ continue;
509
+ let entries;
510
+ try {
511
+ entries = vfs.readdir(dir);
512
+ }
513
+ catch {
514
+ continue;
515
+ }
516
+ for (const e of entries) {
517
+ const childAbs = dir + '/' + e.name;
518
+ const rel = childAbs.startsWith(root + '/') ? childAbs.substring(root.length + 1) : childAbs;
519
+ if (e.type === 'directory') {
520
+ if (skipDirs.has(e.name))
521
+ continue;
522
+ stack.push({ dir: childAbs, depth: depth + 1 });
523
+ continue;
524
+ }
525
+ const isHeader = isHeaderExt(e.name);
526
+ const isExtra = extraExts && extraExts.test(e.name);
527
+ if (!isHeader && !isExtra)
528
+ continue;
529
+ let bytes;
530
+ try {
531
+ bytes = vfs.readFile(childAbs);
532
+ }
533
+ catch {
534
+ continue;
535
+ }
536
+ totalBytes += bytes.length;
537
+ fileCount++;
538
+ if (totalBytes > MAX_BYTES || fileCount > MAX_FILES) {
539
+ // Cap reached — stop walking but return what we have.
540
+ return out;
541
+ }
542
+ out[rel] = bytes;
543
+ }
544
+ }
545
+ return out;
546
+ }
547
+ function errorMessage(error) {
548
+ return error instanceof Error ? error.message : String(error);
549
+ }
550
+ // ── ustar parser (supervisor-side) ───────────────────────────────────
551
+ /**
552
+ * Parse a POSIX ustar archive into a path→bytes map. Trims the
553
+ * leading "/" from paths so they are seen as "include/stdio.h"
554
+ * (not "/include/stdio.h"). Directories are NOT recorded — only
555
+ * regular file entries.
556
+ */
557
+ function parseUstar(tarBytes) {
558
+ const files = new Map();
559
+ let off = 0;
560
+ while (off + 512 <= tarBytes.length) {
561
+ let nameEnd = off;
562
+ while (nameEnd < off + 100 && tarBytes[nameEnd] !== 0)
563
+ nameEnd++;
564
+ let name = '';
565
+ for (let i = off; i < nameEnd; i++)
566
+ name += String.fromCharCode(tarBytes[i]);
567
+ if (!name)
568
+ break;
569
+ const typeflag = tarBytes[off + 156];
570
+ let sizeStr = '';
571
+ for (let i = off + 124; i < off + 124 + 11; i++) {
572
+ const c = tarBytes[i];
573
+ if (c >= 0x30 && c <= 0x37)
574
+ sizeStr += String.fromCharCode(c);
575
+ }
576
+ const size = parseInt(sizeStr || '0', 8);
577
+ let prefixEnd = off + 345;
578
+ while (prefixEnd < off + 345 + 155 && tarBytes[prefixEnd] !== 0)
579
+ prefixEnd++;
580
+ let prefix = '';
581
+ for (let i = off + 345; i < prefixEnd; i++)
582
+ prefix += String.fromCharCode(tarBytes[i]);
583
+ const fullName = prefix ? `${prefix}/${name}` : name;
584
+ off += 512;
585
+ const isRegular = typeflag === 0 || typeflag === 0x30; // '0'
586
+ const isDir = typeflag === 0x35 || fullName.endsWith('/'); // '5'
587
+ if (isRegular && !isDir) {
588
+ const bytes = tarBytes.slice(off, off + size);
589
+ files.set(fullName.replace(/\/$/, ''), bytes);
590
+ }
591
+ off += Math.ceil(size / 512) * 512;
592
+ }
593
+ return files;
594
+ }
595
+ /**
596
+ * Filter the sysroot to just what the compile step needs:
597
+ * - include/ (minus include/c++/) — C system headers
598
+ * - lib/clang/8.0.1/include/ — clang intrinsic headers
599
+ *
600
+ * Excludes the C++ standard library headers (libc++/v1) which alone
601
+ * are ~4 MiB and aren't needed for plain C compilation.
602
+ */
603
+ function filterSysrootForCompile(all) {
604
+ const out = {};
605
+ for (const [path, bytes] of all.entries()) {
606
+ if (path.startsWith('include/c++/'))
607
+ continue;
608
+ if (path.startsWith('include/')) {
609
+ out[path] = bytes;
610
+ continue;
611
+ }
612
+ if (path.startsWith('lib/clang/')) {
613
+ out[path] = bytes;
614
+ continue;
615
+ }
616
+ }
617
+ return out;
618
+ }
619
+ /**
620
+ * Filter the sysroot to just what the link step needs for a C program:
621
+ * - lib/wasm32-wasi/crt1.o — the entry-point start file
622
+ * - lib/wasm32-wasi/libc.a — libc archive (printf etc.)
623
+ * - lib/wasm32-wasi/libc.imports — WASI symbol allow-list. Without
624
+ * this, wasm-ld treats `__wasi_fd_close` etc. as undefined
625
+ * symbols (the symbols are SUPPOSED to be unresolved imports,
626
+ * not errors); the .imports file tells lld "these names are
627
+ * external WASI imports, not link errors."
628
+ * - lib/clang/8.0.1/lib/wasi/libclang_rt.builtins-wasm32.a — compiler-rt
629
+ * builtins (e.g. __muloti4 for 128-bit math). Modern wasi-libc's
630
+ * utimensat.o references __muloti4; binji-2020 self-bundled it
631
+ * into libc.a, the modern build expects compiler-rt to provide.
632
+ * wasm-ld dead-strips, so trivial mains pay zero cost.
633
+ *
634
+ * Excludes libc++/libc++abi (C++-only) and the WASI emulated-mman /
635
+ * pthread / canvas variants we don't drive in v1.1.
636
+ */
637
+ function filterSysrootForLink(all) {
638
+ const out = {};
639
+ for (const [path, bytes] of all.entries()) {
640
+ if (path === 'lib/wasm32-wasi/crt1.o') {
641
+ out[path] = bytes;
642
+ continue;
643
+ }
644
+ if (path === 'lib/wasm32-wasi/libc.a') {
645
+ out[path] = bytes;
646
+ continue;
647
+ }
648
+ if (path === 'lib/wasm32-wasi/libc.imports') {
649
+ out[path] = bytes;
650
+ continue;
651
+ }
652
+ if (path === 'lib/clang/8.0.1/lib/wasi/libclang_rt.builtins-wasm32.a') {
653
+ out[path] = bytes;
654
+ continue;
655
+ }
656
+ }
657
+ return out;
658
+ }
659
+ async function createClangFacetRuntime(facets, args) {
660
+ if (!args.clangVfsPath || !args.lldVfsPath || !args.sysrootVfsPath) {
661
+ throw new Error('installed clang manifest is missing required files');
662
+ }
663
+ // Hand the file's own backing buffer to the loader when the Uint8Array
664
+ // spans it exactly (the uncached reads below always allocate a fresh
665
+ // whole buffer) — avoids a second 31 MiB copy of clang.wasm in the DO
666
+ // heap during warm-up. Falls back to a slice for sub-views.
667
+ const toAB = (u8) => (u8.byteOffset === 0 && u8.byteLength === u8.buffer.byteLength
668
+ ? u8.buffer
669
+ : u8.buffer.slice(u8.byteOffset, u8.byteOffset + u8.byteLength));
670
+ // Uncached reads: these are one-shot bulk reads of large runtime blobs
671
+ // (clang 31 MiB, wasm-ld 18.5 MiB, sysroot 9.3 MiB). Routing them
672
+ // through the LRU content cache would evict the user's hot working set
673
+ // and pin ~32 MiB of clang chunks resident in the DO heap for the whole
674
+ // session — a primary cause of supervisor-DO memory pressure that tips
675
+ // heavy sessions into an OOM reset mid-compile.
676
+ const clangBytes = args.vfs.readFileUncached(args.clangVfsPath);
677
+ const lldBytes = args.vfs.readFileUncached(args.lldVfsPath);
678
+ const sysroot = parseUstar(args.vfs.readFileUncached(args.sysrootVfsPath));
679
+ const makeTarget = (primaryName, primaryBytes, sysrootFiles) => ({
680
+ primaryName,
681
+ sysrootFiles,
682
+ facet: facets.open({
683
+ tag: `clang-runner-${primaryName}`,
684
+ concurrency: 1,
685
+ // No `syscalls`: the toolchain is sealed. It sees the sysroot subset and
686
+ // the translation unit it was handed, and the outputs are read back out
687
+ // of that filesystem — a compile cannot reach the session at all.
688
+ //
689
+ // Which is also why one warm facet may serve every tenant: with nothing
690
+ // of the session in it and nothing kept between calls, `clang main.c` is
691
+ // the same compile whoever asks.
692
+ reuse: 'global',
693
+ preamble: CLANG_RUNNER_PREAMBLE,
694
+ wasmModules: { 'primary.wasm': toAB(primaryBytes) },
695
+ }),
696
+ });
697
+ return {
698
+ compile: makeTarget('clang', clangBytes, filterSysrootForCompile(sysroot)),
699
+ link: makeTarget('wasm-ld', lldBytes, filterSysrootForLink(sysroot)),
700
+ };
701
+ }
702
+ async function dispatchClangFacet(target, args) {
703
+ // Encode sysroot files as base64 for facet transport. We do this on
704
+ // the supervisor to keep the facet preamble small and CPU-light.
705
+ const filesB64 = {};
706
+ for (const [path, bytes] of Object.entries(args.sysrootFiles)) {
707
+ filesB64[path] = uint8ToBase64(bytes);
708
+ }
709
+ const facetFn = async function clangFacetCall(inArgs) {
710
+ const wasm = Reflect.get(globalThis, '__NIMBUS_WASM');
711
+ const primaryMod = wasm?.['primary.wasm'];
712
+ if (!primaryMod) {
713
+ return {
714
+ exitCode: 127, stdout: '', stderr: '',
715
+ outputFiles: {},
716
+ error: 'clang-runner: __NIMBUS_WASM missing primary.wasm',
717
+ };
718
+ }
719
+ const fn = Reflect.get(globalThis, '__clangRun');
720
+ if (typeof fn !== 'function') {
721
+ return {
722
+ exitCode: 127, stdout: '', stderr: '',
723
+ outputFiles: {},
724
+ error: 'clang-runner preamble missing: __clangRun not in scope',
725
+ };
726
+ }
727
+ return await fn({
728
+ primaryName: inArgs.primaryName,
729
+ argv: inArgs.argv,
730
+ filesB64: inArgs.filesB64,
731
+ outputPaths: inArgs.outputPaths,
732
+ primaryMod,
733
+ });
734
+ };
735
+ try {
736
+ const result = await target.facet.submit(facetFn, {
737
+ primaryName: target.primaryName,
738
+ argv: args.argv,
739
+ filesB64,
740
+ outputPaths: args.outputPaths,
741
+ }, {
742
+ timeoutMs: 300_000,
743
+ });
744
+ // Decode outputFiles from base64 → Uint8Array.
745
+ const outputFiles = {};
746
+ for (const [path, b64] of Object.entries(result.outputFiles || {})) {
747
+ const bin = atob(b64);
748
+ const u8 = new Uint8Array(bin.length);
749
+ for (let i = 0; i < bin.length; i++)
750
+ u8[i] = bin.charCodeAt(i);
751
+ outputFiles[path] = u8;
752
+ }
753
+ return {
754
+ exitCode: result.exitCode,
755
+ stdout: result.stdout || '',
756
+ stderr: result.stderr || '',
757
+ outputFiles,
758
+ error: result.error,
759
+ };
760
+ }
761
+ catch (e) {
762
+ return {
763
+ exitCode: 1,
764
+ stdout: '',
765
+ stderr: '',
766
+ outputFiles: {},
767
+ error: `clang-runner dispatch failed: ${errorMessage(e)}`,
768
+ };
769
+ }
770
+ }
771
+ function uint8ToBase64(u8) {
772
+ // Chunked to avoid String.fromCharCode call-stack limits on big arrays.
773
+ const CHUNK = 0x8000;
774
+ let s = '';
775
+ for (let i = 0; i < u8.length; i += CHUNK) {
776
+ s += String.fromCharCode.apply(null, Array.from(u8.subarray(i, Math.min(i + CHUNK, u8.length))));
777
+ }
778
+ return btoa(s);
779
+ }
780
+ // ── Facet preamble ───────────────────────────────────────────────────
781
+ const CLANG_RUNNER_PREAMBLE_TAIL = `
782
+ // ── BEGIN: clang-runner preamble ──────────────────────────────────────
783
+ //
784
+ // The toolchain is a plain wasi_unstable (preview0) guest: clang.wasm
785
+ // declares 27 imports and wasm-ld 25, every one of them in that namespace
786
+ // and every one of them implemented by the WASI layer above. Its filesystem
787
+ // is that layer's, seeded with the sysroot subset and the translation unit
788
+ // and sealed — no supervisor is bound, so a compile cannot reach or disturb
789
+ // the session VFS, and the named outputs are read back out at the end.
790
+
791
+ globalThis.__clangRun = async function __clangRun(args) {
792
+ const stdout = [];
793
+ const stderr = [];
794
+
795
+ // Everything the seed carries is readable; directories are traversable.
796
+ // The layer denies by default for a mapped inode with no mode, and the
797
+ // producer here is a tar, which has no cred to project.
798
+ const modes = {};
799
+ const dirs = new Set();
800
+ for (const path of Object.keys(args.filesB64 || {})) {
801
+ const canon = path.replace(/^\\/+/, '');
802
+ modes[canon] = 6;
803
+ const parts = canon.split('/');
804
+ for (let i = 1; i < parts.length; i++) {
805
+ const dir = parts.slice(0, i).join('/');
806
+ dirs.add(dir);
807
+ modes[dir] = 7;
808
+ }
809
+ }
810
+ modes[''] = 7;
811
+
812
+ __wasiInitFS({
813
+ root: '',
814
+ // One preopen with the EMPTY name. That is what makes a bare relative
815
+ // input resolve: this toolchain's wasi-libc predates cwd support, so a
816
+ // relative path is matched only against a zero-length preopen name, and
817
+ // an absolute one ('-isysroot /' puts the sysroot at /include) against
818
+ // the same entry with the leading slash stripped. Naming it '/' serves
819
+ // the absolute paths and silently loses every relative one — the input
820
+ // file then fails to open with no path_open ever reaching this layer.
821
+ preopens: [{ wasiPath: '', vfsPath: '' }],
822
+ files: args.filesB64 || {},
823
+ dirs: Array.from(dirs).filter(Boolean),
824
+ modes,
825
+ });
826
+
827
+ let memory = null;
828
+ const wasi = __wasiMakeImports({
829
+ abi: 'preview0',
830
+ argv: args.argv || [],
831
+ env: { USER: 'user', HOME: '/', PWD: '/' },
832
+ getMemory: () => memory,
833
+ stdoutWrite: (s) => { stdout.push(s); },
834
+ stderrWrite: (s) => { stderr.push(s); },
835
+ });
836
+
837
+ let instance;
838
+ try {
839
+ const r = await WebAssembly.instantiate(args.primaryMod, {
840
+ ${WASI_ABI_NAMESPACE.preview0}: wasi.wasiImport,
841
+ });
842
+ instance = (r instanceof WebAssembly.Instance ? r : r.instance);
843
+ } catch (e) {
844
+ return {
845
+ exitCode: 1, stdout: stdout.join(''), stderr: stderr.join(''), outputFiles: {},
846
+ error: 'primary (' + args.primaryName + ') instantiate failed: ' + (e && e.message),
847
+ };
848
+ }
849
+ memory = instance.exports.memory;
850
+
851
+ const run = await __wasiRunStartAsync(instance, { memory });
852
+ if (run.error) {
853
+ stderr.push('[clang-runner] ' + args.primaryName + ' trapped: ' + run.error + '\\n');
854
+ }
855
+
856
+ return {
857
+ exitCode: run.exitCode,
858
+ stdout: stdout.join(''),
859
+ stderr: stderr.join(''),
860
+ outputFiles: __wasiReadFilesB64(args.outputPaths || []),
861
+ };
862
+ };
863
+
864
+ // ── END: clang-runner preamble ────────────────────────────────────────
865
+ `;
866
+ export const CLANG_RUNNER_PREAMBLE = `${WASI_INSTANCE_PREAMBLE_SRC}\n${CLANG_RUNNER_PREAMBLE_TAIL}`;