@swiftbrowser/web 0.9.0 → 0.9.1

package/src/playground/read-once.ts

@@ -0,0 +1,75 @@
+// Inputs a tool reads whole once, dropped as soon as it has. The linker
+// reads each archive into its own memory and keeps it there, so after that
+// the worker's copy is a second one, a hundred megabytes for an app's
+// archives, at the point where a phone's browser has least to spare.
+
+import { File, OpenFile, wasi } from '@bjorn3/browser_wasi_shim';
+
+/**
+ * A read-only file whose bytes go once every open of it is closed and one
+ * read all of them. Its size stays. Opening it after that fails
+ * (`onReopen` is told), since its bytes are gone: the tool has to run again
+ * with them kept.
+ */
+export class ReadOnceFile extends File {
+  private readonly length: bigint;
+  private open = 0;
+  private wholeRead = false;
+  released = false;
+
+  constructor(data: Uint8Array, private readonly onReopen: () => void) {
+    // The buffer, not the view, so the File keeps the bytes rather than a copy.
+    super(data.byteOffset === 0 && data.byteLength === data.buffer.byteLength ? data.buffer : data, { readonly: true });
+    this.length = BigInt(data.byteLength);
+  }
+
+  override get size(): bigint {
+    return this.length;
+  }
+
+  override path_open(oflags: number, fs_rights_base: bigint, fd_flags: number): { ret: number; fd_obj: OpenFile } {
+    if (this.released) {
+      this.onReopen();
+      return { ret: wasi.ERRNO_IO, fd_obj: null as unknown as OpenFile };
+    }
+    const opened = super.path_open(oflags, fs_rights_base, fd_flags);
+    if (opened.ret !== wasi.ERRNO_SUCCESS) return opened;
+    this.open += 1;
+    return { ret: opened.ret, fd_obj: new ReadOnceOpenFile(this) };
+  }
+
+  /** How an open of it ended: whether it read every byte. */
+  closed(readAll: boolean): void {
+    this.open -= 1;
+    this.wholeRead ||= readAll;
+    if (this.open === 0 && this.wholeRead) {
+      this.data = new Uint8Array(0);
+      this.released = true;
+    }
+  }
+}
+
+class ReadOnceOpenFile extends OpenFile {
+  private bytesRead = 0;
+
+  constructor(private readonly once: ReadOnceFile) {
+    super(once);
+  }
+
+  override fd_read(size: number): { ret: number; data: Uint8Array } {
+    const result = super.fd_read(size);
+    this.bytesRead += result.data.byteLength;
+    return result;
+  }
+
+  override fd_pread(size: number, offset: bigint): { ret: number; data: Uint8Array } {
+    const result = super.fd_pread(size, offset);
+    this.bytesRead += result.data.byteLength;
+    return result;
+  }
+
+  override fd_close(): number {
+    this.once.closed(BigInt(this.bytesRead) >= this.once.size);
+    return super.fd_close();
+  }
+}

package/src/playground/sections.ts

@@ -0,0 +1,77 @@
+// A WebAssembly module without its custom sections, as it streams in. A
+// tool's or plugin's run reads none of them, but a browser keeps them beside
+// the compiled code: the function names are a quarter of the compiler and of
+// the linker, and a macro plugin carries its DWARF.
+
+/** Passes a module through, dropping each custom section (id 0). */
+export function withoutCustomSections(): TransformStream<Uint8Array, Uint8Array> {
+  // The preamble's bytes still to pass, then a section header's bytes so far
+  // (its id and LEB128 size), then the section's bytes to pass or to drop.
+  let preamble = 8;
+  let header: number[] = [];
+  let pass = 0;
+  let drop = 0;
+  return new TransformStream({
+    transform(chunk, controller) {
+      let at = 0;
+      while (at < chunk.length) {
+        if (preamble > 0 || pass > 0) {
+          const count = Math.min(preamble > 0 ? preamble : pass, chunk.length - at);
+          controller.enqueue(chunk.subarray(at, at + count));
+          if (preamble > 0) preamble -= count;
+          else pass -= count;
+          at += count;
+        } else if (drop > 0) {
+          const count = Math.min(drop, chunk.length - at);
+          drop -= count;
+          at += count;
+        } else {
+          const byte = chunk[at++]!;
+          header.push(byte);
+          if (header.length === 1 || byte & 0x80) {
+            if (header.length > 6) throw new Error('a WebAssembly section size is longer than five bytes');
+            continue;
+          }
+          let size = 0;
+          for (let i = header.length - 1; i >= 1; i--) size = size * 128 + (header[i]! & 0x7f);
+          if (header[0] === 0) drop = size;
+          else {
+            controller.enqueue(new Uint8Array(header));
+            pass = size;
+          }
+          header = [];
+        }
+      }
+    },
+    flush() {
+      if (preamble > 0 || pass > 0 || drop > 0 || header.length > 0) throw new Error('a WebAssembly module ended early');
+    },
+  });
+}
+
+/** `module` without its custom sections, at once: what `withoutCustomSections` streams. */
+export function stripCustomSections(module: Uint8Array): Uint8Array {
+  const kept: Uint8Array[] = [module.subarray(0, 8)];
+  let at = 8;
+  while (at < module.length) {
+    const start = at;
+    const id = module[at++]!;
+    let size = 0;
+    for (let shift = 0; ; shift += 7) {
+      if (at >= module.length) throw new Error('a WebAssembly module ended early');
+      const byte = module[at++]!;
+      size += (byte & 0x7f) * 2 ** shift;
+      if (!(byte & 0x80)) break;
+    }
+    if (at + size > module.length) throw new Error('a WebAssembly module ended early');
+    if (id !== 0) kept.push(module.subarray(start, at + size));
+    at += size;
+  }
+  const out = new Uint8Array(kept.reduce((total, part) => total + part.length, 0));
+  let offset = 0;
+  for (const part of kept) {
+    out.set(part, offset);
+    offset += part.length;
+  }
+  return out;
+}

package/src/playground/tool-host.ts

@@ -0,0 +1,163 @@
+// Each tool run in a worker of its own, which loads the tool and the SDK's
+// and kit's files it reads from the cache (assets.ts), runs it over an
+// in-memory file system, sends back what it wrote and is ended. A phone's
+// browser runs out of memory where a desktop's does not: this way the
+// compiler's code, its memory and the Swift modules are gone before the
+// linker loads, with its own memory and the archives, and the page's worker
+// holds only the small files. A run compiles its tool again, which a
+// browser does fast enough (a WebAssembly.Module could be sent to the
+// worker instead, but its code would stay as long as the page kept it).
+
+import type { Directory } from '@bjorn3/browser_wasi_shim';
+import { type MemoryHost, type RunResult, type ToolName, directoryFiles, memoryDirectory, sdkPaths } from '@swiftbrowser/wasm-compiler';
+import type { Download } from './download';
+
+/** What a tool's worker runs. */
+export interface ToolJob {
+  /** argv, the program name first. */
+  args: string[];
+  env: Record<string, string>;
+  module: Download;
+  /** Guest directory → the files it holds, every mount the tool sees in order. */
+  files: Record<string, [string, Uint8Array][]>;
+  /** Guest directory → files of it the worker downloads. */
+  downloads: Record<string, [string, Download][]>;
+  /** The macro plugins, as guest paths of downloads. */
+  plugins: string[];
+  /** The mounts whose files the worker sends back: those a tool writes to. */
+  writable: string[];
+  /**
+   * Whether the downloads go once the tool has read them (read-once.ts): the
+   * linker's, which it keeps whole. Not those named in `keep` (file names).
+   */
+  readOnce: boolean;
+  keep: string[];
+}
+
+/** What a tool's worker is sent: a tool to run, or downloads to put in the cache. */
+export type ToToolWorker = { type: 'run'; job: ToolJob } | { type: 'prefetch'; files: Download[] };
+export type ToolReply =
+  /** `reopened`: a download the tool opened again after it was dropped (`readOnce`): it has to run again keeping them. */
+  | { type: 'done'; result: RunResult; files: Record<string, [string, Uint8Array][]>; reopened?: string }
+  /** One of a prefetch's files is in the cache, by its index. */
+  | { type: 'fetched'; index: number }
+  | { type: 'prefetched' }
+  | { type: 'error'; error: string };
+
+/**
+ * Whether a tool reads a file of the SDK or the kit: the linker their
+ * archives and objects, the compiler the rest.
+ */
+export const toolReads = (tool: ToolName, file: string): boolean => /\.[ao]$/.test(file) === (tool === 'wasm-ld');
+
+export interface ToolHostOptions {
+  tools: Partial<Record<ToolName, Download>>;
+  /** Guest directory → the files held here. */
+  mounts: Record<string, Directory>;
+  /** Guest directory → files of it a tool's worker downloads, those it reads (`toolReads`). */
+  downloads: Record<string, [string, Download][]>;
+  /** Macro plugins by guest path, for the compiler: its worker downloads them too. */
+  plugins: Record<string, Download>;
+  /** The Swift SDK's `wasm32-unknown-wasip1` directory, as a guest path. */
+  sdkRoot: string;
+  /** Clang's module cache to start from (memoryHost's). */
+  moduleCache?: Iterable<readonly [string, Uint8Array]>;
+}
+
+/** The module cache's directory under the host's /tmp, as memoryHost has it. */
+const MODULE_CACHE = 'module-cache';
+
+/**
+ * Sends `message` to a new tool worker and answers with its last reply (a
+ * `done` or `prefetched`), ending the worker then: with it goes whatever it
+ * loaded, which no browser frees as surely otherwise.
+ */
+function inWorker<Reply extends ToolReply>(message: ToToolWorker, onReply: (reply: ToolReply) => void = () => {}): Promise<Reply> {
+  const worker = new Worker(new URL('./tool-worker.ts', import.meta.url), { type: 'module' });
+  const name = message.type === 'run' ? message.job.args[0] : 'the download';
+  return new Promise<Reply>((resolve, reject) => {
+    worker.onmessage = (event: MessageEvent<ToolReply>) => {
+      const reply = event.data;
+      if (reply.type === 'error') reject(new Error(reply.error));
+      else if (reply.type === 'done' || reply.type === 'prefetched') resolve(reply as Reply);
+      else onReply(reply);
+    };
+    worker.onerror = (event) => {
+      event.preventDefault();
+      reject(new Error(`${name}'s worker failed: ${event.message || 'no message'}`));
+    };
+    worker.postMessage(message);
+  }).finally(() => worker.terminate());
+}
+
+/**
+ * Puts `files` in the cache from a worker of their own, calling `onFetched`
+ * with each one's index as it is in: the brotli decoder's memory, which only
+ * grows, goes with the worker.
+ */
+export async function prefetchInWorker(files: Download[], onFetched: (index: number) => void = () => {}): Promise<void> {
+  await inWorker({ type: 'prefetch', files }, (reply) => reply.type === 'fetched' && onFetched(reply.index));
+}
+
+/**
+ * The archives and objects a link names more than once, by file name: the
+ * linker reads such a file again each time (the SDK's flags and the
+ * objects' autolink entries both name -lwasi-emulated-mman, for one).
+ */
+export function namedTwice(args: readonly string[]): string[] {
+  const counts = new Map<string, number>();
+  for (const arg of args) {
+    const name = arg.startsWith('-l') ? `lib${arg.slice(2)}.a` : /\.[ao]$/.test(arg) ? arg.split('/').pop()! : null;
+    if (name) counts.set(name, (counts.get(name) ?? 0) + 1);
+  }
+  return [...counts].filter(([, count]) => count > 1).map(([name]) => name).sort();
+}
+
+/** A host like memoryHost (wasm-compiler) that runs each tool in its own worker. */
+export function toolHost(options: ToolHostOptions): MemoryHost {
+  const tmp = memoryDirectory(options.moduleCache ? [...options.moduleCache].map(([file, data]): [string, Uint8Array] => [`${MODULE_CACHE}/${file}`, data]) : []);
+  const plugins = Object.entries(options.plugins);
+  const pluginMounts = new Set(plugins.map(([guest]) => `/${guest.split('/')[1]}`));
+  const mounts: Record<string, Directory> = { '/': memoryDirectory([]), '/tmp': tmp, ...options.mounts };
+  for (const dir of [...Object.keys(options.downloads), ...pluginMounts]) mounts[dir] ??= memoryDirectory([]);
+  const downloads: Record<string, [string, Download][]> = { ...options.downloads };
+  for (const [guest, file] of plugins) {
+    const dir = `/${guest.split('/')[1]}`;
+    downloads[dir] = [...(downloads[dir] ?? []), [guest.slice(dir.length + 1), file]];
+  }
+  const writable = Object.keys(mounts).filter((dir) => !(dir in downloads));
+  return {
+    description: 'WebAssembly tools, each in a worker, over an in-memory file system',
+    sdk: sdkPaths(options.sdkRoot),
+    moduleCachePath: `/tmp/${MODULE_CACHE}`,
+    moduleCacheFiles() {
+      const prefix = `${MODULE_CACHE}/`;
+      return directoryFiles(mounts['/tmp']!).filter(([file]) => file.startsWith(prefix)).map(([file, data]) => [file.slice(prefix.length), data]);
+    },
+    async readFile(file: string): Promise<Uint8Array> {
+      const dir = Object.keys(mounts).filter((mount) => file.startsWith(mount === '/' ? '/' : `${mount}/`)).sort((a, b) => b.length - a.length)[0];
+      const data = dir === undefined ? undefined : directoryFiles(mounts[dir]!).find(([path]) => `${dir === '/' ? '' : dir}/${path}` === file)?.[1];
+      if (!data) throw new Error(`${file}: no such file`);
+      return data;
+    },
+    async run(tool: ToolName, args: readonly string[]): Promise<RunResult> {
+      const module = options.tools[tool];
+      if (!module) throw new Error(`no WebAssembly module for ${tool}`);
+      const job: ToolJob = {
+        args: [tool, ...args],
+        env: { TMPDIR: '/tmp' },
+        module,
+        files: Object.fromEntries(Object.entries(mounts).map(([dir, contents]) => [dir, directoryFiles(contents)])),
+        downloads: Object.fromEntries(Object.entries(downloads).map(([dir, list]) => [dir, list.filter(([file]) => toolReads(tool, file))])),
+        plugins: tool === 'swift-frontend' ? plugins.map(([guest]) => guest) : [],
+        writable,
+        readOnce: tool === 'wasm-ld',
+        keep: namedTwice(args),
+      };
+      let reply = await inWorker<Extract<ToolReply, { type: 'done' }>>({ type: 'run', job });
+      if (reply.reopened) reply = await inWorker<Extract<ToolReply, { type: 'done' }>>({ type: 'run', job: { ...job, readOnce: false } });
+      for (const [dir, files] of Object.entries(reply.files)) mounts[dir] = memoryDirectory(files);
+      return reply.result;
+    },
+  };
+}

package/src/playground/tool-worker.ts

@@ -0,0 +1,67 @@
+// A tool's worker (tool-host.ts): loads the tool and the files it reads from
+// the cache, runs it once, sends back the files of the mounts it writes to.
+// Or it puts downloads in the cache. The page's worker ends it then, and with
+// it everything it loaded.
+
+import { Directory, File } from '@bjorn3/browser_wasi_shim';
+import { directoryFiles, memoryDirectory, runMemoryCommand } from '@swiftbrowser/wasm-compiler';
+import { CONCURRENCY, compileModule, download, downloadBytes, openCache, pooled, prefetch } from './download';
+import { ReadOnceFile } from './read-once';
+import type { ToToolWorker, ToolJob, ToolReply } from './tool-host';
+
+const post = (message: ToolReply, transfer: Transferable[] = []): void => (self as unknown as Worker).postMessage(message, transfer);
+
+/**
+ * The tool's mounts and its plugins compiled, with the downloads in their
+ * places; nothing else keeps the downloads' bytes once this returns. With
+ * `job.readOnce` the downloads (but `job.keep`) are dropped as soon as the
+ * tool has read them (read-once.ts), and `onReopen` hears of one opened
+ * again after.
+ */
+async function load(job: ToolJob, cache: Cache | null, onReopen: (path: string) => void): Promise<{ mounts: Record<string, Directory>; plugins: Record<string, WebAssembly.Module> }> {
+  const wanted = Object.entries(job.downloads).flatMap(([dir, list]) => list.map(([file, from]) => ({ dir, file, from })));
+  const downloaded = await pooled(wanted, CONCURRENCY, async ({ dir, file, from }) => ({ dir, file, data: await downloadBytes(cache, from) }));
+  const mounts = Object.fromEntries(Object.entries(job.files).map(([dir, list]) => [dir, memoryDirectory(list)]));
+  for (const { dir, file, data } of downloaded) {
+    const parts = file.split('/');
+    let parent = (mounts[dir] ??= memoryDirectory([]));
+    for (const part of parts.slice(0, -1)) {
+      const next = parent.contents.get(part) ?? new Directory(new Map());
+      parent.contents.set(part, next);
+      parent = next as Directory;
+    }
+    // The buffer (a download's bytes are all of it), so the File keeps it rather than a copy.
+    const once = job.readOnce && !job.keep.includes(parts.at(-1)!);
+    parent.contents.set(parts.at(-1)!, once ? new ReadOnceFile(data, () => onReopen(`${dir}/${file}`)) : new File(data.buffer));
+  }
+  const plugins = Object.fromEntries(await Promise.all(job.plugins.map(async (guest) => {
+    const { data } = downloaded.find(({ dir, file }) => `${dir}/${file}` === guest)!;
+    return [guest, await WebAssembly.compile(data)] as const;
+  })));
+  return { mounts, plugins };
+}
+
+async function run(job: ToolJob): Promise<void> {
+  const cache = await openCache();
+  let reopened: string | undefined;
+  const [module, { mounts, plugins }] = await Promise.all([download(cache, job.module).then(compileModule), load(job, cache, (path) => (reopened ??= path))]);
+  const result = await runMemoryCommand({ module, args: job.args, mounts, env: job.env, plugins });
+  const written = Object.fromEntries(job.writable.map((dir) => [dir, directoryFiles(mounts[dir]!)]));
+  const buffers = new Set(Object.values(written).flatMap((list) => list.map(([, data]) => data.buffer as ArrayBuffer)));
+  post({ type: 'done', result, files: written, ...(reopened ? { reopened } : {}) }, [...buffers]);
+}
+
+self.onmessage = async (event: MessageEvent<ToToolWorker>) => {
+  const message = event.data;
+  try {
+    if (message.type === 'run') return await run(message.job);
+    const cache = await openCache();
+    await pooled(message.files.map((file, index) => ({ file, index })), CONCURRENCY, async ({ file, index }) => {
+      await prefetch(cache, file);
+      post({ type: 'fetched', index });
+    });
+    post({ type: 'prefetched' });
+  } catch (error) {
+    post({ type: 'error', error: error instanceof Error ? (error.stack ?? error.message) : String(error) });
+  }
+};

package/src/playground/worker.ts

@@ -1,46 +1,70 @@
 // The playground's compiler, in a worker: the page sends an app's sources,
-// the worker answers with the linked module or the diagnostics. The tools
-// and files are loaded once and kept, and each compile starts from the
-// module cache the previous one left. The macro plugins arrive after the
-// tools: a compile that expands a macro before then waits for them and
-// runs again.
+// the worker answers with the linked module or the diagnostics. The files
+// are downloaded once, and each compile starts from the module cache the
+// previous one left; each tool runs in a worker of its own (tool-host.ts).
+// The macro plugins load only when a compile expands a macro: it fails for
+// want of them, waits for them and runs again. Until then they are only
+// downloaded into the cache, once a compile is done.
 
-import { type AppSources, missingPlugin, buildApp } from './build';
+import { memoryDirectory } from '@swiftbrowser/wasm-compiler';
 import { type LoadedToolchain, loadToolchain } from './assets';
+import { type AppSources, type Toolchain, buildApp, missingPlugin } from './build';
 import type { PlaygroundSource } from './files';
+import { toolHost } from './tool-host';
 
 export type ToWorker = { type: 'compile'; id: number; source: PlaygroundSource; app: AppSources };
 export type FromWorker =
   | { type: 'progress'; text: string }
   | { type: 'warning'; text: string }
-  | { type: 'result'; id: number; ok: boolean; output: string; wasm?: Uint8Array<ArrayBuffer>; steps: { tool: string; seconds: number }[] }
+  | { type: 'result'; id: number; ok: boolean; output: string; wasm?: Uint8Array<ArrayBuffer>; steps: { tool: string; seconds: number; memoryBytes?: number }[] }
   | { type: 'failure'; id: number; error: string };
 
 const post = (message: FromWorker, transfer: Transferable[] = []): void => (self as unknown as Worker).postMessage(message, transfer);
 let loaded: Promise<LoadedToolchain> | null = null;
 let moduleCache: [string, Uint8Array][] | null = null;
+/** Whether compiles load the macro plugins: once one has wanted them. */
+let withPlugins = false;
+
+/** The toolchain the compiles run on, with or without the plugins. */
+function toolchain(loaded: LoadedToolchain, plugins: boolean): Toolchain {
+  const names = plugins ? Object.keys(loaded.plugins) : [];
+  return {
+    host: ({ mounts, moduleCache }) =>
+      toolHost({
+        tools: loaded.tools,
+        mounts: { ...mounts, '/sdk': memoryDirectory(loaded.sdk.files) },
+        downloads: { '/sdk': loaded.sdk.downloads, '/kit': loaded.kit },
+        plugins: Object.fromEntries(names.map((name) => [`/plugins/${name}.wasm`, loaded.plugins[name]!])),
+        sdkRoot: '/sdk',
+        ...(moduleCache ? { moduleCache } : {}),
+      }),
+    plugins: names,
+    kitManifest: loaded.kitManifest,
+    icu: loaded.icu,
+  };
+}
 
 self.onmessage = async (event: MessageEvent<ToWorker>) => {
   const { id, source, app } = event.data;
   try {
     loaded ??= loadToolchain(source, (text) => post({ type: 'progress', text })).then((toolchain) => {
-      // Later compiles use the plugins as soon as they are in.
-      toolchain.plugins.then((plugins) => (toolchain.toolchain.plugins = plugins), () => {});
       if (toolchain.warning) post({ type: 'warning', text: toolchain.warning });
       return toolchain;
     });
-    const { toolchain, plugins, moduleCache: prebuilt } = await loaded;
-    post({ type: 'progress', text: `compiling ${app.name}…` });
-    let result = await buildApp(toolchain, app, { moduleCache: moduleCache ?? prebuilt });
+    const files = await loaded;
+    const onStep = (tool: string): void => post({ type: 'progress', text: `${tool === 'wasm-ld' ? 'linking' : 'compiling'} ${app.name}${withPlugins ? ' with the macro plugins' : ''}…` });
+    let result = await buildApp(toolchain(files, withPlugins), app, { moduleCache: moduleCache ?? files.moduleCache, onStep });
     moduleCache = result.moduleCache;
-    if (!result.ok && Object.keys(toolchain.plugins).length === 0 && missingPlugin(result.output)) {
+    if (!result.ok && !withPlugins && missingPlugin(result.output)) {
       post({ type: 'progress', text: 'waiting for the macro plugins…' });
-      toolchain.plugins = await plugins;
-      post({ type: 'progress', text: `compiling ${app.name} with the macro plugins…` });
-      result = await buildApp(toolchain, app, { moduleCache });
+      await files.prefetchPlugins();
+      withPlugins = true;
+      result = await buildApp(toolchain(files, withPlugins), app, { moduleCache, onStep });
       moduleCache = result.moduleCache;
     }
-    const steps = result.steps.map(({ tool, seconds }) => ({ tool, seconds }));
+    // Behind the result: a macro later finds its plugin downloaded.
+    if (!withPlugins) void files.prefetchPlugins().catch(() => {});
+    const steps = result.steps.map(({ tool, seconds, memoryBytes }) => ({ tool, seconds, ...(memoryBytes !== undefined ? { memoryBytes } : {}) }));
     if (result.wasm) {
       const wasm = new Uint8Array(result.wasm);
       post({ type: 'result', id, ok: result.ok, output: result.output, wasm, steps }, [wasm.buffer]);