@swiftbrowser/web 0.8.0 → 0.9.1

package/node/dist/src/playground/sections.js

@@ -0,0 +1,88 @@
+// A WebAssembly module without its custom sections, as it streams in. A
+// tool's or plugin's run reads none of them, but a browser keeps them beside
+// the compiled code: the function names are a quarter of the compiler and of
+// the linker, and a macro plugin carries its DWARF.
+/** Passes a module through, dropping each custom section (id 0). */
+export function withoutCustomSections() {
+    // The preamble's bytes still to pass, then a section header's bytes so far
+    // (its id and LEB128 size), then the section's bytes to pass or to drop.
+    let preamble = 8;
+    let header = [];
+    let pass = 0;
+    let drop = 0;
+    return new TransformStream({
+        transform(chunk, controller) {
+            let at = 0;
+            while (at < chunk.length) {
+                if (preamble > 0 || pass > 0) {
+                    const count = Math.min(preamble > 0 ? preamble : pass, chunk.length - at);
+                    controller.enqueue(chunk.subarray(at, at + count));
+                    if (preamble > 0)
+                        preamble -= count;
+                    else
+                        pass -= count;
+                    at += count;
+                }
+                else if (drop > 0) {
+                    const count = Math.min(drop, chunk.length - at);
+                    drop -= count;
+                    at += count;
+                }
+                else {
+                    const byte = chunk[at++];
+                    header.push(byte);
+                    if (header.length === 1 || byte & 0x80) {
+                        if (header.length > 6)
+                            throw new Error('a WebAssembly section size is longer than five bytes');
+                        continue;
+                    }
+                    let size = 0;
+                    for (let i = header.length - 1; i >= 1; i--)
+                        size = size * 128 + (header[i] & 0x7f);
+                    if (header[0] === 0)
+                        drop = size;
+                    else {
+                        controller.enqueue(new Uint8Array(header));
+                        pass = size;
+                    }
+                    header = [];
+                }
+            }
+        },
+        flush() {
+            if (preamble > 0 || pass > 0 || drop > 0 || header.length > 0)
+                throw new Error('a WebAssembly module ended early');
+        },
+    });
+}
+/** `module` without its custom sections, at once: what `withoutCustomSections` streams. */
+export function stripCustomSections(module) {
+    const kept = [module.subarray(0, 8)];
+    let at = 8;
+    while (at < module.length) {
+        const start = at;
+        const id = module[at++];
+        let size = 0;
+        for (let shift = 0;; shift += 7) {
+            if (at >= module.length)
+                throw new Error('a WebAssembly module ended early');
+            const byte = module[at++];
+            size += (byte & 0x7f) * 2 ** shift;
+            if (!(byte & 0x80))
+                break;
+        }
+        if (at + size > module.length)
+            throw new Error('a WebAssembly module ended early');
+        if (id !== 0)
+            kept.push(module.subarray(start, at + size));
+        at += size;
+    }
+    const out = new Uint8Array(kept.reduce((total, part) => total + part.length, 0));
+    let offset = 0;
+    for (const part of kept) {
+        out.set(part, offset);
+        offset += part.length;
+    }
+    return out;
+}
+//# sourceMappingURL=sections.js.map

package/node/dist/src/playground/sections.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sections.js","sourceRoot":"","sources":["../../../../src/playground/sections.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,6EAA6E;AAC7E,6EAA6E;AAC7E,oDAAoD;AAEpD,oEAAoE;AACpE,MAAM,UAAU,qBAAqB;IACnC,2EAA2E;IAC3E,yEAAyE;IACzE,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,OAAO,IAAI,eAAe,CAAC;QACzB,SAAS,CAAC,KAAK,EAAE,UAAU;YACzB,IAAI,EAAE,GAAG,CAAC,CAAC;YACX,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,QAAQ,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;oBAC1E,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;oBACnD,IAAI,QAAQ,GAAG,CAAC;wBAAE,QAAQ,IAAI,KAAK,CAAC;;wBAC/B,IAAI,IAAI,KAAK,CAAC;oBACnB,EAAE,IAAI,KAAK,CAAC;gBACd,CAAC;qBAAM,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;oBACpB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;oBAChD,IAAI,IAAI,KAAK,CAAC;oBACd,EAAE,IAAI,KAAK,CAAC;gBACd,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,EAAE,CAAE,CAAC;oBAC1B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC;wBACvC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;4BAAE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;wBAC/F,SAAS;oBACX,CAAC;oBACD,IAAI,IAAI,GAAG,CAAC,CAAC;oBACb,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;wBAAE,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,GAAG,IAAI,CAAC,CAAC;oBACrF,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;wBAAE,IAAI,GAAG,IAAI,CAAC;yBAC5B,CAAC;wBACJ,UAAU,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;wBAC3C,IAAI,GAAG,IAAI,CAAC;oBACd,CAAC;oBACD,MAAM,GAAG,EAAE,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK;YACH,IAAI,QAAQ,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACrH,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,mBAAmB,CAAC,MAAkB;IACpD,MAAM,IAAI,GAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACnD,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,EAAE,CAAE,CAAC;QACzB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,KAAK,GAAG,CAAC,GAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACjC,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YAC7E,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,EAAE,CAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC;YACnC,IAAI,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;gBAAE,MAAM;QAC5B,CAAC;QACD,IAAI,EAAE,GAAG,IAAI,GAAG,MAAM,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACnF,IAAI,EAAE,KAAK,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAC3D,EAAE,IAAI,IAAI,CAAC;IACb,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACjF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACtB,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@swiftbrowser/web",
-  "version": "0.8.0",
+  "version": "0.9.1",
   "type": "module",
   "description": "SwiftBrowser web renderer and dev server: applies render ops from a Swift/Wasm app to the DOM inside an iPhone frame.",
   "repository": {
@@ -56,6 +56,7 @@
     "@playwright/test": "^1.56.0",
     "@types/node": "^26.6.4",
     "@types/qrcode": "^1.5.6",
+    "brotli-dec-wasm": "^2.3.2",
     "typescript": "^5.9.2",
     "vitest": "^5.0.3"
   }

package/plugins/playground.ts

@@ -3,33 +3,55 @@
  * by the WebAssembly compiler (docs/in-browser-compiler.md, Milestone 3).
  * Only this repository's site has it: its code imports
  * @swiftbrowser/wasm-compiler and the CLI's browser entry from their sources
- * (the aliases below). playground/build-assets.ts writes the assets and
- * their manifest to a directory; the build reads the manifest, and the page
- * fetches the files from /playground-assets/, where the dev server serves
- * that directory, or, deployed, from where they were uploaded
- * (playground/publish-assets.mjs).
+ * (the aliases below). The page fetches the npm packages it reads
+ * (src/playground/files.ts) from a CDN that serves them file by file, at one
+ * version, except those the site serves itself (`localPackages`): the kit
+ * and @swiftbrowser/playground made from the same commit as the site's
+ * renderer, so the two cannot drift, and any package the CDN does not have
+ * at that version, the compiler's included. The dev server serves this repository's packages at
+ * /npm/, laid out as the CDN serves them, and a built site carries the
+ * files the page reads of its local packages there.
  */
 
-import { createReadStream, existsSync, readFileSync, statSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { createReadStream, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, statSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
+import { brotliCompressSync, brotliDecompressSync, constants } from 'node:zlib';
 import type { Plugin } from 'vite';
-import { PLAYGROUND_MANIFEST, type PlaygroundFiles } from '../src/playground/files.js';
+import {
+  PLAYGROUND_ICU,
+  PLAYGROUND_MANIFEST,
+  PLAYGROUND_MODULE_CACHE,
+  PLAYGROUND_PACKAGE,
+  SITE_BROTLI,
+  isBrotli,
+  type PackageFile,
+  type PlaygroundManifest,
+  type PlaygroundSource,
+} from '../src/playground/files.js';
+import { stripCustomSections } from '../src/playground/sections.js';
 
-export interface PlaygroundOptions {
-  /** The directory build-assets wrote: its manifest, and the files the dev server serves at /playground-assets/. */
-  assetsDir: string;
-  /** Where the page fetches the assets from; default /playground-assets/. */
-  assetsUrl?: string;
-}
+export type PlaygroundOptions = PlaygroundSource;
 
-/** The downloads the assets' manifest names. */
-export function playgroundFiles(options: PlaygroundOptions): PlaygroundFiles {
-  const manifest = path.join(options.assetsDir, PLAYGROUND_MANIFEST);
-  if (!existsSync(manifest)) throw new Error(`${manifest} is missing: build the playground's assets (playground/build-assets.ts)`);
-  return JSON.parse(readFileSync(manifest, 'utf8')) as PlaygroundFiles;
-}
+/** Where the dev server serves the workspaces' packages, and a built site its local ones. */
+export const PLAYGROUND_NPM_PATH = '/npm/';
+/** The kit, which must come from the same commit as the renderer. */
+export const KIT_PACKAGE = '@swiftbrowser/kit';
+/**
+ * Files a site stores brotli-compressed past this size: Cloudflare Pages
+ * serves files of up to 25 MiB.
+ */
+export const SITE_FILE_LIMIT = 24 * 1024 * 1024;
+/** Bumped whenever writeLocalPackages writes other files for the same packages. */
+const SITE_LAYOUT = 3;
 
-export const PLAYGROUND_ASSETS_PATH = '/playground-assets/';
+/**
+ * Files a site stores brotli-compressed from this size, when compressing
+ * saves a tenth: a static host may not compress what it serves in transit
+ * (Cloudflare Pages leaves application/octet-stream as it is), and the
+ * archives and modules compress to about a fifth.
+ */
+export const SITE_COMPRESS_FROM = 64 * 1024;
 
 /** The source aliases of the playground's imports, from the repository root. */
 export function playgroundAliases(repoRoot: string): Record<string, string> {
@@ -39,18 +61,152 @@ export function playgroundAliases(repoRoot: string): Record<string, string> {
   };
 }
 
-export function playgroundAssets(options: PlaygroundOptions): Plugin {
+/** The workspaces' package directories by package name. */
+export function workspacePackages(repoRoot: string): Map<string, string> {
+  const dirs = new Map<string, string>();
+  const packagesDir = path.join(repoRoot, 'packages');
+  for (const name of readdirSync(packagesDir)) {
+    const manifest = path.join(packagesDir, name, 'package.json');
+    if (existsSync(manifest)) dirs.set((JSON.parse(readFileSync(manifest, 'utf8')) as { name: string }).name, path.join(packagesDir, name));
+  }
+  return dirs;
+}
+
+/** `@scope/name@version/file` → [name, file], whatever the version. */
+export function packageRequest(pathname: string): [string, string] | null {
+  const match = /^(@[^/]+\/[^/@]+)@[^/]+\/(.+)$/.exec(pathname);
+  return match ? [match[1]!, match[2]!] : null;
+}
+
+/** Every file of `manifest` by package, as the package holds it. */
+export function manifestFiles(manifest: PlaygroundManifest): Map<string, string[]> {
+  const files = new Map<string, string[]>();
+  const add = ({ package: name, file }: PackageFile): void => void (files.get(name) ?? files.set(name, []).get(name)!).push(file);
+  for (const file of [manifest.frontend, manifest.linker, ...Object.values(manifest.plugins)]) add(file);
+  for (const [name, list] of Object.entries(manifest.sdk)) for (const file of list) add({ package: name, file: `wasm32-unknown-wasip1/${file}` });
+  for (const file of [...manifest.kit, 'kit.json']) add({ package: KIT_PACKAGE, file });
+  for (const file of [PLAYGROUND_MANIFEST, PLAYGROUND_ICU, PLAYGROUND_MODULE_CACHE]) add({ package: PLAYGROUND_PACKAGE, file });
+  return files;
+}
+
+/**
+ * The packages the site serves itself: @swiftbrowser/playground and the kit
+ * always, and `others`, each at /npm/<name>@site-<hash>/, the hash naming
+ * what the workspaces hold so a browser's cache keeps no stale file.
+ */
+export function localPackages(repoRoot: string, others: readonly string[] = []): Record<string, string> {
+  const dirs = workspacePackages(repoRoot);
+  const own = dirs.get(PLAYGROUND_PACKAGE)!;
+  const manifestFile = path.join(own, PLAYGROUND_MANIFEST);
+  if (!existsSync(manifestFile)) throw new Error(`${manifestFile} is missing: run \`npm run playground:assets --workspace @swiftbrowser/web\``);
+  const hash = createHash('sha256');
+  for (const file of [PLAYGROUND_MANIFEST, PLAYGROUND_ICU, PLAYGROUND_MODULE_CACHE]) hash.update(readFileSync(path.join(own, file)));
+  hash.update(readFileSync(path.join(dirs.get(KIT_PACKAGE)!, 'kit.json')));
+  const names = [...new Set([PLAYGROUND_PACKAGE, KIT_PACKAGE, ...others])].sort();
+  for (const name of names) hash.update(`${name}\n`);
+  // And how writeLocalPackages lays them out: another layout is other files.
+  hash.update(`layout ${SITE_LAYOUT} ${SITE_FILE_LIMIT} ${SITE_COMPRESS_FROM}\n`);
+  const version = `site-${hash.digest('hex').slice(0, 12)}`;
+  return Object.fromEntries(names.map((name) => [name, `${PLAYGROUND_NPM_PATH}${name}@${version}/`]));
+}
+
+/** `data` brotli-compressed at `quality`, with the window the tools are published with. */
+const brotli = (data: Uint8Array, quality: number): Uint8Array =>
+  brotliCompressSync(data, { params: { [constants.BROTLI_PARAM_QUALITY]: quality, [constants.BROTLI_PARAM_LGWIN]: 24, [constants.BROTLI_PARAM_SIZE_HINT]: data.length } });
+
+/**
+ * Writes the files the page reads of each local package under `outDir`, at
+ * the paths `packages` names: brotli-compressed as `<file>.brotli` from
+ * SITE_COMPRESS_FROM (and always over the size limit), and playground.json
+ * naming those (`brotli`). A tool or plugin goes without its custom
+ * sections, which the page drops anyway (sections.ts), as
+ * `<module>.brotli`: so the compiler fits the size limit, compressed harder
+ * when it must, and the site can serve it when npm has not got this one.
+ */
+export function writeLocalPackages(repoRoot: string, packages: Record<string, string>, outDir: string, limit = SITE_FILE_LIMIT, compressFrom = SITE_COMPRESS_FROM): { files: number; bytes: number } {
+  const dirs = workspacePackages(repoRoot);
+  const manifest = JSON.parse(readFileSync(path.join(dirs.get(PLAYGROUND_PACKAGE)!, PLAYGROUND_MANIFEST), 'utf8')) as PlaygroundManifest;
+  const key = (file: PackageFile): string => `${file.package}/${file.file}`;
+  const modules = new Set([manifest.frontend, manifest.linker, ...Object.values(manifest.plugins)].map(key));
+  /** Files stored under another name: brotli-compressed by the site. */
+  const renamed = new Map<string, string>();
+  const rename = (file: PackageFile): PackageFile => ({ ...file, file: renamed.get(key(file)) ?? file.file });
+  const written = { files: 0, bytes: 0 };
+  for (const [name, files] of manifestFiles(manifest)) {
+    const base = packages[name];
+    if (!base) continue;
+    const dir = dirs.get(name);
+    if (!dir) throw new Error(`the site serves ${name}, which is not a package of this repository`);
+    for (const file of files) {
+      if (name === PLAYGROUND_PACKAGE && file === PLAYGROUND_MANIFEST) continue;
+      let data: Uint8Array = readFileSync(path.join(dir, file));
+      let target = file;
+      if (modules.has(`${name}/${file}`)) {
+        const module = stripCustomSections(isBrotli(file) ? brotliDecompressSync(data) : data);
+        data = brotli(module, 5);
+        if (data.length > limit) data = brotli(module, 11);
+        if (data.length > limit) throw new Error(`${name}/${file} is over ${limit} bytes even compressed, without its custom sections`);
+        target = `${file.replace(/\.br$/, '')}${SITE_BROTLI}`;
+      } else if (isBrotli(file)) {
+        if (data.length > limit) throw new Error(`${name}/${file} is over ${limit} bytes compressed: the site cannot serve it`);
+      } else if (data.length >= compressFrom || data.length > limit) {
+        const compressed = brotli(data, 5);
+        if (compressed.length > limit) throw new Error(`${name}/${file} is over ${limit} bytes even compressed`);
+        if (data.length > limit || compressed.length < data.length * 0.9) {
+          data = compressed;
+          target = `${file}${SITE_BROTLI}`;
+        }
+      }
+      if (target !== file) renamed.set(`${name}/${file}`, target);
+      const out = path.join(outDir, base, target);
+      mkdirSync(path.dirname(out), { recursive: true });
+      writeFileSync(out, data);
+      written.files += 1;
+      written.bytes += data.length;
+    }
+  }
+  if (packages[PLAYGROUND_PACKAGE]) {
+    // A tool or plugin is named by its compressed file; the rest by the list.
+    const listed = [...renamed.keys()].filter((entry) => !modules.has(entry)).sort();
+    const site: PlaygroundManifest = {
+      ...manifest,
+      frontend: rename(manifest.frontend),
+      linker: rename(manifest.linker),
+      plugins: Object.fromEntries(Object.entries(manifest.plugins).map(([macro, file]) => [macro, rename(file)])),
+      ...(listed.length > 0 ? { brotli: listed } : {}),
+    };
+    const out = path.join(outDir, packages[PLAYGROUND_PACKAGE], PLAYGROUND_MANIFEST);
+    mkdirSync(path.dirname(out), { recursive: true });
+    writeFileSync(out, `${JSON.stringify(site)}\n`);
+    written.files += 1;
+  }
+  return written;
+}
+
+export function playgroundPackages(repoRoot: string, options: PlaygroundOptions): Plugin {
+  let outDir = '';
   return {
-    name: 'swiftbrowser-playground-assets',
+    name: 'swiftbrowser-playground-packages',
+    configResolved(config) {
+      outDir = path.resolve(config.root, config.build.outDir);
+    },
     configureServer(server) {
-      const dir = path.resolve(options.assetsDir);
-      server.middlewares.use(PLAYGROUND_ASSETS_PATH, (req, res, next) => {
-        const file = path.join(dir, decodeURIComponent(new URL(req.url ?? '/', 'http://x').pathname));
-        if (!file.startsWith(dir) || !existsSync(file) || statSync(file).isDirectory()) return next();
-        res.setHeader('Content-Type', file.endsWith('.gz') ? 'application/gzip' : 'application/octet-stream');
+      const packages = workspacePackages(repoRoot);
+      server.middlewares.use(PLAYGROUND_NPM_PATH, (req, res, next) => {
+        const request = packageRequest(decodeURIComponent(new URL(req.url ?? '/', 'http://x').pathname).replace(/^\//, ''));
+        const dir = request && packages.get(request[0]);
+        if (!request || !dir) return next();
+        const file = path.join(dir, request[1]);
+        if (!existsSync(file) || statSync(file).isDirectory() || !realpathSync(file).startsWith(realpathSync(dir))) return next();
+        res.setHeader('Content-Type', file.endsWith('.wasm') ? 'application/wasm' : file.endsWith('.json') ? 'application/json' : 'application/octet-stream');
         res.setHeader('Content-Length', statSync(file).size);
         createReadStream(file).pipe(res);
       });
     },
+    writeBundle() {
+      if (!options.packages) return;
+      const { files, bytes } = writeLocalPackages(repoRoot, options.packages, outDir);
+      this.info(`the playground's local packages: ${files} files, ${(bytes / 1e6).toFixed(1)} MB`);
+    },
   };
 }

package/src/playground/assets.ts

@@ -1,96 +1,116 @@
-// The playground's downloads (playground/build-assets.ts makes them): the
-// compiler, the linker, the macro plugins and the pack of SDK and kit files,
-// fetched from one base URL (a public bucket, or the dev server's copy) and
-// kept in the Cache API, so a compile needs no server request after the
-// first visit. Each file is gzipped, unpacked here as it arrives (no server
-// or CDN has to set Content-Encoding), and named by its contents, so a new
-// build is a new download.
+// The playground's downloads (files.ts): its manifest from
+// @swiftbrowser/playground, then the tools, the SDK's and the kit's files it
+// names, the ICU data and the module cache, every one a file of an npm
+// package, from the CDN at the source's version or from the site itself,
+// into the cache (download.ts). The page's worker holds only the small
+// files: each tool's own worker reads the rest from the cache (tool-host.ts).
 
-import { memoryDirectory } from '@swiftbrowser/wasm-compiler';
 import type { Toolchain } from './build';
-import type { PlaygroundFiles } from './files';
+import { type Download, cacheKey, downloadBytes, openCache } from './download';
+import {
+  PLAYGROUND_ICU,
+  PLAYGROUND_MANIFEST,
+  PLAYGROUND_MODULE_CACHE,
+  PLAYGROUND_PACKAGE,
+  SITE_BROTLI,
+  type PackageFile,
+  type PlaygroundManifest,
+  type PlaygroundSource,
+  packageUrl,
+} from './files';
 import { readPack } from './pack';
+import { prefetchInWorker } from './tool-host';
 
-/** What the pack's index records besides its files. */
-export interface PlaygroundMeta {
-  /** The symbol Foundation's ICU reads its data through (`icudtNN_dat`). */
-  icuSymbol: string;
-  /** The apps whose compiles the pack was recorded from. */
-  apps: string[];
-}
-
-const CACHE = 'swiftbrowser-playground';
-
-/** A response for `url`, from the cache when it has one. */
-async function cached(url: string): Promise<Response> {
-  const cache = typeof caches === 'undefined' ? null : await caches.open(CACHE).catch(() => null);
-  const hit = await cache?.match(url);
-  if (hit) return hit;
-  const response = await fetch(url);
-  if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`);
-  await cache?.put(url, response.clone()).catch(() => {});
-  return response;
-}
-
-/** The file at `url`, unpacked when it is gzipped. */
-async function body(url: string): Promise<Response> {
-  const response = await cached(url);
-  if (!url.endsWith('.gz') || !response.body) return response;
-  const type = url.endsWith('.wasm.gz') ? 'application/wasm' : 'application/octet-stream';
-  return new Response(response.body.pipeThrough(new DecompressionStream('gzip')), { headers: { 'content-type': type } });
-}
-
-async function compileModule(url: string): Promise<WebAssembly.Module> {
-  const response = await body(url);
-  if (typeof WebAssembly.compileStreaming === 'function' && (response.headers.get('content-type') ?? '').includes('application/wasm')) {
-    return WebAssembly.compileStreaming(response);
-  }
-  return WebAssembly.compile(await response.arrayBuffer());
-}
-
+/**
+ * What the playground's compiles read, downloaded into the cache. Only the
+ * small files are held here: each tool's worker takes the rest from the
+ * cache, those its tool reads (tool-host.ts).
+ */
 export interface LoadedToolchain {
-  /** The tools and files, without macro plugins until `plugins` settles. */
-  toolchain: Toolchain;
-  /**
-   * The macro plugins, downloading behind the first compile: most apps
-   * expand no macro, and the plugins are a fifth of the download.
-   */
-  plugins: Promise<Toolchain['plugins']>;
+  tools: Record<'swift-frontend' | 'wasm-ld', Download>;
+  /** The macro plugins by the module whose macros they expand: downloaded only by `prefetchPlugins`. */
+  plugins: Record<string, Download>;
+  /** The Swift SDK's files, relative to its `wasm32-unknown-wasip1` directory: to download, and those the manifest carries. */
+  sdk: { downloads: [string, Download][]; files: [string, Uint8Array][] };
+  /** The kit's files, relative to @swiftbrowser/kit. */
+  kit: [string, Download][];
+  kitManifest: Toolchain['kitManifest'];
+  icu: Toolchain['icu'];
   /** The module cache built with the playground's flags. */
   moduleCache: [string, Uint8Array][];
-  meta: PlaygroundMeta;
+  /**
+   * Downloads the plugins into the cache, keeping nothing: what the page
+   * does once a compile is done, and before a compile that expands a macro.
+   */
+  prefetchPlugins: () => Promise<void>;
+  manifest: PlaygroundManifest;
+  /** Why the kit may not suit this page's renderer, when it may not. */
+  warning?: string;
+}
+
+/**
+ * Whether the kit the manifest names was built from the sources the page's
+ * renderer was: the two speak one protocol, which either may change.
+ */
+export function kitWarning(manifest: Pick<PlaygroundManifest, 'shimSources'>, source: Pick<PlaygroundSource, 'shimSources'>): string | undefined {
+  if (!manifest.shimSources || !source.shimSources || manifest.shimSources === source.shimSources) return undefined;
+  return `The kit was built from other SwiftBrowser sources (${manifest.shimSources}) than this page's renderer (${source.shimSources}): an app may not run as it should.`;
 }
 
-export async function loadToolchain(base: string, files: PlaygroundFiles, onProgress: (text: string) => void = () => {}): Promise<LoadedToolchain> {
-  const url = (file: string): string => new URL(file, base).href;
+export async function loadToolchain(source: PlaygroundSource, onProgress: (text: string) => void = () => {}): Promise<LoadedToolchain> {
+  const cache = await openCache();
+  const from = ({ package: name, file }: PackageFile, module = false): Download => ({ url: packageUrl(source, name, file), ...(module ? { module } : {}) });
+  const bytes = (file: PackageFile): Promise<Uint8Array<ArrayBuffer>> => downloadBytes(cache, from(file));
   const started = performance.now();
   const lap = (text: string): void => onProgress(`${text} (${((performance.now() - started) / 1000).toFixed(1)} s)`);
-  const [frontend, linker, packBytes] = await Promise.all([
-    compileModule(url(files.frontend)).then((module) => (lap('compiler ready'), module)),
-    compileModule(url(files.linker)).then((module) => (lap('linker ready'), module)),
-    body(url(files.pack)).then((response) => response.arrayBuffer()).then((buffer) => (lap('SDK ready'), new Uint8Array(buffer))),
+  const own = (file: string): PackageFile => ({ package: PLAYGROUND_PACKAGE, file });
+  const manifest = JSON.parse(new TextDecoder().decode(await bytes(own(PLAYGROUND_MANIFEST)))) as PlaygroundManifest;
+
+  const kitPackage = '@swiftbrowser/kit';
+  // What a site stores brotli-compressed it serves under another name.
+  const brotli = new Set(manifest.brotli ?? []);
+  const served = (name: string, file: string): PackageFile => ({ package: name, file: brotli.has(`${name}/${file}`) ? `${file}${SITE_BROTLI}` : file });
+  const sdk = Object.entries(manifest.sdk).flatMap(([name, files]) => files.map((file): [string, Download] => [file, from(served(name, `wasm32-unknown-wasip1/${file}`))]));
+  const kit = manifest.kit.map((file): [string, Download] => [file, from(served(kitPackage, file))]);
+  const tools = { 'swift-frontend': from(manifest.frontend, true), 'wasm-ld': from(manifest.linker, true) };
+  const plugins = Object.fromEntries(Object.entries(manifest.plugins).map(([name, file]) => [name, from(file, true)]));
+  // The tools first, which take longest; one worker downloads them all.
+  const downloads = [tools['swift-frontend'], tools['wasm-ld'], ...sdk.map(([, file]) => file), ...kit.map(([, file]) => file)];
+  let done = 0;
+  const total = sdk.length + kit.length;
+  const fetched = (index: number): void => {
+    if (index < 2) return lap(index === 0 ? 'compiler ready' : 'linker ready');
+    if (++done % 50 === 0 || done === total) onProgress(`SDK and kit: ${done} of ${total} files`);
+  };
+  const [, kitJson, icu, moduleCacheBytes] = await Promise.all([
+    prefetchInWorker(downloads, fetched).then(() => lap('SDK ready')),
+    bytes(served(kitPackage, 'kit.json')),
+    bytes(served(PLAYGROUND_PACKAGE, PLAYGROUND_ICU)),
+    bytes(served(PLAYGROUND_PACKAGE, PLAYGROUND_MODULE_CACHE)),
   ]);
-  const pack = readPack<PlaygroundMeta>(packBytes);
-  const plugins = Promise.all(Object.entries(files.plugins).map(async ([name, file]) => {
-    const bytes = new Uint8Array(await (await body(url(file))).arrayBuffer());
-    return [name, { module: await WebAssembly.compile(bytes), bytes }] as const;
-  })).then(Object.fromEntries<Toolchain['plugins'][string]>);
-  // A failed download surfaces when a compile needs the plugins.
-  plugins.catch(() => {});
-  const kitJson = pack.file('kit.json');
-  const icu = pack.file('icu.dat');
-  if (!kitJson || !icu) throw new Error('playground.pack has no kit.json or icu.dat');
+  const encoder = new TextEncoder();
+  const inline = Object.entries(manifest.inline ?? {}).map(([path, text]): [string, Uint8Array] => [path, encoder.encode(text)]);
+
+  // Every file is named by its version: drop what an older one left.
+  void (async () => {
+    const wanted = new Set([
+      ...[...Object.values(tools), ...Object.values(plugins), ...sdk.map(([, file]) => file), ...kit.map(([, file]) => file)].map(cacheKey),
+      ...[own(PLAYGROUND_MANIFEST), served(kitPackage, 'kit.json'), served(PLAYGROUND_PACKAGE, PLAYGROUND_ICU), served(PLAYGROUND_PACKAGE, PLAYGROUND_MODULE_CACHE)].map((file) => cacheKey(from(file))),
+    ]);
+    for (const request of (await cache?.keys()) ?? []) if (!wanted.has(request.url)) await cache!.delete(request);
+  })().catch(() => {});
+
+  const warning = kitWarning(manifest, source);
   return {
-    toolchain: {
-      tools: { 'swift-frontend': frontend, 'wasm-ld': linker },
-      plugins: {},
-      sdk: memoryDirectory(pack.files('sdk')),
-      kit: memoryDirectory(pack.files('kit')),
-      kitManifest: JSON.parse(new TextDecoder().decode(kitJson)) as Toolchain['kitManifest'],
-      icu: { symbol: pack.meta.icuSymbol, data: icu },
-    },
+    tools,
     plugins,
-    moduleCache: pack.files('module-cache'),
-    meta: pack.meta,
+    sdk: { downloads: sdk, files: inline },
+    kit,
+    kitManifest: JSON.parse(new TextDecoder().decode(kitJson)) as Toolchain['kitManifest'],
+    icu: { symbol: manifest.icuSymbol, data: icu },
+    moduleCache: readPack(moduleCacheBytes).files('module-cache'),
+    prefetchPlugins: () => prefetchInWorker(Object.values(plugins)),
+    manifest,
+    ...(warning ? { warning } : {}),
   };
 }