@sproutboat/toolchain 0.8.0 → 0.9.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @sproutboat/toolchain
 
+## 0.9.0
+
+### Minor Changes
+
+- 2541458: More of the Web platform, from sproutd's W2 conformance suite against workerd:
+  
+  - `CompressionStream` and `DecompressionStream` for `gzip`, `deflate` and `deflate-raw`, in JavaScript, because the native runtime links no zlib. Output decompresses with zlib, and zlib's output (stored, fixed and dynamic blocks) decompresses here. A corrupt or truncated input errors with a `TypeError`.
+  - A `Response` or `Request` whose body is a `ReadableStream` is read to the end by `text()`, `json()`, `arrayBuffer()` and `blob()`. Before, these returned `"[object ReadableStream]"`. Chunks must be bytes, as in workerd.
+  - `JSON.parse` applies its reviver: children first, `undefined` deletes the property, and `this` is the holder.
+
+## 0.8.1
+
+### Patch Changes
+
+- 742e61c: A CLI compiled with `bun build --compile` builds again. 0.8.0 embedded the patch sources with text imports to fingerprint them, but Bun bundles a text import of `patch.ts` as the module `patch.ts` is everywhere else, so compiling failed with `No matching export ... for import "default"`. The fingerprint now hashes the sources when they are on disk, and falls back to the fingerprint they were released with (`RELEASED_FINGERPRINT`, kept current by a test). The value is unchanged, so existing caches stay valid.
+
 ## 0.8.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/toolchain",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/acquire.test.ts

@@ -151,6 +151,8 @@ async function fixture(): Promise<{ archive: string; sha256: string }> {
     "compiler/builtins/json.ts":
       "let __Porffor_json_bufLimit: i32 = 0;\n    depth += 1;\n    depth -= 1;\n    depth += 1;\n    depth -= 1;\n  let out: i32 = __Porffor_json_serialize(buffer, value, 0, space);\n" +
       "// sb_json_utf16_v1\n" +
+      // patchJsonReviver's anchor.
+      "export const __JSON_parse = (_: bytestring) => {\n};\n" +
       "export const __Porffor_json_serialize = (_buffer: i32, value: any): i32 => {\n" +
       "    for (const x of (value as any[])) {\n    }\n" +
       "      const val: any = value[key];\n" +

package/src/fetch-classes.ts

@@ -74,6 +74,40 @@ export const REQUEST_CLASS_SOURCE = String.raw`class Request {
     return (Porffor.type(body) | 0b10000000) == Porffor.TYPES.bytestring ? body : String(body);
   }
 
+  // A ReadableStream body is read to its end, and its chunks must be bytes, as in Fetch and
+  // workerd. Readers of such a body answer with a promise; other bodies keep answering at
+  // once, since the prelude reads them synchronously. The prelude defines ReadableStream on
+  // globalThis after this file is compiled.
+  static _isStream(body) {
+    const Stream = globalThis.ReadableStream;
+    return Stream != null && body instanceof Stream;
+  }
+  static async _streamBytes(stream) {
+    const reader = stream.getReader();
+    const chunks = [];
+    let size = 0;
+    for (;;) {
+      const step = await reader.read();
+      if (step.done) break;
+      const value = step.value;
+      if (!(value instanceof Uint8Array)) throw new TypeError('This ReadableStream did not return bytes.');
+      chunks.push(value);
+      size += value.length;
+    }
+    const out = new Uint8Array(size);
+    let at = 0;
+    for (const chunk of chunks) {
+      out.set(chunk, at);
+      at += chunk.length;
+    }
+    return out;
+  }
+  static _read(message, finish) {
+    const body = Request._consume(message);
+    if (Request._isStream(body)) return Request._streamBytes(body).then(finish);
+    return finish(Request._bodyBytes(body));
+  }
+
   constructor(input, init = undefined) {
     let url = input;
     let method = 'GET';
@@ -115,14 +149,22 @@ export const REQUEST_CLASS_SOURCE = String.raw`class Request {
   get redirect() { return this._redirect ?? 'follow'; }
   get bodyUsed() { return this._bodyUsed === true; }
 
-  arrayBuffer() { return Request._bodyBytes(Request._consume(this)).buffer; }
-  blob() { return new Blob([Request._bodyBytes(Request._consume(this))]); }
+  arrayBuffer() { return Request._read(this, (bytes) => bytes.buffer); }
+  blob() { return Request._read(this, (bytes) => new Blob([bytes])); }
   clone() {
     if (this._bodyUsed === true) throw new TypeError('Body has already been used');
     return new Request(this);
   }
-  json() { return JSON.parse(Request._bodyText(Request._consume(this))); }
-  text() { return Request._bodyText(Request._consume(this)); }
+  json() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)));
+    return JSON.parse(Request._bodyText(body));
+  }
+  text() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => new TextDecoder().decode(bytes));
+    return Request._bodyText(body);
+  }
 }
 
 `;
@@ -152,8 +194,8 @@ export const RESPONSE_CLASS_SOURCE = String.raw`class Response {
   get ok() { return this.status >= 200 && this.status < 300; }
   get bodyUsed() { return this._bodyUsed === true; }
 
-  arrayBuffer() { return Request._bodyBytes(Request._consume(this)).buffer; }
-  blob() { return new Blob([Request._bodyBytes(Request._consume(this))]); }
+  arrayBuffer() { return Request._read(this, (bytes) => bytes.buffer); }
+  blob() { return Request._read(this, (bytes) => new Blob([bytes])); }
   clone() {
     if (this._bodyUsed === true) throw new TypeError('Body has already been used');
     const copy = new Response(null, { status: this.status == 0 ? 200 : this.status, statusText: this._statusText, headers: this.headers });
@@ -162,8 +204,16 @@ export const RESPONSE_CLASS_SOURCE = String.raw`class Response {
     if (this._type != null) copy._type = this._type;
     return copy;
   }
-  json() { return JSON.parse(Request._bodyText(Request._consume(this))); }
-  text() { return Request._bodyText(Request._consume(this)); }
+  json() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)));
+    return JSON.parse(Request._bodyText(body));
+  }
+  text() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => new TextDecoder().decode(bytes));
+    return Request._bodyText(body);
+  }
 }
 `;
 

package/src/fingerprint.test.ts

@@ -1,6 +1,6 @@
 import { expect, test } from "bun:test";
 import { readdir } from "node:fs/promises";
-import { PATCH_FINGERPRINT, PATCH_SOURCES } from "./fingerprint";
+import { PATCH_FINGERPRINT, PATCH_SOURCES, RELEASED_FINGERPRINT, sourceFingerprint } from "./fingerprint";
 
 // Not part of the patched compiler: acquisition, the barrel, generators and this module.
 const NOT_PATCH_SOURCES = new Set(["acquire.ts", "index.ts", "fingerprint.ts"]);
@@ -9,8 +9,20 @@ test("the patch fingerprint covers every toolchain module that shapes the compil
   const modules = (await readdir(import.meta.dir)).filter(
     (name) => name.endsWith(".ts") && !name.endsWith(".test.ts") && !name.startsWith("gen-") && !NOT_PATCH_SOURCES.has(name),
   );
-  expect(Object.keys(PATCH_SOURCES).sort()).toEqual(modules.sort());
-  const embedded = new Map<string, string>(Object.entries(PATCH_SOURCES));
-  for (const name of modules) expect(embedded.get(name)).toBe(await Bun.file(`${import.meta.dir}/${name}`).text());
+  expect(modules.sort()).toEqual(PATCH_SOURCES.map(String).sort());
   expect(PATCH_FINGERPRINT).toMatch(/^[0-9a-f]{12}$/);
+  expect(PATCH_FINGERPRINT).toBe(sourceFingerprint(import.meta.dir)!);
+  expect(sourceFingerprint("/nonexistent")).toBeNull();
+});
+
+// A compiled CLI falls back to RELEASED_FINGERPRINT, so it must match the sources.
+test("RELEASED_FINGERPRINT matches the sources", async () => {
+  const actual = sourceFingerprint(import.meta.dir)!;
+  if (process.env.UPDATE_FINGERPRINT && actual !== RELEASED_FINGERPRINT) {
+    const file = `${import.meta.dir}/fingerprint.ts`;
+    const text = await Bun.file(file).text();
+    await Bun.write(file, text.replace(`RELEASED_FINGERPRINT = "${RELEASED_FINGERPRINT}"`, `RELEASED_FINGERPRINT = "${actual}"`));
+    return;
+  }
+  expect(RELEASED_FINGERPRINT, "run UPDATE_FINGERPRINT=1 bun test packages/toolchain/src/fingerprint.test.ts").toBe(actual);
 });

package/src/fingerprint.ts

@@ -1,48 +1,44 @@
 import { createHash } from "node:crypto";
-// The patch chain's own source text, embedded by Bun's text loader (it survives
-// `bun build --compile`, where these files do not exist on disk). TypeScript sees
-// a module, not a string, hence the expect-errors.
-// @ts-expect-error text import
-import caseMap from "./case-map.ts" with { type: "text" };
-// @ts-expect-error text import
-import fetchClasses from "./fetch-classes.ts" with { type: "text" };
-// @ts-expect-error text import
-import headersClass from "./headers-class.ts" with { type: "text" };
-// @ts-expect-error text import
-import patch from "./patch.ts" with { type: "text" };
-// @ts-expect-error text import
-import pin from "./pin.ts" with { type: "text" };
-// @ts-expect-error text import
-import streaming from "./streaming.ts" with { type: "text" };
-// @ts-expect-error text import
-import textDecoder from "./text-decoder.ts" with { type: "text" };
-// @ts-expect-error text import
-import unicode from "./unicode.ts" with { type: "text" };
-// @ts-expect-error text import
-import urlClass from "./url-class.ts" with { type: "text" };
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
 
 /** Every module that shapes the patched compiler. fingerprint.test.ts fails when one is missing. */
-export const PATCH_SOURCES = {
-  "case-map.ts": caseMap,
-  "fetch-classes.ts": fetchClasses,
-  "headers-class.ts": headersClass,
-  "patch.ts": patch,
-  "pin.ts": pin,
-  "streaming.ts": streaming,
-  "text-decoder.ts": textDecoder,
-  "unicode.ts": unicode,
-  "url-class.ts": urlClass,
-} satisfies Record<string, string>;
+export const PATCH_SOURCES = [
+  "case-map.ts",
+  "fetch-classes.ts",
+  "headers-class.ts",
+  "patch.ts",
+  "pin.ts",
+  "streaming.ts",
+  "text-decoder.ts",
+  "unicode.ts",
+  "url-class.ts",
+] as const;
+
+/**
+ * The fingerprint of the sources as released. A compiled CLI (`bun build --compile`) has
+ * no source files to hash, and cannot embed them either: Bun bundles a text import of
+ * patch.ts as the module patch.ts is everywhere else. Kept current by fingerprint.test.ts
+ * (UPDATE_FINGERPRINT=1 bun test packages/toolchain/src/fingerprint.test.ts).
+ */
+export const RELEASED_FINGERPRINT = "5a243546eb20";
+
+/** The fingerprint of the sources in dir, or null where they are not on disk. */
+export function sourceFingerprint(dir: string): string | null {
+  const hash = createHash("sha256");
+  try {
+    for (const name of PATCH_SOURCES) hash.update(`${name}\0${readFileSync(join(dir, name), "utf8")}\0`);
+  } catch {
+    return null;
+  }
+  return hash.digest("hex").slice(0, 12);
+}
 
 /**
  * Identifies the patch chain, not just the toolchain release. A cache entry is keyed by it, so
  * an edited patch gets a fresh, fully patched compiler instead of one a same-version build
  * already marked complete (B19: the prelude began calling sb_stream_*, which only the new
- * chain defines, and every build against an older entry failed to link).
+ * chain defines, and every build against an older entry failed to link). From the sources
+ * when they are on disk, so an unreleased edit counts at once; else as released.
  */
-export const PATCH_FINGERPRINT = (() => {
-  const hash = createHash("sha256");
-  for (const [name, source] of Object.entries(PATCH_SOURCES).sort(([a], [b]) => a.localeCompare(b)))
-    hash.update(`${name}\0${source}\0`);
-  return hash.digest("hex").slice(0, 12);
-})();
+export const PATCH_FINGERPRINT = sourceFingerprint(import.meta.dir) ?? RELEASED_FINGERPRINT;

package/src/patch.ts

@@ -1938,6 +1938,55 @@ export async function patchJsonCycles(root: string): Promise<void> {
   await writeFile(file, src);
 }
 
+// JSON.parse ignored its reviver (W2 conformance: json/reviver). The parser keeps its body
+// under a new name, and __JSON_parse walks the result as the spec's InternalizeJSONProperty
+// does: children first, a reviver answer of undefined deletes the property.
+const JSON_REVIVER_MARKER = "sproutboat: JSON.parse reviver";
+// Porffor declares (_: bytestring); patchUnicode widens it to any later, on the renamed parser.
+const JSON_PARSE_ANCHOR = /export const __JSON_parse = \(_: (bytestring|any)\) => \{\n/;
+const JSON_REVIVER_SOURCE = `
+// ${JSON_REVIVER_MARKER}
+export const __Porffor_json_revive = (holder: any, key: any, reviver: any): any => {
+  const value: any = holder[key];
+  if (Porffor.type(value) == Porffor.TYPES.array) {
+    const len: i32 = value.length;
+    for (let i: i32 = 0; i < len; i++) {
+      const next: any = __Porffor_json_revive(value, ecma262.ToString(i), reviver);
+      if (next === undefined) delete value[i];
+      else value[i] = next;
+    }
+  } else if (Porffor.type(value) == Porffor.TYPES.object) {
+    for (const k: any in value) {
+      const next: any = __Porffor_json_revive(value, k, reviver);
+      if (next === undefined) delete value[k];
+      else value[k] = next;
+    }
+  }
+  return reviver.call(holder, key, value);
+};
+
+export const __JSON_parse = (text: any, reviver: any) => {
+  const value: any = __Porffor_json_parseText(text);
+  if (Porffor.type(reviver) != Porffor.TYPES.function) return value;
+  const root: object = {};
+  root[''] = value;
+  return __Porffor_json_revive(root, '', reviver);
+};
+`;
+
+/** JSON.parse applies its reviver, as InternalizeJSONProperty does. */
+export async function patchJsonReviver(root: string): Promise<void> {
+  const file = resolve(root, "compiler/builtins/json.ts");
+  const src = await readFile(file, "utf8");
+  if (src.includes(JSON_REVIVER_MARKER)) return;
+  if ((src.match(new RegExp(JSON_PARSE_ANCHOR, "g")) ?? []).length !== 1)
+    throw new Error(`could not patch Porffor's JSON.parse reviver: anchor not found in ${file}`);
+  await writeFile(
+    file,
+    src.replace(JSON_PARSE_ANCHOR, (_line, type) => `export const __Porffor_json_parseText = (_: ${type}) => {\n`).trimEnd() + "\n" + JSON_REVIVER_SOURCE,
+  );
+}
+
 // Object.prototype.toString gave "[object Object]" for typed arrays: their lazily
 // built prototypes have no toStringTag, and the fallback list of builtin tags left
 // them out (W2 conformance; V8 gives "[object Uint8Array]").
@@ -2251,6 +2300,7 @@ export async function ensurePorfforPatched(root: string): Promise<void> {
   await patchPrivateFieldsHidden(root);
   await patchRepeatDoubling(root);
   await patchJsonCycles(root);
+  await patchJsonReviver(root);
   await patchTypedArrayTag(root);
   await patchConsoleColors(root);
   await patchCaseMapping(root);

package/src/unicode-native.test.ts

@@ -110,4 +110,6 @@ test("native Unicode JSON and UTF-8 encoding preserve code points and buffer bou
     }
     await rm(work, { recursive: true, force: true });
   }
-}, 60_000);
+  // An optimised native build of the whole prelude: about 17 s warm, but 50 to 110 s cold
+  // (a new patch fingerprint or prelude recompiles everything), which 60 s did not cover.
+}, 180_000);