@sproutboat/toolchain 0.8.1 → 0.9.0

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @sproutboat/toolchain
 
+## 0.9.0
+
+### Minor Changes
+
+- 2541458: More of the Web platform, from sproutd's W2 conformance suite against workerd:
+  
+  - `CompressionStream` and `DecompressionStream` for `gzip`, `deflate` and `deflate-raw`, in JavaScript, because the native runtime links no zlib. Output decompresses with zlib, and zlib's output (stored, fixed and dynamic blocks) decompresses here. A corrupt or truncated input errors with a `TypeError`.
+  - A `Response` or `Request` whose body is a `ReadableStream` is read to the end by `text()`, `json()`, `arrayBuffer()` and `blob()`. Before, these returned `"[object ReadableStream]"`. Chunks must be bytes, as in workerd.
+  - `JSON.parse` applies its reviver: children first, `undefined` deletes the property, and `this` is the holder.
+
 ## 0.8.1
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/toolchain",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/acquire.test.ts

@@ -151,6 +151,8 @@ async function fixture(): Promise<{ archive: string; sha256: string }> {
     "compiler/builtins/json.ts":
       "let __Porffor_json_bufLimit: i32 = 0;\n    depth += 1;\n    depth -= 1;\n    depth += 1;\n    depth -= 1;\n  let out: i32 = __Porffor_json_serialize(buffer, value, 0, space);\n" +
       "// sb_json_utf16_v1\n" +
+      // patchJsonReviver's anchor.
+      "export const __JSON_parse = (_: bytestring) => {\n};\n" +
       "export const __Porffor_json_serialize = (_buffer: i32, value: any): i32 => {\n" +
       "    for (const x of (value as any[])) {\n    }\n" +
       "      const val: any = value[key];\n" +

package/src/fetch-classes.ts

@@ -74,6 +74,40 @@ export const REQUEST_CLASS_SOURCE = String.raw`class Request {
     return (Porffor.type(body) | 0b10000000) == Porffor.TYPES.bytestring ? body : String(body);
   }
 
+  // A ReadableStream body is read to its end, and its chunks must be bytes, as in Fetch and
+  // workerd. Readers of such a body answer with a promise; other bodies keep answering at
+  // once, since the prelude reads them synchronously. The prelude defines ReadableStream on
+  // globalThis after this file is compiled.
+  static _isStream(body) {
+    const Stream = globalThis.ReadableStream;
+    return Stream != null && body instanceof Stream;
+  }
+  static async _streamBytes(stream) {
+    const reader = stream.getReader();
+    const chunks = [];
+    let size = 0;
+    for (;;) {
+      const step = await reader.read();
+      if (step.done) break;
+      const value = step.value;
+      if (!(value instanceof Uint8Array)) throw new TypeError('This ReadableStream did not return bytes.');
+      chunks.push(value);
+      size += value.length;
+    }
+    const out = new Uint8Array(size);
+    let at = 0;
+    for (const chunk of chunks) {
+      out.set(chunk, at);
+      at += chunk.length;
+    }
+    return out;
+  }
+  static _read(message, finish) {
+    const body = Request._consume(message);
+    if (Request._isStream(body)) return Request._streamBytes(body).then(finish);
+    return finish(Request._bodyBytes(body));
+  }
+
   constructor(input, init = undefined) {
     let url = input;
     let method = 'GET';
@@ -115,14 +149,22 @@ export const REQUEST_CLASS_SOURCE = String.raw`class Request {
   get redirect() { return this._redirect ?? 'follow'; }
   get bodyUsed() { return this._bodyUsed === true; }
 
-  arrayBuffer() { return Request._bodyBytes(Request._consume(this)).buffer; }
-  blob() { return new Blob([Request._bodyBytes(Request._consume(this))]); }
+  arrayBuffer() { return Request._read(this, (bytes) => bytes.buffer); }
+  blob() { return Request._read(this, (bytes) => new Blob([bytes])); }
   clone() {
     if (this._bodyUsed === true) throw new TypeError('Body has already been used');
     return new Request(this);
   }
-  json() { return JSON.parse(Request._bodyText(Request._consume(this))); }
-  text() { return Request._bodyText(Request._consume(this)); }
+  json() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)));
+    return JSON.parse(Request._bodyText(body));
+  }
+  text() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => new TextDecoder().decode(bytes));
+    return Request._bodyText(body);
+  }
 }
 
 `;
@@ -152,8 +194,8 @@ export const RESPONSE_CLASS_SOURCE = String.raw`class Response {
   get ok() { return this.status >= 200 && this.status < 300; }
   get bodyUsed() { return this._bodyUsed === true; }
 
-  arrayBuffer() { return Request._bodyBytes(Request._consume(this)).buffer; }
-  blob() { return new Blob([Request._bodyBytes(Request._consume(this))]); }
+  arrayBuffer() { return Request._read(this, (bytes) => bytes.buffer); }
+  blob() { return Request._read(this, (bytes) => new Blob([bytes])); }
   clone() {
     if (this._bodyUsed === true) throw new TypeError('Body has already been used');
     const copy = new Response(null, { status: this.status == 0 ? 200 : this.status, statusText: this._statusText, headers: this.headers });
@@ -162,8 +204,16 @@ export const RESPONSE_CLASS_SOURCE = String.raw`class Response {
     if (this._type != null) copy._type = this._type;
     return copy;
   }
-  json() { return JSON.parse(Request._bodyText(Request._consume(this))); }
-  text() { return Request._bodyText(Request._consume(this)); }
+  json() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)));
+    return JSON.parse(Request._bodyText(body));
+  }
+  text() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => new TextDecoder().decode(bytes));
+    return Request._bodyText(body);
+  }
 }
 `;
 

package/src/fingerprint.ts

@@ -21,7 +21,7 @@ export const PATCH_SOURCES = [
  * patch.ts as the module patch.ts is everywhere else. Kept current by fingerprint.test.ts
  * (UPDATE_FINGERPRINT=1 bun test packages/toolchain/src/fingerprint.test.ts).
  */
-export const RELEASED_FINGERPRINT = "f0ff426f97b0";
+export const RELEASED_FINGERPRINT = "5a243546eb20";
 
 /** The fingerprint of the sources in dir, or null where they are not on disk. */
 export function sourceFingerprint(dir: string): string | null {

package/src/patch.ts

@@ -1938,6 +1938,55 @@ export async function patchJsonCycles(root: string): Promise<void> {
   await writeFile(file, src);
 }
 
+// JSON.parse ignored its reviver (W2 conformance: json/reviver). The parser keeps its body
+// under a new name, and __JSON_parse walks the result as the spec's InternalizeJSONProperty
+// does: children first, a reviver answer of undefined deletes the property.
+const JSON_REVIVER_MARKER = "sproutboat: JSON.parse reviver";
+// Porffor declares (_: bytestring); patchUnicode widens it to any later, on the renamed parser.
+const JSON_PARSE_ANCHOR = /export const __JSON_parse = \(_: (bytestring|any)\) => \{\n/;
+const JSON_REVIVER_SOURCE = `
+// ${JSON_REVIVER_MARKER}
+export const __Porffor_json_revive = (holder: any, key: any, reviver: any): any => {
+  const value: any = holder[key];
+  if (Porffor.type(value) == Porffor.TYPES.array) {
+    const len: i32 = value.length;
+    for (let i: i32 = 0; i < len; i++) {
+      const next: any = __Porffor_json_revive(value, ecma262.ToString(i), reviver);
+      if (next === undefined) delete value[i];
+      else value[i] = next;
+    }
+  } else if (Porffor.type(value) == Porffor.TYPES.object) {
+    for (const k: any in value) {
+      const next: any = __Porffor_json_revive(value, k, reviver);
+      if (next === undefined) delete value[k];
+      else value[k] = next;
+    }
+  }
+  return reviver.call(holder, key, value);
+};
+
+export const __JSON_parse = (text: any, reviver: any) => {
+  const value: any = __Porffor_json_parseText(text);
+  if (Porffor.type(reviver) != Porffor.TYPES.function) return value;
+  const root: object = {};
+  root[''] = value;
+  return __Porffor_json_revive(root, '', reviver);
+};
+`;
+
+/** JSON.parse applies its reviver, as InternalizeJSONProperty does. */
+export async function patchJsonReviver(root: string): Promise<void> {
+  const file = resolve(root, "compiler/builtins/json.ts");
+  const src = await readFile(file, "utf8");
+  if (src.includes(JSON_REVIVER_MARKER)) return;
+  if ((src.match(new RegExp(JSON_PARSE_ANCHOR, "g")) ?? []).length !== 1)
+    throw new Error(`could not patch Porffor's JSON.parse reviver: anchor not found in ${file}`);
+  await writeFile(
+    file,
+    src.replace(JSON_PARSE_ANCHOR, (_line, type) => `export const __Porffor_json_parseText = (_: ${type}) => {\n`).trimEnd() + "\n" + JSON_REVIVER_SOURCE,
+  );
+}
+
 // Object.prototype.toString gave "[object Object]" for typed arrays: their lazily
 // built prototypes have no toStringTag, and the fallback list of builtin tags left
 // them out (W2 conformance; V8 gives "[object Uint8Array]").
@@ -2251,6 +2300,7 @@ export async function ensurePorfforPatched(root: string): Promise<void> {
   await patchPrivateFieldsHidden(root);
   await patchRepeatDoubling(root);
   await patchJsonCycles(root);
+  await patchJsonReviver(root);
   await patchTypedArrayTag(root);
   await patchConsoleColors(root);
   await patchCaseMapping(root);

package/src/unicode-native.test.ts

@@ -110,4 +110,6 @@ test("native Unicode JSON and UTF-8 encoding preserve code points and buffer bou
     }
     await rm(work, { recursive: true, force: true });
   }
-}, 60_000);
+  // An optimised native build of the whole prelude: about 17 s warm, but 50 to 110 s cold
+  // (a new patch fingerprint or prelude recompiles everything), which 60 s did not cover.
+}, 180_000);