@sproutboat/toolchain 0.8.1 → 0.10.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @sproutboat/toolchain
 
+## 0.10.0
+
+### Minor Changes
+
+- 4274bef: `Response.body` and `Request.body` are `ReadableStream`s for every body type, as in the Fetch spec and workerd. Before, they held the value given (a string, bytes or a `Blob`). `res.body.getReader()`, `res.body.pipeThrough(...)` and `new Response(other.body)` now work on any response, including the incoming request. The value as given stays internal, so responses are still written with a fixed length unless the body really is a stream. A body whose stream has been locked can no longer be read with `text()` and the other readers, which throw a `TypeError`.
+
+## 0.9.0
+
+### Minor Changes
+
+- 2541458: More of the Web platform, from sproutd's W2 conformance suite against workerd:
+  
+  - `CompressionStream` and `DecompressionStream` for `gzip`, `deflate` and `deflate-raw`, in JavaScript, because the native runtime links no zlib. Output decompresses with zlib, and zlib's output (stored, fixed and dynamic blocks) decompresses here. A corrupt or truncated input errors with a `TypeError`.
+  - A `Response` or `Request` whose body is a `ReadableStream` is read to the end by `text()`, `json()`, `arrayBuffer()` and `blob()`. Before, these returned `"[object ReadableStream]"`. Chunks must be bytes, as in workerd.
+  - `JSON.parse` applies its reviver: children first, `undefined` deletes the property, and `this` is the holder.
+
 ## 0.8.1
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sproutboat/toolchain",
-  "version": "0.8.1",
+  "version": "0.10.0",
   "license": "MIT",
   "type": "module",
   "repository": {

package/src/acquire.test.ts

@@ -151,6 +151,8 @@ async function fixture(): Promise<{ archive: string; sha256: string }> {
     "compiler/builtins/json.ts":
       "let __Porffor_json_bufLimit: i32 = 0;\n    depth += 1;\n    depth -= 1;\n    depth += 1;\n    depth -= 1;\n  let out: i32 = __Porffor_json_serialize(buffer, value, 0, space);\n" +
       "// sb_json_utf16_v1\n" +
+      // patchJsonReviver's anchor.
+      "export const __JSON_parse = (_: bytestring) => {\n};\n" +
       "export const __Porffor_json_serialize = (_buffer: i32, value: any): i32 => {\n" +
       "    for (const x of (value as any[])) {\n    }\n" +
       "      const val: any = value[key];\n" +

package/src/fetch-classes.ts

@@ -65,7 +65,22 @@ export const REQUEST_CLASS_SOURCE = String.raw`class Request {
   static _consume(message) {
     if (message._bodyUsed === true) throw new TypeError('Body has already been used');
     message._bodyUsed = true;
-    return message.body;
+    // A stream the body getter handed out is what the reader reads: a locked one throws.
+    return message._bodyStream != null ? message._bodyStream : message._bodyInit;
+  }
+
+  // body is a ReadableStream, as in Fetch: a stream body itself, else a stream made once over
+  // the bytes. The value as given stays in _bodyInit, which the native writer, streaming and
+  // the prelude read; assigning body (the native ingress does) sets it.
+  static _bodyAsStream(message) {
+    const init = message._bodyInit;
+    if (init == null) return null;
+    if (Request._isStream(init)) return init;
+    if (message._bodyStream == null) {
+      const bytes = Request._bodyBytes(init);
+      message._bodyStream = new globalThis.ReadableStream({ start(c) { if (bytes.length) c.enqueue(bytes); c.close(); } });
+    }
+    return message._bodyStream;
   }
 
   static _bodyText(body) {
@@ -74,6 +89,40 @@ export const REQUEST_CLASS_SOURCE = String.raw`class Request {
     return (Porffor.type(body) | 0b10000000) == Porffor.TYPES.bytestring ? body : String(body);
   }
 
+  // A ReadableStream body is read to its end, and its chunks must be bytes, as in Fetch and
+  // workerd. Readers of such a body answer with a promise; other bodies keep answering at
+  // once, since the prelude reads them synchronously. The prelude defines ReadableStream on
+  // globalThis after this file is compiled.
+  static _isStream(body) {
+    const Stream = globalThis.ReadableStream;
+    return Stream != null && body instanceof Stream;
+  }
+  static async _streamBytes(stream) {
+    const reader = stream.getReader();
+    const chunks = [];
+    let size = 0;
+    for (;;) {
+      const step = await reader.read();
+      if (step.done) break;
+      const value = step.value;
+      if (!(value instanceof Uint8Array)) throw new TypeError('This ReadableStream did not return bytes.');
+      chunks.push(value);
+      size += value.length;
+    }
+    const out = new Uint8Array(size);
+    let at = 0;
+    for (const chunk of chunks) {
+      out.set(chunk, at);
+      at += chunk.length;
+    }
+    return out;
+  }
+  static _read(message, finish) {
+    const body = Request._consume(message);
+    if (Request._isStream(body)) return Request._streamBytes(body).then(finish);
+    return finish(Request._bodyBytes(body));
+  }
+
   constructor(input, init = undefined) {
     let url = input;
     let method = 'GET';
@@ -85,7 +134,7 @@ export const REQUEST_CLASS_SOURCE = String.raw`class Request {
       url = input.url;
       method = input.method;
       headers = input.headers;
-      body = input.body;
+      body = input._bodyStream != null ? input._bodyStream : input._bodyInit;
       redirect = input.redirect;
     } else if (input != null && Porffor.type(input) == Porffor.TYPES.object && input.href != null) url = input.href;
 
@@ -108,21 +157,31 @@ export const REQUEST_CLASS_SOURCE = String.raw`class Request {
     if (init != null && init.body != null && init.body !== '' && (this.method == 'GET' || this.method == 'HEAD'))
       throw new TypeError('Request with GET/HEAD method cannot have body.');
     this.headers = new Headers(headers);
-    this.body = Request._extractBody(body, this.headers);
+    this._bodyInit = Request._extractBody(body, this.headers);
     if (redirect != null) this._redirect = String(redirect);
   }
 
   get redirect() { return this._redirect ?? 'follow'; }
+  get body() { return Request._bodyAsStream(this); }
+  set body(value) { this._bodyInit = value; this._bodyStream = null; }
   get bodyUsed() { return this._bodyUsed === true; }
 
-  arrayBuffer() { return Request._bodyBytes(Request._consume(this)).buffer; }
-  blob() { return new Blob([Request._bodyBytes(Request._consume(this))]); }
+  arrayBuffer() { return Request._read(this, (bytes) => bytes.buffer); }
+  blob() { return Request._read(this, (bytes) => new Blob([bytes])); }
   clone() {
     if (this._bodyUsed === true) throw new TypeError('Body has already been used');
     return new Request(this);
   }
-  json() { return JSON.parse(Request._bodyText(Request._consume(this))); }
-  text() { return Request._bodyText(Request._consume(this)); }
+  json() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)));
+    return JSON.parse(Request._bodyText(body));
+  }
+  text() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => new TextDecoder().decode(bytes));
+    return Request._bodyText(body);
+  }
 }
 
 `;
@@ -137,7 +196,7 @@ export const RESPONSE_CLASS_SOURCE = String.raw`class Response {
       throw new TypeError('Response with null body status cannot have body');
     if (init?.statusText != null) this._statusText = String(init.statusText);
     this.headers = new Headers(init?.headers);
-    this.body = Request._extractBody(body, this.headers);
+    this._bodyInit = Request._extractBody(body, this.headers);
   }
 
   static error() {
@@ -150,20 +209,30 @@ export const RESPONSE_CLASS_SOURCE = String.raw`class Response {
   get statusText() { return this._statusText ?? ''; }
   get type() { return this._type ?? 'default'; }
   get ok() { return this.status >= 200 && this.status < 300; }
+  get body() { return Request._bodyAsStream(this); }
+  set body(value) { this._bodyInit = value; this._bodyStream = null; }
   get bodyUsed() { return this._bodyUsed === true; }
 
-  arrayBuffer() { return Request._bodyBytes(Request._consume(this)).buffer; }
-  blob() { return new Blob([Request._bodyBytes(Request._consume(this))]); }
+  arrayBuffer() { return Request._read(this, (bytes) => bytes.buffer); }
+  blob() { return Request._read(this, (bytes) => new Blob([bytes])); }
   clone() {
     if (this._bodyUsed === true) throw new TypeError('Body has already been used');
     const copy = new Response(null, { status: this.status == 0 ? 200 : this.status, statusText: this._statusText, headers: this.headers });
-    copy.body = this.body;
+    copy._bodyInit = this._bodyInit;
     copy.status = this.status;
     if (this._type != null) copy._type = this._type;
     return copy;
   }
-  json() { return JSON.parse(Request._bodyText(Request._consume(this))); }
-  text() { return Request._bodyText(Request._consume(this)); }
+  json() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)));
+    return JSON.parse(Request._bodyText(body));
+  }
+  text() {
+    const body = Request._consume(this);
+    if (Request._isStream(body)) return Request._streamBytes(body).then((bytes) => new TextDecoder().decode(bytes));
+    return Request._bodyText(body);
+  }
 }
 `;
 
@@ -173,7 +242,7 @@ export const RESPONSE_BYTES_ANCHOR =
 export const RESPONSE_BYTES_INJECT =
   "  // sproutboat: byte bodies go out as their bytes. A Latin-1 string, one character per byte,\n" +
   "  // built in windows (concatenating one long string is quadratic), sent raw by the writer.\n" +
-  "  let body = response.body;\n" +
+  "  let body = response._bodyInit !== undefined ? response._bodyInit : response.body;\n" +
   "  let headersEntries = response.headers._entries;\n" +
   "  if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) {\n" +
   "    const view = body instanceof ArrayBuffer ? new Uint8Array(body) : new Uint8Array(body.buffer, body.byteOffset, body.byteLength);\n" +

package/src/fingerprint.ts

@@ -21,7 +21,7 @@ export const PATCH_SOURCES = [
  * patch.ts as the module patch.ts is everywhere else. Kept current by fingerprint.test.ts
  * (UPDATE_FINGERPRINT=1 bun test packages/toolchain/src/fingerprint.test.ts).
  */
-export const RELEASED_FINGERPRINT = "f0ff426f97b0";
+export const RELEASED_FINGERPRINT = "405c8f04fb7d";
 
 /** The fingerprint of the sources in dir, or null where they are not on disk. */
 export function sourceFingerprint(dir: string): string | null {

package/src/patch.ts

@@ -1938,6 +1938,55 @@ export async function patchJsonCycles(root: string): Promise<void> {
   await writeFile(file, src);
 }
 
+// JSON.parse ignored its reviver (W2 conformance: json/reviver). The parser keeps its body
+// under a new name, and __JSON_parse walks the result as the spec's InternalizeJSONProperty
+// does: children first, a reviver answer of undefined deletes the property.
+const JSON_REVIVER_MARKER = "sproutboat: JSON.parse reviver";
+// Porffor declares (_: bytestring); patchUnicode widens it to any later, on the renamed parser.
+const JSON_PARSE_ANCHOR = /export const __JSON_parse = \(_: (bytestring|any)\) => \{\n/;
+const JSON_REVIVER_SOURCE = `
+// ${JSON_REVIVER_MARKER}
+export const __Porffor_json_revive = (holder: any, key: any, reviver: any): any => {
+  const value: any = holder[key];
+  if (Porffor.type(value) == Porffor.TYPES.array) {
+    const len: i32 = value.length;
+    for (let i: i32 = 0; i < len; i++) {
+      const next: any = __Porffor_json_revive(value, ecma262.ToString(i), reviver);
+      if (next === undefined) delete value[i];
+      else value[i] = next;
+    }
+  } else if (Porffor.type(value) == Porffor.TYPES.object) {
+    for (const k: any in value) {
+      const next: any = __Porffor_json_revive(value, k, reviver);
+      if (next === undefined) delete value[k];
+      else value[k] = next;
+    }
+  }
+  return reviver.call(holder, key, value);
+};
+
+export const __JSON_parse = (text: any, reviver: any) => {
+  const value: any = __Porffor_json_parseText(text);
+  if (Porffor.type(reviver) != Porffor.TYPES.function) return value;
+  const root: object = {};
+  root[''] = value;
+  return __Porffor_json_revive(root, '', reviver);
+};
+`;
+
+/** JSON.parse applies its reviver, as InternalizeJSONProperty does. */
+export async function patchJsonReviver(root: string): Promise<void> {
+  const file = resolve(root, "compiler/builtins/json.ts");
+  const src = await readFile(file, "utf8");
+  if (src.includes(JSON_REVIVER_MARKER)) return;
+  if ((src.match(new RegExp(JSON_PARSE_ANCHOR, "g")) ?? []).length !== 1)
+    throw new Error(`could not patch Porffor's JSON.parse reviver: anchor not found in ${file}`);
+  await writeFile(
+    file,
+    src.replace(JSON_PARSE_ANCHOR, (_line, type) => `export const __Porffor_json_parseText = (_: ${type}) => {\n`).trimEnd() + "\n" + JSON_REVIVER_SOURCE,
+  );
+}
+
 // Object.prototype.toString gave "[object Object]" for typed arrays: their lazily
 // built prototypes have no toStringTag, and the fallback list of builtin tags left
 // them out (W2 conformance; V8 gives "[object Uint8Array]").
@@ -2251,6 +2300,7 @@ export async function ensurePorfforPatched(root: string): Promise<void> {
   await patchPrivateFieldsHidden(root);
   await patchRepeatDoubling(root);
   await patchJsonCycles(root);
+  await patchJsonReviver(root);
   await patchTypedArrayTag(root);
   await patchConsoleColors(root);
   await patchCaseMapping(root);

package/src/unicode-native.test.ts

@@ -110,4 +110,6 @@ test("native Unicode JSON and UTF-8 encoding preserve code points and buffer bou
     }
     await rm(work, { recursive: true, force: true });
   }
-}, 60_000);
+  // An optimised native build of the whole prelude: about 17 s warm, but 50 to 110 s cold
+  // (a new patch fingerprint or prelude recompiles everything), which 60 s did not cover.
+}, 180_000);