buninu-browser 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/AGENTS.md +4 -0
  2. package/README.md +74 -9
  3. package/bunfig.toml +1 -0
  4. package/buninu-browser.js +4 -0
  5. package/index.js +5 -0
  6. package/lib/audio/piano.js +24 -1
  7. package/lib/audio/pulse-client.js +120 -12
  8. package/lib/cdp-server.js +19 -12
  9. package/lib/happy-dom/parser.js +24 -4
  10. package/lib/happy-dom/selector-matcher.js +83 -0
  11. package/lib/headless-shell.js +120 -28
  12. package/lib/input/hit-test.js +25 -7
  13. package/lib/layout/text-layout.js +365 -54
  14. package/lib/log.js +5 -2
  15. package/lib/media/byte-pipe.js +82 -0
  16. package/lib/media/position-audio.js +42 -0
  17. package/lib/network/network-service.js +326 -46
  18. package/lib/paint/display-list.js +322 -21
  19. package/lib/paint/gradient.js +11 -5
  20. package/lib/render-tree/index.js +62 -2
  21. package/lib/render-tree/svg.js +2 -1
  22. package/lib/renderer/bindings.js +989 -72
  23. package/lib/renderer/canvas-2d.js +92 -0
  24. package/lib/renderer/host.js +159 -9
  25. package/lib/renderer/interfaces.js +215 -0
  26. package/lib/renderer/main.js +160 -12
  27. package/lib/renderer/media-source.js +819 -0
  28. package/lib/renderer/messaging.js +411 -0
  29. package/lib/renderer/native-functions.js +113 -0
  30. package/lib/renderer/page-realm.js +1407 -200
  31. package/lib/renderer/page-renderer.js +1899 -394
  32. package/lib/renderer/rejections.js +31 -0
  33. package/lib/renderer/rewrite-worker.js +22 -0
  34. package/lib/renderer/script-rewrite.js +805 -57
  35. package/lib/renderer/trusted-types.js +129 -0
  36. package/lib/renderer/workers.js +327 -0
  37. package/lib/screenshot-cli.js +2 -0
  38. package/lib/style/computed-style.js +463 -242
  39. package/lib/style/selector-keys.js +302 -0
  40. package/lib/style/selector-traits.js +240 -0
  41. package/lib/user-agent.js +14 -2
  42. package/package.json +4 -1
  43. package/patches/happy-dom/0002-use-bun-platform-adapters.patch +139 -19
  44. package/patches/happy-dom/0006-parse-css-escapes-in-selectors.patch +115 -0
  45. package/patches/happy-dom/README.md +26 -0
  46. package/patches/ses/0002-embedder-native-functions-source-text-sloppy-code.patch +106 -0
  47. package/patches/ses/0003-cache-sloppy-evaluators.patch +47 -0
  48. package/patches/ses/README.md +4 -0
  49. package/test/canvas-2d.test.js +55 -0
  50. package/test/compositor.test.js +3 -1
  51. package/test/computed-style.test.js +98 -1
  52. package/test/display-list.test.js +74 -1
  53. package/test/download.test.js +35 -0
  54. package/test/fixtures/long-video.mp4 +0 -0
  55. package/test/fixtures/media-sync.js +57 -0
  56. package/test/happy-dom-platform-adapters.test.js +25 -0
  57. package/test/headless-shell.test.js +29 -0
  58. package/test/hit-test.test.js +25 -1
  59. package/test/log.test.js +4 -1
  60. package/test/media-source.test.js +156 -0
  61. package/test/media-sync.test.js +12 -0
  62. package/test/mobile.test.js +2 -2
  63. package/test/navigation.test.js +61 -0
  64. package/test/network-service.test.js +81 -0
  65. package/test/page-scripts.test.js +1094 -7
  66. package/test/position-audio.test.js +67 -0
  67. package/test/positioning.test.js +95 -1
  68. package/test/pulse-client.test.js +69 -0
  69. package/test/render-tree.test.js +22 -1
  70. package/test/renderer-host.test.js +5 -2
  71. package/test/screenshot-cli.test.js +25 -1
  72. package/test/script-rewrite.test.js +111 -6
  73. package/test/selector-keys.test.js +39 -0
  74. package/test/selector-traits.test.js +19 -0
  75. package/test/setup.js +5 -0
  76. package/test/text-layout.test.js +69 -1
  77. package/test/user-agent.test.js +60 -4
  78. package/vendor/happy-dom/packages/happy-dom/src/nodes/html-image-element/HTMLImageElement.ts +52 -22
  79. package/vendor/happy-dom/packages/happy-dom/src/query-selector/SelectorParser.ts +18 -17
  80. package/vendor/ses/dist/ses.mjs +52 -6
package/lib/log.js CHANGED
@@ -34,9 +34,11 @@ export function installLog(path) {
34
34
  logPath = path;
35
35
  for (const level of ["log", "info", "warn", "error", "debug", "trace"]) {
36
36
  const original = console[level].bind(console);
37
+ // The file first: whoever sees a line on the console (and then stops
38
+ // this process) finds it in the log too.
37
39
  console[level] = (...args) => {
38
- original(...args);
39
40
  writeLog(level, formatWithOptions({ colors: false }, ...args));
41
+ original(...args);
40
42
  };
41
43
  }
42
44
  writeLog("log", `started pid ${process.pid}, bun ${Bun.version}, cwd ${process.cwd()}, args ${JSON.stringify(Bun.argv.slice(2))}`);
@@ -69,13 +71,14 @@ export async function pipeToLog(stream, source, echo) {
69
71
  let pending = "";
70
72
  try {
71
73
  for await (const chunk of stream) {
72
- echo(chunk);
73
74
  pending += decoder.decode(chunk, { stream: true });
74
75
  const end = pending.lastIndexOf("\n");
75
76
  if (end >= 0) {
76
77
  writeLog(source, pending.slice(0, end));
77
78
  pending = pending.slice(end + 1);
78
79
  }
80
+ // Echoed after the complete lines are in the log (see installLog).
81
+ echo(chunk);
79
82
  }
80
83
  } catch {
81
84
  // The child went away mid-read.
@@ -0,0 +1,82 @@
1
+ /**
2
+ * A growing media buffer shared by a progressive download and by MSE appends.
3
+ *
4
+ * wasmpeg can only open a decoder on a complete snapshot, so playback keeps
5
+ * this concatenation, reopens when the decoder runs off the end of what it
6
+ * was opened on, and seeks back. `generation` changes on every append and
7
+ * on finish/fail, which is how a decoder knows its snapshot is stale.
8
+ * `waitFor` resolves when `size` reaches the mark or the pipe is finished.
9
+ */
10
+ export function createBytePipe() {
11
+ const chunks = [];
12
+ let size = 0;
13
+ let generation = 0;
14
+ let done = false;
15
+ let error = null;
16
+ /** @type {Array<() => void>} */
17
+ const waiters = [];
18
+
19
+ const wake = () => {
20
+ for (const waiter of waiters.splice(0)) waiter();
21
+ };
22
+
23
+ return {
24
+ append(bytes) {
25
+ if (done || bytes == null) return;
26
+ const chunk = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
27
+ if (!chunk.byteLength) return;
28
+ chunks.push(chunk);
29
+ size += chunk.byteLength;
30
+ generation += 1;
31
+ wake();
32
+ },
33
+ finish() {
34
+ if (done) return;
35
+ done = true;
36
+ generation += 1;
37
+ wake();
38
+ },
39
+ fail(reason) {
40
+ if (done) return;
41
+ error = reason instanceof Error ? reason : new Error(String(reason ?? "media fetch failed"));
42
+ done = true;
43
+ generation += 1;
44
+ wake();
45
+ },
46
+ /** One contiguous snapshot. Collapses chunks so the next call is cheap. */
47
+ bytes() {
48
+ if (chunks.length === 1) return chunks[0];
49
+ const out = new Uint8Array(size);
50
+ let offset = 0;
51
+ for (const chunk of chunks) {
52
+ out.set(chunk, offset);
53
+ offset += chunk.byteLength;
54
+ }
55
+ chunks.length = 0;
56
+ if (out.byteLength) chunks.push(out);
57
+ return out;
58
+ },
59
+ get size() {
60
+ return size;
61
+ },
62
+ get generation() {
63
+ return generation;
64
+ },
65
+ get done() {
66
+ return done;
67
+ },
68
+ get error() {
69
+ return error;
70
+ },
71
+ waitFor(mark) {
72
+ if (size >= mark || done) return Promise.resolve();
73
+ return new Promise((resolve) => {
74
+ const check = () => {
75
+ if (size >= mark || done) resolve();
76
+ else waiters.push(check);
77
+ };
78
+ waiters.push(check);
79
+ });
80
+ },
81
+ };
82
+ }
@@ -0,0 +1,42 @@
1
+ /** Advance a PCM decoder to an exact sample cursor, never playing discarded PCM.
2
+ * Reopened snapshots start at sample zero, including their pending first chunk.
3
+ * The caller holds output requests until this transaction has completed.
4
+ */
5
+ export async function advanceAudio(getSound, seconds, { reopen, needsMore, active = () => true }) {
6
+ while (active()) {
7
+ const sound = getSound();
8
+ const { sampleRate, channels } = sound.decoder;
9
+ const target = Math.max(0, Math.round(seconds * sampleRate));
10
+ if (sound.position === target) return true;
11
+ if (sound.position > target) {
12
+ if (!await reopen()) return false;
13
+ continue;
14
+ }
15
+ // This is also the first chunk of EVERY replacement decoder. Skipping it
16
+ // used to leave the opening sound queued at exact chunk boundaries.
17
+ let samples = sound.pending;
18
+ sound.pending = null;
19
+ if (!samples) {
20
+ try { samples = sound.decoder.nextSamples(); }
21
+ catch (error) {
22
+ if (!needsMore(sound)) throw error;
23
+ }
24
+ }
25
+ if (samples?.length) {
26
+ const take = Math.min(target - sound.position, samples.length / channels);
27
+ sound.position += take;
28
+ if (take * channels < samples.length) sound.pending = samples.subarray(take * channels);
29
+ continue;
30
+ }
31
+ if (!needsMore(sound)) return false;
32
+ const pipe = sound.pipe;
33
+ if (pipe.error) throw pipe.error;
34
+ // Data may already have arrived while this snapshot was being decoded.
35
+ if (pipe.generation === sound.openedGeneration) await pipe.waitFor(pipe.size + 1);
36
+ if (!active()) return false;
37
+ if (pipe.error) throw pipe.error;
38
+ if (pipe.generation === sound.openedGeneration) return false;
39
+ if (!await reopen()) return false;
40
+ }
41
+ return false;
42
+ }
@@ -15,6 +15,7 @@ import { fetchResolved, HostResolver } from "./resolver.js";
15
15
 
16
16
  const MAX_REDIRECTS = 20;
17
17
  const MAX_BODY_BYTES = 64 * 1024 * 1024;
18
+ const MAX_CACHED_SUBRESOURCES = 256;
18
19
  const SIMPLE_METHODS = new Set(["GET", "HEAD", "POST"]);
19
20
  const FORBIDDEN_METHODS = new Set(["CONNECT", "TRACE", "TRACK"]);
20
21
  // Fetch §2.2.2 CORS-safelisted request headers (value checks simplified).
@@ -29,12 +30,23 @@ const FORBIDDEN_REQUEST_HEADERS = new Set([
29
30
  const SAFELISTED_RESPONSE_HEADERS = new Set([
30
31
  "cache-control", "content-language", "content-length", "content-type", "expires", "last-modified", "pragma",
31
32
  ]);
33
+ // Requests in flight to one host at a time, as browsers allow over HTTP/1.1.
34
+ const MAX_REQUESTS_PER_HOST = 6;
35
+ // Which waiting request to a busy host goes next (lower first): documents,
36
+ // then what blocks rendering or that a script waits for, then the rest.
37
+ const PRIORITY = { document: 0, iframe: 1, stylesheet: 1, fetch: 1, script: 2, metadata: 2, image: 3, video: 3, audio: 3 };
32
38
 
33
39
  export class NetworkService {
34
40
  #fetch;
35
41
  #resolver;
36
42
  #cookies;
37
43
  #userAgent;
44
+ // Browser-process HTTP cache. Entries are partitioned by the request
45
+ // credentials that affect a subresource response; concurrent identical
46
+ // requests share one transfer.
47
+ #subresourceCache = new Map();
48
+ #subresourceInflight = new Map();
49
+ #hosts = new HostThrottle(MAX_REQUESTS_PER_HOST);
38
50
 
39
51
  /**
40
52
  * `resolver` looks host names up itself (see resolver.js); it is on for the
@@ -55,20 +67,139 @@ export class NetworkService {
55
67
  }
56
68
 
57
69
  set userAgent(value) {
58
- this.#userAgent = typeof value === "string" && value ? value : null;
70
+ const next = typeof value === "string" && value ? value : null;
71
+ if (next !== this.#userAgent) this.#subresourceCache.clear();
72
+ this.#userAgent = next;
59
73
  }
60
74
 
61
75
  /** A top-level document load. */
62
- async navigate(url, { onResponse } = {}) {
63
- return this.#send({ url, method: "GET", headers: [], body: null }, {
76
+ /**
77
+ * Fetches a document to navigate to. A form POST (`post`) sends its body,
78
+ * with Origin and a Referer trimmed to the origin across origins
79
+ * (strict-origin-when-cross-origin); a redirect after it turns into a GET.
80
+ */
81
+ async navigate(url, { onResponse, post = null } = {}) {
82
+ const headers = [];
83
+ let method = "GET";
84
+ let body = null;
85
+ if (post) {
86
+ method = "POST";
87
+ body = post.body;
88
+ headers.push(["content-type", post.contentType]);
89
+ let from = null;
90
+ try { from = new URL(post.referrer); } catch {}
91
+ if (from && ["http:", "https:"].includes(from.protocol)) {
92
+ const target = new URL(url);
93
+ headers.push(["origin", from.origin]);
94
+ const downgrade = from.protocol === "https:" && target.protocol === "http:";
95
+ if (!downgrade) {
96
+ headers.push(["referer", from.origin === target.origin ? withoutFragment(from.href) : `${from.origin}/`]);
97
+ }
98
+ }
99
+ }
100
+ return this.#send({ url, method, headers, body }, {
64
101
  credentials: "include", site: null, topLevelNavigation: true,
65
- }, { onResponse });
102
+ }, { onResponse, priority: PRIORITY.document });
66
103
  }
67
104
 
68
- /** A subresource the renderer needs to render (no-cors, credentials included). */
69
- async subresource(url, documentUrl) {
105
+ /**
106
+ * A subresource the renderer needs to render (no-cors, credentials included).
107
+ * `kind` ("stylesheet", "script", "image"...) decides its turn at a busy host.
108
+ */
109
+ async subresource(url, documentUrl, kind = "metadata") {
110
+ const site = siteOf(documentUrl);
111
+ const cookie = this.#cookies.header(url, { site });
112
+ const key = `${url}\n${site ?? ""}\n${cookie}\n${this.#userAgent ?? ""}`;
113
+ const cached = this.#subresourceCache.get(key);
114
+ if (cached && cached.expires > Date.now()) return cached.result;
115
+ const pending = this.#subresourceInflight.get(key);
116
+ if (pending) return pending;
117
+ const request = (async () => {
118
+ const headers = cached?.etag ? [["if-none-match", cached.etag]] : [];
119
+ let result = await this.#send({ url, method: "GET", headers, body: null }, {
120
+ credentials: "include", site,
121
+ }, { priority: PRIORITY[kind] ?? PRIORITY.metadata });
122
+ if (result.status === 304 && cached) {
123
+ const rawHeaders = [...new Map([...cached.result.rawHeaders, ...result.rawHeaders])];
124
+ result = {
125
+ ...cached.result,
126
+ rawHeaders,
127
+ headers: [...new Map([...cached.result.headers, ...result.headers])],
128
+ };
129
+ const policy = cachePolicy(rawHeaders);
130
+ cached.expires = policy.expires;
131
+ return result;
132
+ }
133
+ const policy = cachePolicy(result.rawHeaders);
134
+ if (result.ok && policy.cacheable) {
135
+ if (!this.#subresourceCache.has(key) && this.#subresourceCache.size >= MAX_CACHED_SUBRESOURCES) {
136
+ this.#subresourceCache.delete(this.#subresourceCache.keys().next().value);
137
+ }
138
+ this.#subresourceCache.set(key, { result, expires: policy.expires, etag: policy.etag });
139
+ } else {
140
+ this.#subresourceCache.delete(key);
141
+ }
142
+ return result;
143
+ })();
144
+ this.#subresourceInflight.set(key, request);
145
+ try {
146
+ return await request;
147
+ } finally {
148
+ if (this.#subresourceInflight.get(key) === request) this.#subresourceInflight.delete(key);
149
+ }
150
+ }
151
+
152
+ /**
153
+ * A video or audio subresource whose body is delivered as it arrives.
154
+ * `emit` is called with `{ type: "head", ... }`, then `{ type: "chunk", data }`,
155
+ * then `{ type: "end" }` or `{ type: "error", error }`. Nothing is cached:
156
+ * a growing body is not a reusable response. The host slot stays held until
157
+ * the body has been read, same as a buffered subresource.
158
+ */
159
+ streamSubresource(url, documentUrl, kind, emit) {
160
+ const site = siteOf(documentUrl);
70
161
  return this.#send({ url, method: "GET", headers: [], body: null }, {
71
- credentials: "include", site: siteOf(documentUrl),
162
+ credentials: "include", site,
163
+ }, {
164
+ priority: PRIORITY[kind] ?? PRIORITY.metadata,
165
+ onResponse: async (response, finalUrl) => {
166
+ const contentLength = Number(response.headers.get("content-length"));
167
+ try {
168
+ emit({
169
+ type: "head",
170
+ ok: response.status >= 200 && response.status <= 299,
171
+ status: response.status,
172
+ statusText: response.statusText ?? "",
173
+ url: finalUrl,
174
+ contentType: response.headers.get("content-type") ?? "",
175
+ contentLength: Number.isFinite(contentLength) ? contentLength : null,
176
+ });
177
+ const reader = response.body?.getReader?.();
178
+ if (!reader) {
179
+ emit({ type: "end" });
180
+ return { streamed: true };
181
+ }
182
+ let total = 0;
183
+ for (;;) {
184
+ const { done, value } = await reader.read();
185
+ if (done) break;
186
+ if (!value?.byteLength) continue;
187
+ total += value.byteLength;
188
+ if (total > MAX_BODY_BYTES) {
189
+ emit({ type: "error", error: "Response body too large" });
190
+ await reader.cancel().catch(() => {});
191
+ return { streamed: true };
192
+ }
193
+ const copy = new Uint8Array(value.byteLength);
194
+ copy.set(value);
195
+ emit({ type: "chunk", data: copy });
196
+ }
197
+ emit({ type: "end" });
198
+ } catch (error) {
199
+ emit({ type: "error", error: String(error?.message ?? error) });
200
+ }
201
+ return { streamed: true };
202
+ },
72
203
  });
73
204
  }
74
205
 
@@ -111,7 +242,7 @@ export class NetworkService {
111
242
  * redirect?: "follow" | "error" | "manual" }} request
112
243
  * @param {string} documentUrl the requesting document (controller's record)
113
244
  */
114
- async pageFetch(request, documentUrl) {
245
+ async pageFetch(request, documentUrl, { signal, onChunk = null } = {}) {
115
246
  const origin = new URL(documentUrl).origin;
116
247
  const url = new URL(request.url);
117
248
  const method = String(request.method ?? "GET").toUpperCase();
@@ -127,26 +258,40 @@ export class NetworkService {
127
258
  if (mode === "no-cors" && !SIMPLE_METHODS.has(method)) throw new TypeError(`${method} is not allowed in no-cors mode`);
128
259
 
129
260
  const sendCredentials = credentials === "include" || (credentials === "same-origin" && sameOrigin);
130
- const context = { credentials: sendCredentials ? "include" : "omit", site: siteOf(documentUrl), origin };
261
+ const context = { credentials: sendCredentials ? "include" : "omit", site: siteOf(documentUrl), origin, referrer: documentUrl };
262
+ const stream = onChunk ? (response, finalUrl, redirected, type, responseHeaders = [...response.headers]) =>
263
+ this.#streamResult(response, finalUrl, redirected, type, responseHeaders, onChunk) : null;
131
264
 
132
265
  if (sameOrigin) {
133
266
  return this.#send({ url: url.href, method, headers, body: request.body ?? null }, context, {
134
- redirect: request.redirect ?? "follow", corsOrigin: null,
267
+ redirect: request.redirect ?? "follow", corsOrigin: null, priority: PRIORITY.fetch, signal,
268
+ onResponse: stream ? (response, finalUrl, redirected) => stream(response, finalUrl, redirected, "basic") : null,
135
269
  });
136
270
  }
137
271
  if (mode === "no-cors") {
138
272
  const response = await this.#send({ url: url.href, method, headers: headers.filter(isSafelistedHeader), body: request.body ?? null },
139
- context, { redirect: "follow", corsOrigin: null });
273
+ context, { redirect: "follow", corsOrigin: null, priority: PRIORITY.fetch, signal });
140
274
  return { ...response, type: "opaque", status: 0, statusText: "", url: "", headers: [], body: new Uint8Array(0), redirected: false };
141
275
  }
142
276
  // CORS: preflight unless the request is "simple" (Fetch §4.8).
143
277
  const needsPreflight = !SIMPLE_METHODS.has(method) || headers.some((header) => !isSafelistedHeader(header));
144
278
  if (needsPreflight) await this.#preflight(url, method, headers, origin, context);
145
279
  const response = await this.#send({ url: url.href, method, headers, body: request.body ?? null }, context, {
146
- redirect: request.redirect ?? "follow", corsOrigin: origin,
280
+ redirect: request.redirect ?? "follow", corsOrigin: origin, priority: PRIORITY.fetch, signal,
281
+ onResponse: stream ? async (incoming, finalUrl, redirected) => {
282
+ const raw = [...incoming.headers];
283
+ if (!corsAllowed(raw, origin, context.credentials === "include")) return null;
284
+ const exposed = new Set((header(raw, "access-control-expose-headers") ?? "")
285
+ .split(",").map((name) => name.trim().toLowerCase()).filter(Boolean));
286
+ const exposeAll = exposed.has("*") && context.credentials !== "include";
287
+ const visible = raw.filter(([name]) => SAFELISTED_RESPONSE_HEADERS.has(name) || exposed.has(name) || exposeAll);
288
+ return stream(incoming, finalUrl, redirected, "cors", visible);
289
+ } : null,
147
290
  });
291
+ if (response.streaming) return response;
148
292
  const allowed = corsAllowed(response.rawHeaders, origin, context.credentials === "include");
149
293
  if (!allowed) throw new TypeError(`CORS: ${url.href} did not allow origin ${origin}`);
294
+ if (response.type === "opaqueredirect") return response;
150
295
  const exposed = new Set((header(response.rawHeaders, "access-control-expose-headers") ?? "")
151
296
  .split(",").map((name) => name.trim().toLowerCase()).filter(Boolean));
152
297
  const exposeAll = exposed.has("*") && context.credentials !== "include";
@@ -159,17 +304,23 @@ export class NetworkService {
159
304
 
160
305
  async #preflight(url, method, headers, origin, context) {
161
306
  const requestHeaders = headers.filter((entry) => !isSafelistedHeader(entry)).map(([name]) => name).sort();
162
- const response = await this.#request(url.href, {
163
- method: "OPTIONS",
164
- redirect: "manual",
165
- headers: {
166
- origin,
167
- "access-control-request-method": method,
168
- ...(requestHeaders.length ? { "access-control-request-headers": requestHeaders.join(",") } : {}),
169
- ...(this.#userAgent ? { "user-agent": this.#userAgent } : {}),
170
- },
171
- });
172
- await response.arrayBuffer().catch(() => {});
307
+ const release = await this.#hosts.acquire(url.href, PRIORITY.fetch);
308
+ let response;
309
+ try {
310
+ response = await this.#request(url.href, {
311
+ method: "OPTIONS",
312
+ redirect: "manual",
313
+ headers: {
314
+ origin,
315
+ "access-control-request-method": method,
316
+ ...(requestHeaders.length ? { "access-control-request-headers": requestHeaders.join(",") } : {}),
317
+ ...(this.#userAgent ? { "user-agent": this.#userAgent } : {}),
318
+ },
319
+ });
320
+ await response.arrayBuffer().catch(() => {});
321
+ } finally {
322
+ release();
323
+ }
173
324
  const raw = [...response.headers];
174
325
  if (response.status < 200 || response.status > 299 || !corsAllowed(raw, origin, context.credentials === "include")) {
175
326
  throw new TypeError(`CORS preflight for ${url.href} was rejected`);
@@ -191,43 +342,67 @@ export class NetworkService {
191
342
  return this.#resolver ? fetchResolved(this.#fetch, this.#resolver, url, init) : this.#fetch(url, init);
192
343
  }
193
344
 
194
- /** Sends a request, following redirects by hand so cookies apply to every hop. */
195
- async #send(request, context, { redirect = "follow", corsOrigin = null, onResponse = null } = {}) {
345
+ /**
346
+ * Sends a request, following redirects by hand so cookies apply to every hop.
347
+ * Each hop holds one of its host's request slots until its body is read.
348
+ */
349
+ async #send(request, context, {
350
+ redirect = "follow", corsOrigin = null, onResponse = null, priority = PRIORITY.metadata, signal = undefined,
351
+ } = {}) {
196
352
  let { url, method, body } = request;
197
353
  const baseHeaders = request.headers;
198
354
  let redirected = false;
199
355
  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
200
356
  const headers = new Headers(baseHeaders);
201
- if (this.#userAgent) headers.set("user-agent", this.#userAgent);
357
+ if (this.#userAgent) {
358
+ headers.set("user-agent", this.#userAgent);
359
+ const chrome = /(?:Chrome|Chromium)\/(\d+)/.exec(this.#userAgent)?.[1];
360
+ if (chrome) {
361
+ headers.set("sec-ch-ua", `"Not:A-Brand";v="99", "Chromium";v="${chrome}", "Google Chrome";v="${chrome}"`);
362
+ headers.set("sec-ch-ua-mobile", /\bMobile\b/.test(this.#userAgent) ? "?1" : "?0");
363
+ headers.set("sec-ch-ua-platform", `"${/Android/.test(this.#userAgent) ? "Android" : /Windows/.test(this.#userAgent) ? "Windows" : /Macintosh/.test(this.#userAgent) ? "macOS" : "Linux"}"`);
364
+ }
365
+ }
202
366
  if (corsOrigin) headers.set("origin", corsOrigin);
367
+ if (context.referrer && !headers.has("referer")) {
368
+ const from = new URL(context.referrer);
369
+ const target = new URL(url);
370
+ const downgrade = from.protocol === "https:" && target.protocol === "http:";
371
+ if (!downgrade) headers.set("referer", from.origin === target.origin ? withoutFragment(from.href) : `${from.origin}/`);
372
+ }
203
373
  if (context.credentials === "include") {
204
374
  const cookie = this.#cookies.header(url, {
205
375
  site: context.site, topLevelNavigation: Boolean(context.topLevelNavigation), method,
206
376
  });
207
377
  if (cookie) headers.set("cookie", cookie);
208
378
  }
209
- const response = await this.#request(url, { method, headers, body, redirect: "manual" });
210
- if (context.credentials === "include") this.#cookies.store(url, response.headers.getSetCookie?.() ?? []);
211
- const location = response.headers.get("location");
212
- if ([301, 302, 303, 307, 308].includes(response.status) && location) {
213
- await response.arrayBuffer().catch(() => {});
214
- if (redirect === "error") throw new TypeError("Redirect was not allowed");
215
- if (redirect === "manual") return this.#result(response, url, redirected, "opaqueredirect");
216
- const next = new URL(location, url);
217
- if (!["http:", "https:"].includes(next.protocol)) throw new TypeError(`Redirect to ${next.protocol} is not allowed`);
218
- if (response.status === 303 || ((response.status === 301 || response.status === 302) && method === "POST")) {
219
- method = method === "HEAD" ? "HEAD" : "GET";
220
- body = null;
379
+ const release = await this.#hosts.acquire(url, priority);
380
+ try {
381
+ const response = await this.#request(url, { method, headers, body, redirect: "manual", signal });
382
+ if (context.credentials === "include") this.#cookies.store(url, response.headers.getSetCookie?.() ?? []);
383
+ const location = response.headers.get("location");
384
+ if ([301, 302, 303, 307, 308].includes(response.status) && location) {
385
+ await response.arrayBuffer().catch(() => {});
386
+ if (redirect === "error") throw new TypeError("Redirect was not allowed");
387
+ if (redirect === "manual") return opaqueRedirect(response);
388
+ const next = new URL(location, url);
389
+ if (!["http:", "https:"].includes(next.protocol)) throw new TypeError(`Redirect to ${next.protocol} is not allowed`);
390
+ if (response.status === 303 || ((response.status === 301 || response.status === 302) && method === "POST")) {
391
+ method = method === "HEAD" ? "HEAD" : "GET";
392
+ body = null;
393
+ }
394
+ url = next.href;
395
+ redirected = true;
396
+ continue;
221
397
  }
222
- url = next.href;
223
- redirected = true;
224
- continue;
225
- }
226
- if (onResponse) {
227
- const handled = await onResponse(response, url);
228
- if (handled) return handled;
398
+ if (onResponse) {
399
+ const handled = await onResponse(response, url, redirected);
400
+ if (handled) return handled;
401
+ }
402
+ return await this.#result(response, url, redirected, "basic");
403
+ } finally {
404
+ release();
229
405
  }
230
- return this.#result(response, url, redirected, "basic");
231
406
  }
232
407
  throw new TypeError("Too many redirects");
233
408
  }
@@ -250,6 +425,47 @@ export class NetworkService {
250
425
  body: buffer,
251
426
  };
252
427
  }
428
+
429
+ async #streamResult(response, url, redirected, type, visibleHeaders, onChunk) {
430
+ const rawHeaders = [...response.headers];
431
+ const result = {
432
+ ok: response.status >= 200 && response.status <= 299,
433
+ status: response.status, statusText: response.statusText, url, redirected, type,
434
+ headers: visibleHeaders.filter(([name]) => name !== "set-cookie" && name !== "set-cookie2"),
435
+ contentType: response.headers.get("content-type") ?? "",
436
+ };
437
+ onChunk({ type: "head", result });
438
+ const reader = response.body?.getReader?.();
439
+ if (reader) {
440
+ while (true) {
441
+ const { done, value } = await reader.read();
442
+ if (done) break;
443
+ if (value?.byteLength) onChunk({ type: "chunk", data: new Uint8Array(value) });
444
+ }
445
+ }
446
+ onChunk({ type: "end" });
447
+ return { ...result, rawHeaders, body: new Uint8Array(0), streaming: true };
448
+ }
449
+ }
450
+
451
+ /**
452
+ * The opaque-redirect filtered response a `redirect: "manual"` fetch gets
453
+ * (Fetch §2.2.6): status 0 and no headers or body for the page. The redirect's
454
+ * body was already drained to free the connection; the raw headers stay for
455
+ * the CORS check.
456
+ */
457
+ function opaqueRedirect(response) {
458
+ return {
459
+ ok: false, status: 0, statusText: "", url: "", redirected: false, type: "opaqueredirect",
460
+ rawHeaders: [...response.headers], headers: [], contentType: "", body: new Uint8Array(0),
461
+ };
462
+ }
463
+
464
+ /** A URL without its fragment, as a Referer carries it. */
465
+ function withoutFragment(href) {
466
+ const url = new URL(href);
467
+ url.hash = "";
468
+ return url.href;
253
469
  }
254
470
 
255
471
  function isSafelistedHeader([name, value]) {
@@ -262,6 +478,20 @@ function header(rawHeaders, name) {
262
478
  return rawHeaders.find(([key]) => key === name)?.[1] ?? null;
263
479
  }
264
480
 
481
+ /** Freshness and validator data used by the in-process HTTP cache. */
482
+ function cachePolicy(rawHeaders, now = Date.now()) {
483
+ const control = (header(rawHeaders, "cache-control") ?? "").toLowerCase();
484
+ if (/(?:^|,)\s*no-store(?:\s|,|$)/.test(control)) return { cacheable: false, expires: 0, etag: null };
485
+ const age = control.match(/(?:^|,)\s*max-age\s*=\s*(\d+)/)?.[1];
486
+ const expiresHeader = header(rawHeaders, "expires");
487
+ const expires = age !== undefined
488
+ ? now + Number(age) * 1000
489
+ : expiresHeader && Number.isFinite(Date.parse(expiresHeader)) ? Date.parse(expiresHeader)
490
+ : /(?:^|,)\s*immutable(?:\s|,|$)/.test(control) ? now + 365 * 24 * 60 * 60 * 1000 : 0;
491
+ const etag = header(rawHeaders, "etag");
492
+ return { cacheable: expires > now || etag !== null, expires, etag };
493
+ }
494
+
265
495
  /** Fetch §4.9 CORS check. */
266
496
  function corsAllowed(rawHeaders, origin, withCredentials) {
267
497
  const allowOrigin = header(rawHeaders, "access-control-allow-origin");
@@ -270,3 +500,53 @@ function corsAllowed(rawHeaders, origin, withCredentials) {
270
500
  if (allowOrigin !== origin) return false;
271
501
  return !withCredentials || header(rawHeaders, "access-control-allow-credentials") === "true";
272
502
  }
503
+
504
+ /**
505
+ * Limits the requests in flight to each host (scheme, host and port). A burst
506
+ * of requests that each open a connection slows every transfer, and servers
507
+ * reset some; with a few connections kept alive and reused, a page's
508
+ * resources arrive sooner. Waiting requests go by priority, then in order.
509
+ */
510
+ class HostThrottle {
511
+ #limit;
512
+ #hosts = new Map();
513
+
514
+ constructor(limit) {
515
+ this.#limit = limit;
516
+ }
517
+
518
+ /** Resolves with a function that gives the slot back (once) when the request is done. */
519
+ acquire(url, priority) {
520
+ let key;
521
+ try {
522
+ const target = new URL(url);
523
+ key = `${target.protocol}//${target.host}`;
524
+ } catch {
525
+ key = String(url);
526
+ }
527
+ let host = this.#hosts.get(key);
528
+ if (!host) this.#hosts.set(key, host = { active: 0, waiting: [] });
529
+ const release = this.#releaser(key, host);
530
+ if (host.active < this.#limit) {
531
+ host.active++;
532
+ return Promise.resolve(release);
533
+ }
534
+ return new Promise((resolve) => {
535
+ const entry = { priority, start: () => resolve(release) };
536
+ const index = host.waiting.findIndex((other) => other.priority > priority);
537
+ if (index === -1) host.waiting.push(entry);
538
+ else host.waiting.splice(index, 0, entry);
539
+ });
540
+ }
541
+
542
+ #releaser(key, host) {
543
+ let released = false;
544
+ return () => {
545
+ if (released) return;
546
+ released = true;
547
+ const next = host.waiting.shift();
548
+ if (next) next.start(); // the slot passes on
549
+ else if (--host.active === 0) this.#hosts.delete(key);
550
+ };
551
+ }
552
+ }