@tinvar/renderer 0.0.0-stage → 0.4.0

package/src/html-stream.js

@@ -0,0 +1,479 @@
+// Interleaves the RSC payload into the HTML stream as inline scripts.
+//
+// tinvar-spec.md, "Rendered output", shows the shape this produces:
+//   <script>(self.__rsc = self.__rsc || []).push([0, "0:[\"$\",\"$L1\",..."])</script>
+// and notes that `self.__rsc` is a framework choice, not part of React's format.
+//
+// The HTML and the payload are drained concurrently so a Suspense boundary's
+// markup and the rows that hydrate it both reach the browser as they become
+// ready, rather than all at the end -- the rows in the gaps between React's
+// flushes, never inside one (see merge()).
+
+const encoder = new TextEncoder();
+
+const LINE_SEPARATOR = '
';
+const PARAGRAPH_SEPARATOR = '
';
+
+/**
+ * Escapes a payload row so it cannot close the script element carrying it.
+ *
+ * JSON.stringify handles quotes and control characters. It leaves `<` and the
+ * two Unicode line separators alone, and both are hazards inside a script tag.
+ */
+function scriptSafe(text) {
+  return JSON.stringify(text)
+    .split('<')
+    .join('\\u003c')
+    .split(LINE_SEPARATOR)
+    .join('\\u2028')
+    .split(PARAGRAPH_SEPARATOR)
+    .join('\\u2029');
+}
+
+/** The id of the element carrying the page metadata; the router reads it. */
+export const PAGE_METADATA_ID = 'tinvar-page';
+
+/**
+ * The hosts PHP sent, as route() reads them, or null for none or for
+ * anything not their shape.
+ *
+ * @param {unknown} value
+ * @returns {{ scheme: string, port: string, roots: Record<string, string>, primary: string, current?: { pattern: string | null, params: Record<string, string> } } | null}
+ */
+function hostsOf(value) {
+  const v = /** @type {any} */ (value);
+  if (
+    typeof v !== 'object' ||
+    v === null ||
+    typeof v.scheme !== 'string' ||
+    typeof v.port !== 'string' ||
+    typeof v.primary !== 'string' ||
+    typeof v.roots !== 'object' ||
+    v.roots === null
+  ) {
+    return null;
+  }
+  /** @type {Record<string, string>} */
+  const roots = {};
+  for (const [name, host] of Object.entries(v.roots)) {
+    if (typeof host === 'string') roots[name] = host;
+  }
+  // The host the page is drawn on: its pattern, and what it took.
+  const c = v.current;
+  if (typeof c === 'object' && c !== null && (c.pattern === null || typeof c.pattern === 'string')) {
+    /** @type {Record<string, string>} */
+    const params = {};
+    for (const [name, value] of Object.entries(Array.isArray(c.params) ? {} : (c.params ?? {}))) {
+      if (typeof value === 'string') params[name] = value;
+    }
+    return {
+      scheme: v.scheme,
+      port: v.port,
+      roots,
+      primary: v.primary,
+      current: { pattern: c.pattern, params },
+    };
+  }
+  return { scheme: v.scheme, port: v.port, roots, primary: v.primary };
+}
+
+/**
+ * The initial page metadata, as a data block.
+ *
+ * tinvar-spec.md, "Protocol": "in a `<script type=\"application/json\">`
+ * element with every `/` escaped as `\/`" -- so `</script>` cannot appear in
+ * it whatever the URL holds. `<` and the line separators are escaped as for
+ * payload rows. A data block is never executed, so it takes no nonce.
+ *
+ * Only the known fields are copied: the frame comes from PHP, but what
+ * reaches the page is decided here, where the markup is written. The
+ * history key (ADR 0019) is written only for a page that has one.
+ *
+ * @param {{ url: unknown, version: unknown, routeToken?: unknown, view?: unknown, historyKey?: unknown, route?: unknown, hosts?: unknown }} page
+ */
+export function pageMetadataTag(page) {
+  if (typeof page?.url !== 'string' || typeof page.version !== 'string') {
+    throw new TypeError('Page metadata needs a string url and version.');
+  }
+  const routeToken = typeof page.routeToken === 'string' ? page.routeToken : null;
+  const view = typeof page.view === 'string' ? page.view : null;
+  const historyKey = typeof page.historyKey === 'string' ? { historyKey: page.historyKey } : {};
+  // What the page's reports are filed under (ADR 0129): absent, not null,
+  // where PHP gave none.
+  const route = typeof page.route === 'string' ? { route: page.route } : {};
+  // What route() writes absolute URLs with where PHP routes by host (ADR
+  // 0149): only its four fields, each of the type it is.
+  const hosts = hostsOf(page.hosts);
+
+  const json = JSON.stringify({
+    url: page.url,
+    version: page.version,
+    routeToken,
+    view,
+    ...historyKey,
+    ...route,
+    ...(hosts === null ? {} : { hosts }),
+  })
+    .split('/')
+    .join('\\/')
+    .split('<')
+    .join('\\u003c')
+    .split(LINE_SEPARATOR)
+    .join('\\u2028')
+    .split(PARAGRAPH_SEPARATOR)
+    .join('\\u2029');
+
+  return `<script type="application/json" id="${PAGE_METADATA_ID}">${json}</script>`;
+}
+
+const DOCTYPE = encoder.encode('<!doctype html>');
+
+/**
+ * Where the prelude goes in the first bytes of the document: after the
+ * doctype if they open with one, otherwise at the very start. -1 while
+ * they are a prefix of a doctype that has not finished arriving.
+ *
+ * React's chunk boundaries fall where its buffer fills, not between tags,
+ * so the doctype can arrive in pieces. Decided on the first piece alone,
+ * `<!DOCTY` was not a doctype, and the prelude went ahead of it -- which
+ * puts the browser in quirks mode.
+ *
+ * @param {Uint8Array} head
+ * @param {boolean} ended no more bytes are coming
+ */
+function preludeOffset(head, ended) {
+  const compared = Math.min(head.length, DOCTYPE.length);
+  for (let i = 0; i < compared; i++) {
+    const byte = head[i] >= 0x41 && head[i] <= 0x5a ? head[i] + 0x20 : head[i];
+    if (byte !== DOCTYPE[i]) return 0;
+  }
+  if (head.length >= DOCTYPE.length) return DOCTYPE.length;
+  return ended ? 0 : -1;
+}
+
+/**
+ * The first bytes of the document with the prelude at its very start --
+ * after the doctype, if there is one, and before everything else. Nothing
+ * the app renders can come before it.
+ *
+ * The prelude claims the payload globals and carries the page metadata.
+ * Both have to come first, for the same reason: page content can name
+ * things. An element with `id="__rsc"` -- a heading anchored by a user's
+ * slug, raw user HTML -- is reachable as `self.__rsc` until a script
+ * assigns the property, so the first row's push threw on it and every row
+ * was lost; two such elements made it a collection whose hrefs were read as
+ * rows. An assignment shadows the named element. And the router reads the
+ * FIRST metadata block, which a planted one could otherwise precede.
+ *
+ * @param {Uint8Array} head
+ * @param {number} offset from preludeOffset()
+ * @param {Uint8Array} prelude
+ */
+function withPrelude(head, offset, prelude) {
+  const out = new Uint8Array(head.length + prelude.length);
+  out.set(head.subarray(0, offset), 0);
+  out.set(prelude, offset);
+  out.set(head.subarray(offset), offset + prelude.length);
+  return out;
+}
+
+/** @param {Uint8Array} a @param {Uint8Array} b */
+function concat(a, b) {
+  if (a.length === 0) return b;
+  const out = new Uint8Array(a.length + b.length);
+  out.set(a, 0);
+  out.set(b, a.length);
+  return out;
+}
+
+// Fatal, so bytes that are not UTF-8 are found rather than replaced; and a
+// byte order mark is payload like any other byte, not something to strip.
+const strictUtf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
+
+/**
+ * How many bytes at the end of `bytes` begin a UTF-8 sequence that has not
+ * finished arriving: 0 to 3.
+ *
+ * @param {Uint8Array} bytes
+ */
+function unfinishedSequence(bytes) {
+  let lead = bytes.length - 1;
+  while (lead >= 0 && bytes.length - lead <= 3 && (bytes[lead] & 0xc0) === 0x80) lead--;
+  if (lead < 0) return 0;
+
+  const byte = bytes[lead];
+  const length = (byte & 0xe0) === 0xc0 ? 2 : (byte & 0xf0) === 0xe0 ? 3 : (byte & 0xf8) === 0xf0 ? 4 : 1;
+  const present = bytes.length - lead;
+  return present < length ? present : 0;
+}
+
+/**
+ * The payload as the arguments of `self.__rsc.push(...)`.
+ *
+ * Flight is bytes, not text: a Uint8Array or ArrayBuffer prop goes out as
+ * its raw bytes. Decoded as UTF-8 for the document and encoded again in the
+ * browser, every byte that was not valid UTF-8 came back as U+FFFD and the
+ * prop was corrupted. So a read that is valid UTF-8 is written as a string,
+ * as every row always was, and one that is not as {"$b64": "..."} -- the
+ * marker the json-lines codec uses for binary -- which the browser entry
+ * decodes with inlinePayloadStream() from @tinvar/react/client.
+ *
+ * A character split between two reads is held for the next one, so text
+ * that merely arrived in pieces is not mistaken for binary.
+ */
+class PayloadRows {
+  #held = new Uint8Array(0);
+
+  /**
+   * @param {Uint8Array} bytes
+   * @returns {string[]}
+   */
+  push(bytes) {
+    const all = concat(this.#held, bytes);
+    const complete = all.length - unfinishedSequence(all);
+
+    let text;
+    try {
+      text = strictUtf8.decode(all.subarray(0, complete));
+    } catch {
+      this.#held = new Uint8Array(0);
+      return [binaryRow(all)];
+    }
+
+    this.#held = all.slice(complete);
+    return text === '' ? [] : [scriptSafe(text)];
+  }
+
+  /** @returns {string[]} what is still held, which can only be binary now */
+  end() {
+    const held = this.#held;
+    this.#held = new Uint8Array(0);
+    return held.length === 0 ? [] : [binaryRow(held)];
+  }
+}
+
+/** @param {Uint8Array} bytes */
+function binaryRow(bytes) {
+  // Base64 is letters, digits, +, / and =: nothing that can close a script.
+  return `{"$b64":"${Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('base64')}"}`;
+}
+
+/**
+ * A CSP nonce as PHP mints one: base64, or its URL-safe alphabet. The
+ * renderer writes it into an attribute itself, so anything else -- a quote,
+ * a space, an angle bracket -- is refused rather than escaped: a nonce that
+ * is not one is a frame that is not PHP's, and the render fails closed.
+ */
+const NONCE = /^[A-Za-z0-9+/=_-]+$/;
+
+/** @param {unknown} nonce */
+function assertNonce(nonce) {
+  if (nonce === undefined || nonce === null || nonce === '') return;
+  if (typeof nonce !== 'string' || !NONCE.test(nonce)) {
+    throw new TypeError('The CSP nonce is not base64: refusing to write it into the document.');
+  }
+}
+
+function scriptTag(body, nonce) {
+  const attribute = nonce ? ` nonce="${nonce}"` : '';
+  return `<script${attribute}>${body}</script>`;
+}
+
+/**
+ * @param {ReadableStream<Uint8Array>} htmlStream
+ * @param {ReadableStream<Uint8Array>} rscStream
+ * @param {string} [nonce] the per-request CSP nonce PHP minted
+ * @param {{ url: string, version: string, routeToken?: string | null, view?: string | null, historyKey?: string | null, route?: string | null, hosts?: unknown }} [page]
+ *        the initial page metadata, written first in the document
+ * @returns {ReadableStream<Uint8Array>}
+ */
+export function injectRscPayload(htmlStream, rscStream, nonce, page) {
+  assertNonce(nonce);
+  // Built before anything streams, so a malformed frame fails the render
+  // rather than truncating a document already half sent.
+  const metadata = page === undefined || page === null ? '' : pageMetadataTag(page);
+  const prelude = encoder.encode(metadata + scriptTag('self.__rsc=[];self.__rsc_done=false', nonce));
+
+  const next = merge(htmlStream, rscStream, prelude, nonce);
+
+  // Pulled, not pumped: each piece is made when the reader asks for one,
+  // and highWaterMark 0 means nothing is made ahead of the asking. Drained
+  // eagerly from start(), both sources were read as fast as they produced,
+  // and a renderer whose PHP side was busy with a slow browser held the
+  // whole page in memory.
+  return new ReadableStream(
+    {
+      async pull(controller) {
+        const bytes = await next();
+        if (bytes === null) controller.close();
+        else controller.enqueue(bytes);
+      },
+      cancel(reason) {
+        return next.cancel(reason);
+      },
+    },
+    { highWaterMark: 0 },
+  );
+}
+
+/**
+ * The merged document, a piece at a time: `next()` resolves with the next
+ * bytes to write, or null once the document is complete.
+ *
+ * HTML passes through as it arrives. Payload rows are queued and written
+ * only between two of React's flushes. React writes a flush synchronously,
+ * in views of a fixed size, so one flush arrives as several chunks whose
+ * boundaries fall wherever a view filled -- inside a tag, inside an
+ * attribute. A row written as soon as it was ready landed between two of
+ * them, in the middle of the markup.
+ *
+ * Between flushes is when a read of the HTML has stayed unanswered for a
+ * whole macrotask: a flush is written in one go, so if any of it were
+ * waiting the read would have been answered within the task that asked.
+ * Only an outstanding read counts. A chunk read and not yet written says
+ * nothing about the rest of its flush, which may still be in the stream.
+ *
+ * The rows of one payload read are decoded and queued together, and the
+ * payload is read again only once they have been written, so rows wait for
+ * a gap without piling up behind one.
+ *
+ * @param {ReadableStream<Uint8Array>} htmlStream
+ * @param {ReadableStream<Uint8Array>} rscStream
+ * @param {Uint8Array} prelude
+ * @param {string | undefined} nonce
+ * @returns {(() => Promise<Uint8Array | null>) & { cancel: (reason: unknown) => Promise<void> }}
+ */
+function merge(htmlStream, rscStream, prelude, nonce) {
+  const html = htmlStream.getReader();
+  const rsc = rscStream.getReader();
+  const rows = new PayloadRows();
+
+  /** The document's first bytes, held until it is known whether they open with a doctype. */
+  let head = new Uint8Array(0);
+  /** The prelude is written, so the document has started and rows may follow. */
+  let started = false;
+  let htmlDone = false;
+  let rscDone = false;
+  /** An HTML chunk read and not yet written. */
+  let htmlChunk = null;
+  /** The HTML read still waiting for an answer, if there is one. */
+  let htmlRead = null;
+  /** htmlRead has stayed unanswered across a macrotask: React is between flushes. */
+  let betweenFlushes = false;
+  let rscRead = null;
+  /** Payload scripts waiting for a gap between flushes. */
+  const queued = [];
+  let failure = null;
+  let wake = null;
+
+  const changed = () => {
+    const resume = wake;
+    wake = null;
+    resume?.();
+  };
+
+  const fail = (error) => {
+    failure ??= error;
+    changed();
+  };
+
+  const queueRows = (pushed) => {
+    for (const row of pushed) queued.push(encoder.encode(scriptTag(`self.__rsc.push(${row})`, nonce)));
+  };
+
+  const readHtml = () => {
+    const read = html.read().then(({ done, value }) => {
+      htmlRead = null;
+      betweenFlushes = false;
+      if (done) htmlDone = true;
+      // An empty chunk is not the start of the document. Counting it as
+      // one let the metadata and payload rows out ahead of a doctype that
+      // arrived in the next chunk.
+      else if (value?.length) htmlChunk = value;
+      changed();
+    }, fail);
+    htmlRead = read;
+    betweenFlushes = false;
+    setImmediate(() => {
+      if (htmlRead !== read) return;
+      betweenFlushes = true;
+      changed();
+    });
+  };
+
+  const readRsc = () => {
+    rscRead = rsc.read().then(({ done, value }) => {
+      rscRead = null;
+      if (done) {
+        queueRows(rows.end());
+        queued.push(encoder.encode(scriptTag('self.__rsc_done=true', nonce)));
+        rscDone = true;
+      } else {
+        queueRows(rows.push(value));
+      }
+      changed();
+    }, fail);
+  };
+
+  const next = async () => {
+    for (;;) {
+      if (failure) {
+        await next.cancel(failure);
+        throw failure;
+      }
+
+      if (htmlChunk) {
+        const chunk = htmlChunk;
+        htmlChunk = null;
+        if (started) return chunk;
+
+        head = concat(head, chunk);
+        const offset = preludeOffset(head, false);
+        if (offset >= 0) {
+          started = true;
+          return withPrelude(head, offset, prelude);
+        }
+        continue;
+      }
+
+      // An HTML stream that ended without a byte never wrote the prelude;
+      // the rows depend on it.
+      if (htmlDone && !started) {
+        started = true;
+        return withPrelude(head, preludeOffset(head, true), prelude);
+      }
+
+      // The HTML is rendered FROM the payload, so rows are usually ready
+      // first -- and a script written before the doctype puts the browser
+      // in quirks mode. Rows wait for the document to start, and then for
+      // a gap between flushes.
+      if (started && queued.length > 0 && (htmlDone || betweenFlushes)) {
+        return concatAll(queued.splice(0));
+      }
+
+      if (htmlDone && rscDone && queued.length === 0) return null;
+
+      if (!htmlDone && !htmlRead) readHtml();
+      if (!rscDone && !rscRead && queued.length === 0) readRsc();
+      await new Promise((resolve) => (wake = resolve));
+    }
+  };
+
+  next.cancel = async (reason) => {
+    await Promise.allSettled([html.cancel(reason), rsc.cancel(reason)]);
+  };
+
+  return next;
+}
+
+/** @param {Uint8Array[]} parts */
+function concatAll(parts) {
+  if (parts.length === 1) return parts[0];
+  const out = new Uint8Array(parts.reduce((total, part) => total + part.length, 0));
+  let at = 0;
+  for (const part of parts) {
+    out.set(part, at);
+    at += part.length;
+  }
+  return out;
+}

package/src/log.js

@@ -0,0 +1,131 @@
+// The renderer's log lines about a render: one JSON object a line on
+// standard error, in the shape PHP's Logger writes (ADR 0041), carrying the
+// request id PHP put in the render frame.
+//
+//   {"time":"…","level":"error","message":"ssr error","source":"renderer",
+//    "request_id":"4bf92f3577b34da6a3ce929d0e0e4736","error":{…}}
+//
+// So the spec's "shared request ID in both logs" holds: one query for an id
+// finds what PHP and the renderer each wrote about that request. The id is
+// the only thing about telemetry the renderer is given -- no exporter, no
+// credentials, nothing to send anywhere; it writes to its own standard
+// error, which its parent owns.
+//
+// A failure's message and stack can quote what the render was handed --
+// loader data, a customer's address -- so both pass through the same
+// redaction PHP's log applies before they are written.
+
+/** The one shape a request id takes; anything else in the frame is not written. */
+const REQUEST_ID = /^[0-9a-f]{32}$/;
+
+/** Longest string kept, as PHP's Redactor keeps them. */
+const MAX_STRING = 4096;
+
+/** Mirror of Tinvar\Observability\Redactor::VALUES, in its order. */
+const VALUES = [
+  [/\b([a-z][a-z0-9+.-]*:\/\/[^\s/:]*):\S*@/gi, '$1:[redacted]@'],
+  [/\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, '$1 [redacted]'],
+  // The scheme in any case; the credentials by a digit, + / =, or a capital
+  // after the first letter -- spelled out, since /i would make [A-Z] any letter.
+  [
+    /\b([Bb][Aa][Ss][Ii][Cc])\s+(?=[A-Za-z0-9+/]*[0-9+/=]|[A-Za-z0-9+/]+[A-Z])[A-Za-z0-9+/]{8,}={0,2}/g,
+    '$1 [redacted]',
+  ],
+  [/\beyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*/g, '[jwt]'],
+  [/\bv1\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}/g, '[token]'],
+  [
+    /([?&;][a-z0-9_-]*(?:token|secret|passw|signature|sig|key|code|session|auth)[a-z0-9_-]*=)[^&\s#"']*/gi,
+    '$1[redacted]',
+  ],
+  [/[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}/g, '[email]'],
+  [
+    /(?<![A-Za-z0-9_-])(?=[A-Za-z0-9_-]*[0-9])(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9_-]{40,}(?![A-Za-z0-9_-])/g,
+    '[redacted]',
+  ],
+];
+
+/** The frame's request id, when it is one; undefined otherwise. */
+export function requestIdOf(frame) {
+  const id = frame?.rid;
+  return typeof id === 'string' && REQUEST_ID.test(id) ? id : undefined;
+}
+
+/** `text` with every recognisable secret and address cut out, and cut to MAX_STRING. */
+export function redact(text) {
+  let value = String(text);
+  const length = value.length;
+  // Cut, then back to the last whitespace or delimiter: a secret the cut
+  // splits is too short for its pattern, and replacements earlier in the
+  // window could otherwise pull its fragment inside the final cut.
+  if (value.length > MAX_STRING + 512)
+    value = value.slice(0, MAX_STRING + 512).replace(/[^\s"'<>()[\]{},;]*$/, '');
+  for (const [pattern, replacement] of VALUES) value = value.replace(pattern, replacement);
+  // Marked whenever it was longer than what is kept, even where redaction
+  // left it short enough: a cut line must not read as whole.
+  return Math.max(length, value.length) > MAX_STRING ? `${value.slice(0, MAX_STRING)}...` : value;
+}
+
+/**
+ * One line about the renderer itself rather than a render: its server's
+ * connections, the pool's supervisor (ADR 0112). The same JSON shape as a
+ * render's lines, with no request id -- there is no request -- and the pid,
+ * which tells a pool's renderers apart. The message is redacted as a
+ * render's is, since an error's text can quote what a frame carried; so is
+ * every string field.
+ *
+ * Info goes to standard output and anything worse to standard error, as
+ * the plain lines these replace did.
+ *
+ * @param {'debug' | 'info' | 'notice' | 'warning' | 'error'} level
+ * @param {string} message
+ * @param {Record<string, unknown>} fields
+ * @param {unknown} [error] what was thrown, described as a render's failure is
+ */
+export function logLine(level, message, fields = {}, error = undefined) {
+  const line = {
+    time: new Date().toISOString(),
+    level,
+    message: redact(message),
+    source: 'renderer',
+    pid: process.pid,
+    ...Object.fromEntries(
+      Object.entries(fields)
+        .filter(([, value]) => value !== undefined)
+        .map(([key, value]) => [key, typeof value === 'string' ? redact(value) : value]),
+    ),
+    ...(error === undefined ? {} : { error: describe(error) }),
+  };
+  const text = JSON.stringify(line);
+  if (level === 'debug' || level === 'info') console.log(text);
+  else console.error(text);
+}
+
+/**
+ * One error line about a render.
+ *
+ * @param {string} message what happened, in the renderer's own words
+ * @param {unknown} error what was thrown
+ * @param {Record<string, unknown>} fields the request id, and anything else to file it under
+ */
+export function logError(message, error, fields = {}) {
+  const line = {
+    time: new Date().toISOString(),
+    level: 'error',
+    message,
+    source: 'renderer',
+    ...Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== undefined)),
+    error: describe(error),
+  };
+  console.error(JSON.stringify(line));
+}
+
+function describe(error) {
+  if (error === null || typeof error !== 'object') return { message: redact(String(error)) };
+
+  return {
+    class: typeof error.name === 'string' ? error.name : 'Error',
+    message: redact(error.message ?? ''),
+    ...(typeof error.code === 'string' ? { code: error.code } : {}),
+    ...(typeof error.stack === 'string' ? { stack: redact(error.stack) } : {}),
+  };
+}