fulmine.js 5.15.0 → 5.16.0

package/README.md

@@ -296,6 +296,10 @@ A single-stage `node:26-trixie-slim` image works too if you `apt-get install -y
 
 ## Differences from Express
 
+What the two servers answer on the wire, probed from outside, malformed input and smuggling
+attempts included: [fulmine.js on http-probe.com](https://www.http-probe.com/servers/fulmine-js.html)
+against [express](https://www.http-probe.com/servers/express.html).
+
 - `app.listen()` returns the app rather than a separate server object, and the app answers as an `http.Server`: `app instanceof http.Server` is true, which is what the graceful shutdown wrappers and the connection trackers look for. There is still no node server underneath, the socket belongs to µWS, so what is answered is the surface and not the plumbing. There: `close()`, `address()`, `listening`, `getConnections()`, `ref()`, `unref()`, `setTimeout()` and the `keepAliveTimeout` family. Not there: nothing emits `connection`, `request` or `upgrade`, `getConnections()` counts the requests in flight rather than sockets, and the timeouts belong to µWS and are set through `uwsOptions.idleTimeout`. Anything that wants to serve its own protocol on the socket, socket.io being the usual case, still wants `app.uwsApp`. Runnable: [`examples/graceful-shutdown.js`](./examples/graceful-shutdown.js).
 - `x-powered-by` is disabled by default. Express sends `X-Powered-By: Express` unless you turn it off; Fulmine does not send it unless you turn it on with `app.set("x-powered-by", true)`. The header only tells anyone asking which framework is running.
 - request body is only read for POST, PUT, PATCH and QUERY requests by default. You can add additional methods by setting `body methods` to array with uppercased methods.
@@ -498,6 +502,13 @@ Server-Timing: route;desc="native", hdr;desc="not copied", db;dur=3.62, total;du
 app.use(express.compression({ threshold: 1024 }));
 ```
 
+One option is Fulmine's own, `encodings`: the list of what the middleware may answer with, out of `"br"`, `"gzip"` and `"deflate"`. What is not named is never used, however the client ranks it, and an uncompressed answer is always on offer. It exists because the preferred encoding is a cost decision, not only a size one: brotli compresses smaller but what it costs per response depends on the machine, and on a CPU where it runs expensive `encodings: ["gzip"]` buys the cheaper call for every client that accepts both.
+
+```js
+// answer gzip even to a client that also accepts br
+app.use(express.compression({ level: 1, encodings: ["gzip"] }));
+```
+
 Runnable: [`examples/compression.js`](./examples/compression.js).
 
 5. If a route answers with a JSON shape you know in advance, [express-fast-json-stringify](https://www.npmjs.com/package/express-fast-json-stringify) compiles that shape into a serializer and `res.fastJson()` replaces `res.json()`. `JSON.stringify()` has to walk an object it knows nothing about; a compiled serializer does not. It is worth reaching for, and a CPU profile says why: on a route answering 3.6KB of JSON, serialising it is about 25% of the time that is not spent waiting, ahead of the ETag at 19% and of everything the framework does to route the request and build its request and response objects.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "fulmine.js",
-    "version": "5.15.0",
+    "version": "5.16.0",
     "description": "Drop-in Express 5 replacement on uWebSockets.js. Your existing middleware keeps working.",
     "main": "src/index.js",
     "exports": {

package/src/compression.js

@@ -36,7 +36,23 @@ limitations under the License.
 const zlib = require("zlib");
 const bytes = require("bytes");
 const compressible = require("compressible");
-const { negotiateEncoding, ENCODING_ANY, memoizeByString } = require("./utils.js");
+const {
+    negotiateEncoding,
+    ENCODING_ANY,
+    ENCODING_BR,
+    ENCODING_GZIP,
+    ENCODING_DEFLATE,
+    memoizeByString
+} = require("./utils.js");
+
+// what the `encodings` option may name, and the mask each name contributes. identity is 0: an
+// uncompressed answer is always on offer, naming it only makes the list read complete
+const ENCODING_MASKS = new Map([
+    ["br", ENCODING_BR],
+    ["gzip", ENCODING_GZIP],
+    ["deflate", ENCODING_DEFLATE],
+    ["identity", 0]
+]);
 
 // Cache-Control: no-transform forbids recoding the body, which is what this does
 const NO_TRANSFORM = /(?:^|,)\s*?no-transform\s*?(?:,|$)/;
@@ -135,6 +151,11 @@ function toBuffer(chunk, encoding) {
  * @param {string} [options.enforceEncoding] what to use when the request carries no
  *   Accept-Encoding at all. Default "identity", which is to say nothing is compressed.
  * @param {object} [options.brotli] brotli options, `params` included. The default quality is 4.
+ * @param {string[]} [options.encodings] the encodings this middleware may answer with, out of
+ *   "br", "gzip" and "deflate". What is not named is never used, however the client ranks it: a
+ *   server that prefers cheap gzip over brotli passes ["gzip", "deflate"]. An uncompressed answer
+ *   is always on offer, and enforceEncoding stays its own explicit choice, outside this list.
+ *   This option is fulmine's own, the compression module has no equivalent.
  * @param {number} [options.level] zlib compression level, for gzip and deflate.
  * @param {number} [options.chunkSize] zlib chunk size.
  * @param {number} [options.memLevel] zlib memory level.
@@ -157,6 +178,22 @@ function compression(options) {
     // bytes.parse reads "1kb" and hands back null for anything it cannot, an absent option
     // included, which is where the default comes in
     const threshold = bytes.parse(/** @type {any} */ (opts.threshold)) ?? 1024;
+    // the mask handed to the negotiation, built once here: a name nobody knows is a config
+    // mistake and throws now rather than serving the wrong bytes later
+    let allowed = ENCODING_ANY;
+    if (opts.encodings !== undefined) {
+        if (!Array.isArray(opts.encodings)) {
+            throw new TypeError("encodings must be an array of encoding names");
+        }
+        allowed = 0;
+        for (const name of opts.encodings) {
+            const mask = ENCODING_MASKS.get(name);
+            if (mask === undefined) {
+                throw new TypeError(`unknown encoding "${name}" in encodings`);
+            }
+            allowed |= mask;
+        }
+    }
 
     /**
      * A whole body, compressed on this thread. Blocks the event loop for as long as it takes,
@@ -213,8 +250,14 @@ function compression(options) {
         // has to carry. Most requests to most routes cannot: a client that sent no Accept-Encoding,
         // one that refused everything, a HEAD. Those get the Vary and nothing else, since the
         // answer still depends on the header even when this particular client did not ask.
-        const accept = req.headers["accept-encoding"];
-        let chosen = negotiateEncoding(accept === undefined ? "" : accept, ENCODING_ANY);
+        // straight from the raw entries where this request keeps them: reading req.headers here
+        // built the whole object for one name. Folded, so a repeated Accept-Encoding still reads
+        // as the joined list the headers object would have shown
+        const accept =
+            typeof req._foldedHeader === "function"
+                ? req._foldedHeader("accept-encoding")
+                : req.headers["accept-encoding"];
+        let chosen = negotiateEncoding(accept === undefined ? "" : accept, allowed);
         if (accept === undefined && ENFORCEABLE.has(enforceEncoding)) {
             chosen = enforceEncoding;
         }

package/src/middlewares.js

@@ -29,6 +29,20 @@ const qs = require("qs");
 const parseQuery = require("./parse-query.js");
 const { kGetSafe } = require("./usage.js");
 const { AsyncResource } = require("async_hooks");
+
+/**
+ * What AsyncResource.bind answers, without node's generic wrapper: that one builds a rest-args
+ * closure and defines properties onto it per call, ~1.9us on this node, where the resource plus
+ * an arrow through runInAsyncScope restores the same context for ~0.08. The type keeps the bound
+ * function's name, as node's does.
+ *
+ * @param {(...args: any[]) => any} fn called with at most one argument by every caller here
+ * @returns {(err?: any) => any}
+ */
+function bindContext(fn) {
+    const resource = new AsyncResource(fn.name || "bound-anonymous-fn");
+    return (err) => resource.runInAsyncScope(fn, undefined, err);
+}
 const {
     fastQueryParse,
     NullObject,
@@ -758,7 +772,7 @@ function serveStatic(root, options) {
         return res.sendFile(
             _path,
             options,
-            AsyncResource.bind((e) => {
+            bindContext((e) => {
                 if (e) {
                     next(options.fallthrough && FALLTHROUGH_STATUSES.has(e.status) ? undefined : e);
                 }
@@ -911,12 +925,14 @@ function createBodyParser(defaultType, beforeReturn, checkOptions, charsetPolicy
             }
 
             const length = req._rawHeader("content-length");
+            // converted once: four sites read this number on the fast path
+            const lengthNumber = length === undefined ? NaN : +length;
 
             // No content-length and no transfer-encoding means the request carries no body at all,
             // and a body parser must leave it alone rather than parse nothing into an empty value.
             // type-is applies this before matching the type, but the simpleType shortcut below
             // compares strings directly and would otherwise skip the check.
-            if (req._rawHeader("transfer-encoding") === undefined && isNaN(length)) {
+            if (req._rawHeader("transfer-encoding") === undefined && Number.isNaN(lengthNumber)) {
                 return next();
             }
 
@@ -960,7 +976,7 @@ function createBodyParser(defaultType, beforeReturn, checkOptions, charsetPolicy
             // {} for json and urlencoded, '' for text, an empty Buffer for raw - rather than leaving
             // req.body as the placeholder object. there is nothing to read, so run the tail directly,
             // and the verify hook still runs first: webhook signature checks rely on that
-            if (Number(length) === 0) {
+            if (lengthNumber === 0) {
                 req.bodyRead = true;
                 const empty = Buffer.alloc(0);
                 if (!runVerify(req, res, next, options, empty)) {
@@ -969,12 +985,12 @@ function createBodyParser(defaultType, beforeReturn, checkOptions, charsetPolicy
                 return beforeReturn(req, res, next, options, empty, encoding);
             }
 
-            // skip reading too large body
-            if (length && +length > limit) {
+            // skip reading too large body; NaN compares false, so no declared length passes
+            if (lengthNumber > limit) {
                 return next(
                     bodyError("request entity too large", 413, "entity.too.large", {
-                        expected: +length,
-                        length: +length,
+                        expected: lengthNumber,
+                        length: lengthNumber,
                         limit: limit
                     })
                 );
@@ -1024,7 +1040,7 @@ function createBodyParser(defaultType, beforeReturn, checkOptions, charsetPolicy
             // From here the body really gets read, and uWS delivers it on native callbacks that
             // carry no async context, so this is the one continuation that has to be bound: an
             // upstream middleware's AsyncLocalStorage must still be there when next runs
-            next = AsyncResource.bind(next);
+            next = bindContext(next);
 
             // with nothing to decompress, uWS can collect the whole body in native code: one
             // callback instead of one per chunk, the limit enforced before any byte reaches JS,
@@ -1032,8 +1048,8 @@ function createBodyParser(defaultType, beforeReturn, checkOptions, charsetPolicy
             // returns, so a view over uWS's own memory is enough. A declared length was the
             // original case; a chunked body accumulates in the same native vector and only loses
             // the length check, since there is no declaration to hold it to
-            const declared = Number(length);
-            const declaresLength = !isNaN(declared) && declared > 0;
+            const declared = lengthNumber;
+            const declaresLength = !Number.isNaN(declared) && declared > 0;
             if (!req.receivedData && !inflate && req._res.collectBody && (declaresLength || isNaN(declared))) {
                 req.bodyRead = true;
                 req._res.collectBody(limit, (body) => {
@@ -1358,50 +1374,58 @@ const urlencoded = createBodyParser(
     function (req, res, next, options, buf, encoding) {
         try {
             const body = decodeBody(buf, encoding);
-            const count = parameterCount(body, options.parameterLimit);
-            if (count === undefined) {
-                return next(bodyError("too many parameters", 413, "parameters.too.many"));
-            }
             // Express 5 defaults extended to false, so nested keys need opting in
             const extended = typeof options.extended !== "undefined" ? options.extended : false;
             // qs has to know the charset itself for anything but utf-8, and the sentinel options
             // change what a parse means, so those bodies skip the fast parsers
             const needsQs = encoding !== "utf-8" || options.charsetSentinel || options.interpretNumericEntities;
-            if (extended) {
-                // the ceiling body-parser gives qs: the array limit rises to the parameter count,
-                // so a form posting 150 array members still yields an array. count counts "&"
-                // separators where body-parser counts parameters, hence the + 1
-                const qsOptions = {
-                    ...EXTENDED_QS_OPTIONS,
-                    depth: options.depth !== undefined ? options.depth : 32,
-                    arrayLimit: Math.max(100, count + 1),
-                    charsetSentinel: options.charsetSentinel,
-                    interpretNumericEntities: options.interpretNumericEntities,
-                    charset: encoding,
-                    parameterLimit: options.parameterLimit
-                };
-                req.body = needsQs
-                    ? Object.assign(Object.create(null), qs.parse(body, qsOptions))
-                    : fastQueryParse(body, qsOptions);
-            } else if (needsQs) {
-                // body-parser's extended: false is still qs, with depth 0 and the count as the
-                // array ceiling; only qs decodes latin1 percent escapes as latin1
-                req.body = Object.assign(
-                    Object.create(null),
-                    qs.parse(body, {
-                        allowPrototypes: true,
-                        arrayLimit: count + 1,
-                        depth: 0,
-                        strictDepth: true,
+            if (!extended && !needsQs) {
+                // the vendored parser, so an urlencoded body inspects like req.query does. The
+                // parameter limit is enforced inside its scan, so the body is not walked twice;
+                // assigned only when it held, so an overflow leaves req.body the placeholder
+                const parsed = parseQuery(body, undefined, options.parameterLimit);
+                if (parseQuery.overflow === true) {
+                    return next(bodyError("too many parameters", 413, "parameters.too.many"));
+                }
+                req.body = parsed;
+            } else {
+                const count = parameterCount(body, options.parameterLimit);
+                if (count === undefined) {
+                    return next(bodyError("too many parameters", 413, "parameters.too.many"));
+                }
+                if (extended) {
+                    // the ceiling body-parser gives qs: the array limit rises to the parameter
+                    // count, so a form posting 150 array members still yields an array. count
+                    // counts "&" separators where body-parser counts parameters, hence the + 1
+                    const qsOptions = {
+                        ...EXTENDED_QS_OPTIONS,
+                        depth: options.depth !== undefined ? options.depth : 32,
+                        arrayLimit: Math.max(100, count + 1),
                         charsetSentinel: options.charsetSentinel,
                         interpretNumericEntities: options.interpretNumericEntities,
                         charset: encoding,
                         parameterLimit: options.parameterLimit
-                    })
-                );
-            } else {
-                // the vendored parser, so an urlencoded body inspects like req.query does
-                req.body = parseQuery(body);
+                    };
+                    req.body = needsQs
+                        ? Object.assign(Object.create(null), qs.parse(body, qsOptions))
+                        : fastQueryParse(body, qsOptions);
+                } else {
+                    // body-parser's extended: false is still qs, with depth 0 and the count as
+                    // the array ceiling; only qs decodes latin1 percent escapes as latin1
+                    req.body = Object.assign(
+                        Object.create(null),
+                        qs.parse(body, {
+                            allowPrototypes: true,
+                            arrayLimit: count + 1,
+                            depth: 0,
+                            strictDepth: true,
+                            charsetSentinel: options.charsetSentinel,
+                            interpretNumericEntities: options.interpretNumericEntities,
+                            charset: encoding,
+                            parameterLimit: options.parameterLimit
+                        })
+                    );
+                }
             }
         } catch (e) {
             // qs reports a depth overflow as a RangeError with its own wording; body-parser

package/src/parse-query.js

@@ -34,17 +34,30 @@ const plusRegex = /\+/g;
  * node's querystring.parse semantics on a null-prototype result: repeated keys accumulate into
  * arrays, '+' is a space, percent sequences decode when present and stay literal when broken.
  *
+ * `capture` collects the decoded pairs flat, key then value, so a caller can replay the stores
+ * without scanning again; a repeated key marks it invalid instead. See `get query`.
+ *
+ * `separatorLimit` refuses a body with that many "&" separators the way body-parser's
+ * parameterCount does, but inside this scan instead of a scan of its own: the overflow flag on
+ * the function is set, the partial result is to be discarded, and the caller answers 413.
+ *
  * @param {string} input
+ * @param {string[] & {invalid?: boolean}} [capture]
+ * @param {number} [separatorLimit]
  * @returns {Record<string, string | string[]>}
  */
-function parseQuery(input) {
+function parseQuery(input, capture, separatorLimit) {
     const result = Object.create(null);
+    if (separatorLimit !== undefined) {
+        parseQuery.overflow = false;
+    }
 
     if (typeof input !== "string") {
         return result;
     }
 
     const inputLength = input.length;
+    let separators = 0;
     let key;
     let value = "";
     let startingIndex = -1;
@@ -62,6 +75,12 @@ function parseQuery(input) {
 
         // '&' or the end of the input closes the current pair
         if (c === 38) {
+            // real separators only, not the synthetic closing one, counted exactly as
+            // parameterCount counts them
+            if (i !== inputLength && separatorLimit !== undefined && ++separators === separatorLimit) {
+                parseQuery.overflow = true;
+                return result;
+            }
             hasBothKeyValuePair = equalityIndex > startingIndex;
 
             // the equality index doubles as the end of the key when there was no '='
@@ -92,7 +111,13 @@ function parseQuery(input) {
                 const currentValue = result[key];
                 if (currentValue === undefined) {
                     result[key] = value;
+                    if (capture !== undefined) {
+                        capture.push(key, value);
+                    }
                 } else {
+                    if (capture !== undefined) {
+                        capture.invalid = true;
+                    }
                     // value.pop is cheaper than Array.isArray here, as upstream measured
                     if (currentValue.pop) {
                         currentValue.push(value);
@@ -134,4 +159,7 @@ function parseQuery(input) {
     return result;
 }
 
+// whether the last limited call hit its separator limit, see the parameter's doc
+parseQuery.overflow = false;
+
 module.exports = parseQuery;