@nxtedition/nxt-undici 1.0.4 → 1.0.6

package/lib/index.js

@@ -4,6 +4,8 @@ const undici = require('undici')
 const stream = require('stream')
 const { parseHeaders } = require('./utils')
 
+const dispatcherCache = new WeakMap()
+
 // https://github.com/fastify/fastify/blob/main/lib/reqIdGenFactory.js
 // 2,147,483,647 (2^31 − 1) stands for max SMI value (an internal optimization of V8).
 // With this upper bound, if you'll be generating 1k ids/sec, you're going to hit it in ~25 days.
@@ -87,6 +89,7 @@ const dispatchers = {
   signal: require('./interceptor/signal.js'),
   proxy: require('./interceptor/proxy.js'),
   cache: require('./interceptor/cache.js'),
+  requestId: require('./interceptor/request-id.js'),
 }
 
 async function request(url, opts) {
@@ -114,10 +117,9 @@ async function request(url, opts) {
     headers = opts.headers
   }
 
-  headers = {
-    'request-id': genReqId(),
-    'user-agent': opts.userAgent ?? globalThis.userAgent,
-    ...headers,
+  const userAgent = opts.userAgent ?? globalThis.userAgent
+  if (userAgent && headers?.['user-agent'] !== userAgent) {
+    headers = { 'user-agent': userAgent, ...headers }
   }
 
   if (method === 'CONNECT') {
@@ -125,127 +127,138 @@ async function request(url, opts) {
   }
 
   if (
+    headers != null &&
     (method === 'HEAD' || method === 'GET') &&
     (parseInt(headers['content-length']) > 0 || headers['transfer-encoding'])
   ) {
     throw new createError.BadRequest('HEAD and GET cannot have body')
   }
 
-  opts = {
-    url,
-    method,
-    body: opts.body,
-    headers,
-    origin: url.origin,
-    path: url.path ? url.path : url.search ? `${url.pathname}${url.search ?? ''}` : url.pathname,
-    reset: opts.reset ?? false,
-    headersTimeout: opts.headersTimeout,
-    bodyTimeout: opts.bodyTimeout,
-    idempotent,
-    signal: opts.signal,
-    retry: opts.retry ?? 8,
-    proxy: opts.proxy,
-    cache: opts.cache,
-    upgrade: opts.upgrade,
-    follow: { count: opts.maxRedirections ?? 8, ...opts.redirect, ...opts.follow },
-    logger: opts.logger,
-    maxRedirections: 0, // Disable undici's redirect handling.
-  }
-
   const expectsPayload = opts.method === 'PUT' || opts.method === 'POST' || opts.method === 'PATCH'
 
-  if (opts.headers['content-length'] === '0' && !expectsPayload) {
+  if (headers != null && headers['content-length'] === '0' && !expectsPayload) {
     // https://tools.ietf.org/html/rfc7230#section-3.3.2
     // A user agent SHOULD NOT send a Content-Length header field when
     // the request message does not contain a payload body and the method
     // semantics do not anticipate such a body.
 
     // undici will error if provided an unexpected content-length: 0 header.
-    delete opts.headers['content-length']
+    headers = { ...headers }
+    delete headers['content-length']
   }
 
   const dispatcher = opts.dispatcher ?? undici.getGlobalDispatcher()
 
-  return new Promise((resolve) => {
-    let dispatch = (opts, handler) => dispatcher.dispatch(opts, handler)
-
+  let dispatch = dispatcherCache.get(dispatcher)
+  if (dispatch == null) {
+    dispatch = (opts, handler) => dispatcher.dispatch(opts, handler)
     dispatch = dispatchers.catch(dispatch)
     dispatch = dispatchers.abort(dispatch)
     dispatch = dispatchers.log(dispatch)
-    dispatch = opts.upgrade ? dispatch : dispatchers.responseRetry(dispatch)
-    dispatch = opts.upgrade ? dispatch : dispatchers.responseStatusRetry(dispatch)
-    dispatch = opts.upgrade ? dispatch : dispatchers.responseBodyRetry(dispatch)
-    dispatch = opts.upgrade ? dispatch : dispatchers.content(dispatch)
+    dispatch = dispatchers.requestId(dispatch)
+    dispatch = dispatchers.responseRetry(dispatch)
+    dispatch = dispatchers.responseStatusRetry(dispatch)
+    dispatch = dispatchers.responseBodyRetry(dispatch)
+    dispatch = dispatchers.content(dispatch)
     dispatch = dispatchers.redirect(dispatch)
     dispatch = dispatchers.signal(dispatch)
-    dispatch = opts.upgrade ? dispatch : dispatchers.cache(dispatch)
+    dispatch = dispatchers.cache(dispatch)
     dispatch = dispatchers.proxy(dispatch)
+    dispatcherCache.set(dispatcher, dispatch)
+  }
 
-    dispatch(opts, {
-      resolve,
-      logger: opts.logger,
-      /** @type {Function | null} */ abort: null,
-      /** @type {stream.Readable | null} */ body: null,
-      onConnect(abort) {
-        this.abort = abort
+  return new Promise((resolve, reject) =>
+    dispatch(
+      {
+        id: opts.id ?? genReqId(),
+        url,
+        method,
+        body: opts.body,
+        headers,
+        origin: url.origin,
+        path: url.path ?? (url.search ? `${url.pathname}${url.search ?? ''}` : url.pathname),
+        query: opts.query,
+        reset: opts.reset ?? false,
+        headersTimeout: opts.headersTimeout,
+        bodyTimeout: opts.bodyTimeout,
+        idempotent,
+        signal: opts.signal,
+        retry: opts.retry ?? 8,
+        proxy: opts.proxy,
+        cache: opts.cache,
+        upgrade: opts.upgrade,
+        follow: opts.follow ?? 8,
+        logger: opts.logger,
       },
-      onUpgrade(statusCode, rawHeaders, socket) {
-        const headers = parseHeaders(rawHeaders)
+      {
+        resolve,
+        reject,
+        logger: opts.logger,
+        /** @type {Function | null} */ abort: null,
+        /** @type {stream.Readable | null} */ body: null,
+        onConnect(abort) {
+          this.abort = abort
+        },
+        onUpgrade(statusCode, rawHeaders, socket) {
+          const headers = parseHeaders(rawHeaders)
 
-        if (statusCode !== 101) {
-          this.abort(createError(statusCode, { headers }))
-        } else {
-          this.resolve({ headers, socket })
-        }
-      },
-      onHeaders(statusCode, rawHeaders, resume, statusMessage) {
-        assert(this.abort)
-
-        const headers = parseHeaders(rawHeaders)
-
-        if (statusCode >= 400) {
-          this.abort(createError(statusCode, { headers }))
-        } else {
-          assert(statusCode >= 200)
-
-          const contentLength = Number(headers['content-length'] ?? headers['Content-Length'])
-
-          this.body = new Readable({
-            read: resume,
-            highWaterMark: 128 * 1024,
-            statusCode,
-            statusMessage,
-            headers,
-            size: Number.isFinite(contentLength) ? contentLength : null,
-          }).on('error', (err) => {
-            if (this.logger && this.body?.listenerCount('error') === 1) {
-              this.logger.error({ err }, 'unhandled response body error')
-            }
-          })
-
-          this.resolve(this.body)
-          this.resolve = null
-        }
+          if (statusCode !== 101) {
+            this.abort(createError(statusCode, { headers }))
+          } else {
+            this.resolve({ headers, socket })
+          }
+        },
+        onBodySent(chunk) {},
+        onHeaders(statusCode, rawHeaders, resume, statusMessage) {
+          assert(this.abort)
 
-        return false
-      },
-      onData(chunk) {
-        assert(this.body)
-        return this.body.push(chunk)
-      },
-      onComplete() {
-        assert(this.body)
-        this.body.push(null)
-      },
-      onError(err) {
-        if (this.body) {
-          this.body.destroy(err)
-        } else {
-          this.resolve(Promise.reject(err))
-        }
+          const headers = parseHeaders(rawHeaders)
+
+          if (statusCode >= 400) {
+            this.abort(createError(statusCode, { headers }))
+          } else {
+            assert(statusCode >= 200)
+
+            const contentLength = Number(headers['content-length'] ?? headers['Content-Length'])
+
+            this.body = new Readable({
+              read: resume,
+              highWaterMark: 128 * 1024,
+              statusCode,
+              statusMessage,
+              headers,
+              size: Number.isFinite(contentLength) ? contentLength : null,
+            }).on('error', (err) => {
+              if (this.logger && this.body?.listenerCount('error') === 1) {
+                this.logger.error({ err }, 'unhandled response body error')
+              }
+            })
+
+            this.resolve(this.body)
+            this.resolve = null
+          }
+
+          return false
+        },
+        onData(chunk) {
+          assert(this.body)
+          return this.body.push(chunk)
+        },
+        onComplete() {
+          assert(this.body)
+          this.body.push(null)
+        },
+        onError(err) {
+          if (this.body) {
+            this.body.destroy(err)
+          } else {
+            this.reject(err)
+            this.reject = null
+          }
+        },
       },
-    })
-  })
+    ),
+  )
 }
 
 module.exports = { request }

package/lib/interceptor/content.js

@@ -1,6 +1,5 @@
 const crypto = require('node:crypto')
 const stream = require('node:stream')
-const assert = require('node:assert')
 const { findHeader, isStream } = require('../utils')
 
 class Handler {
@@ -8,7 +7,6 @@ class Handler {
     this.handler = handler
     this.md5 = null
     this.length = null
-
     this.hasher = null
     this.pos = 0
   }
@@ -28,9 +26,7 @@ class Handler {
   onHeaders(statusCode, rawHeaders, resume, statusMessage) {
     this.md5 = findHeader(rawHeaders, 'content-md5')
     this.length = findHeader(rawHeaders, 'content-length')
-
     this.hasher = this.md5 != null ? crypto.createHash('md5') : null
-
     return this.handler.onHeaders(statusCode, rawHeaders, resume, statusMessage)
   }
 
@@ -67,6 +63,11 @@ class Handler {
 }
 
 module.exports = (dispatch) => (opts, handler) => {
+  if (opts.upgrade) {
+    return dispatch(opts, handler)
+  }
+
+  // TODO (fix): case-insensitive check?
   const md5 = opts.headers?.['content-md5'] ?? opts.headers?.['Content-MD5']
   const length = opts.headers?.['content-lenght'] ?? opts.headers?.['Content-Length']
 
@@ -86,7 +87,7 @@ module.exports = (dispatch) => (opts, handler) => {
           transform(chunk, encoding, callback) {
             pos += chunk.length
             hasher?.update(chunk)
-            callback(null)
+            callback(null, chunk)
           },
           final(callback) {
             const hash = hasher?.digest('base64')
@@ -133,7 +134,7 @@ module.exports = (dispatch) => (opts, handler) => {
       })
     }
   } else {
-    assert(false, 'not implemented')
+    throw new Error('not implemented')
   }
 
   return dispatch(opts, new Handler(opts, { handler }))

package/lib/interceptor/log.js

@@ -1,26 +1,13 @@
 const { parseHeaders } = require('../utils')
 const { performance } = require('perf_hooks')
 
-// https://github.com/fastify/fastify/blob/main/lib/reqIdGenFactory.js
-// 2,147,483,647 (2^31 − 1) stands for max SMI value (an internal optimization of V8).
-// With this upper bound, if you'll be generating 1k ids/sec, you're going to hit it in ~25 days.
-// This is very likely to happen in real-world applications, hence the limit is enforced.
-// Growing beyond this value will make the id generation slower and cause a deopt.
-// In the worst cases, it will become a float, losing accuracy.
-const maxInt = 2147483647
-let nextReqId = Math.floor(Math.random() * maxInt)
-function genReqId() {
-  nextReqId = (nextReqId + 1) & maxInt
-  return `req-${nextReqId.toString(36)}`
-}
-
 class Handler {
   constructor(opts, { handler }) {
     this.handler = handler
-    this.opts = opts.id ? opts : { ...opts, id: genReqId() }
+    this.opts = opts
     this.abort = null
     this.aborted = false
-    this.logger = opts.logger.child({ ureq: { id: opts.id } })
+    this.logger = opts.logger.child({ ureq: opts })
     this.pos = 0
     this.startTime = 0
   }
@@ -28,7 +15,7 @@ class Handler {
   onConnect(abort) {
     this.abort = abort
     this.startTime = performance.now()
-    this.logger.debug({ ureq: this.opts }, 'upstream request started')
+    this.logger.debug('upstream request started')
     this.handler.onConnect((reason) => {
       this.aborted = true
       this.abort(reason)
@@ -36,9 +23,9 @@ class Handler {
   }
 
   onUpgrade(statusCode, rawHeaders, socket) {
-    this.logger.debug({ ureq: this.opts }, 'upstream request upgraded')
+    this.logger.debug('upstream request upgraded')
     socket.on('close', () => {
-      this.logger.debug({ ureq: this.opts }, 'upstream request socket closed')
+      this.logger.debug('upstream request socket closed')
     })
     return this.handler.onUpgrade(statusCode, rawHeaders, socket)
   }

package/lib/interceptor/proxy.js

@@ -102,7 +102,7 @@ const HOP_EXPR =
 function forEachHeader(headers, fn) {
   if (Array.isArray(headers)) {
     for (let n = 0; n < headers.length; n += 2) {
-      fn(headers[n + 0].toString(), headers[n + 1].toString())
+      fn(headers[n + 0], headers[n + 1])
     }
   } else {
     for (const [key, val] of Object.entries(headers)) {
@@ -123,16 +123,16 @@ function reduceHeaders({ headers, proxyName, httpVersion, socket }, fn, acc) {
 
   forEachHeader(headers, (key, val) => {
     const len = key.length
-    if (len === 3 && !via && key.toLowerCase() === 'via') {
-      via = val
-    } else if (len === 4 && !host && key.toLowerCase() === 'host') {
-      host = val
-    } else if (len === 9 && !forwarded && key.toLowerCase() === 'forwarded') {
-      forwarded = val
-    } else if (len === 10 && !connection && key.toLowerCase() === 'connection') {
-      connection = val
-    } else if (len === 10 && !authority && key.toLowerCase() === ':authority') {
-      authority = val
+    if (len === 3 && !via && key.toString().toLowerCase() === 'via') {
+      via = val.toString()
+    } else if (len === 4 && !host && key.toString().toLowerCase() === 'host') {
+      host = val.toString()
+    } else if (len === 9 && !forwarded && key.toString().toLowerCase() === 'forwarded') {
+      forwarded = val.toString()
+    } else if (len === 10 && !connection && key.toString().toLowerCase() === 'connection') {
+      connection = val.toString()
+    } else if (len === 10 && !authority && key.toString().toLowerCase() === ':authority') {
+      authority = val.toString()
     }
   })
 
@@ -142,8 +142,10 @@ function reduceHeaders({ headers, proxyName, httpVersion, socket }, fn, acc) {
   }
 
   forEachHeader(headers, (key, val) => {
+    key = key.toString()
+
     if (key.charAt(0) !== ':' && !remove.includes(key) && !HOP_EXPR.test(key)) {
-      acc = fn(acc, key, val)
+      acc = fn(acc, key, val.toString())
     }
   })
 

package/lib/interceptor/redirect.js

@@ -11,6 +11,7 @@ class Handler {
     this.abort = null
     this.aborted = false
     this.reason = null
+    this.maxCount = Number.isFinite(opts.follow) ? opts.follow : opts.follow?.count ?? 0
 
     this.count = 0
     this.location = null
@@ -34,11 +35,15 @@ class Handler {
   }
 
   onUpgrade(statusCode, headers, socket) {
-    this.handler.onUpgrade(statusCode, headers, socket)
+    return this.handler.onUpgrade(statusCode, headers, socket)
+  }
+
+  onBodySent(chunk) {
+    return this.handler.onBodySent(chunk)
   }
 
   onError(error) {
-    this.handler.onError(error)
+    return this.handler.onError(error)
   }
 
   onHeaders(statusCode, headers, resume, statusText) {
@@ -53,7 +58,7 @@ class Handler {
     this.location = findHeader(headers, 'location')
 
     if (!this.location) {
-      throw new Error(`Missing redirection location.`)
+      throw new Error(`Missing redirection location .`)
     }
 
     this.count += 1
@@ -63,10 +68,8 @@ class Handler {
         return this.handler.onHeaders(statusCode, headers, resume, statusText)
       }
     } else {
-      const maxCount = this.opts.follow.count ?? 0
-
-      if (this.count >= maxCount) {
-        throw new Error(`Max redirections reached: ${maxCount}.`)
+      if (this.count >= this.maxCount) {
+        throw new Error(`Max redirections reached: ${this.maxCount}.`)
       }
     }
 
@@ -136,13 +139,7 @@ class Handler {
 
       this.dispatch(this.opts, this)
     } else {
-      this.handler.onComplete(trailers)
-    }
-  }
-
-  onBodySent(chunk) {
-    if (this.handler.onBodySent) {
-      this.handler.onBodySent(chunk)
+      return this.handler.onComplete(trailers)
     }
   }
 }
@@ -161,17 +158,12 @@ function shouldRemoveHeader(header, removeContent, unknownOrigin) {
 
 // https://tools.ietf.org/html/rfc7231#section-6.4
 function cleanRequestHeaders(headers, removeContent, unknownOrigin) {
-  const ret = []
-  if (Array.isArray(headers)) {
-    for (let i = 0; i < headers.length; i += 2) {
-      if (!shouldRemoveHeader(headers[i], removeContent, unknownOrigin)) {
-        ret.push(headers[i], headers[i + 1])
-      }
-    }
-  } else if (headers && typeof headers === 'object') {
+  let ret
+  if (headers && typeof headers === 'object') {
     for (const key of Object.keys(headers)) {
       if (!shouldRemoveHeader(key, removeContent, unknownOrigin)) {
-        ret.push(key, headers[key])
+        ret ??= {}
+        ret[key] = headers[key]
       }
     }
   } else {
@@ -181,4 +173,6 @@ function cleanRequestHeaders(headers, removeContent, unknownOrigin) {
 }
 
 module.exports = (dispatch) => (opts, handler) =>
-  opts.follow ? dispatch(opts, new Handler(opts, { handler, dispatch })) : dispatch(opts, handler)
+  opts.follow != null
+    ? dispatch(opts, new Handler(opts, { handler, dispatch }))
+    : dispatch(opts, handler)

package/lib/interceptor/request-id.js

@@ -0,0 +1,29 @@
+// https://github.com/fastify/fastify/blob/main/lib/reqIdGenFactory.js
+// 2,147,483,647 (2^31 − 1) stands for max SMI value (an internal optimization of V8).
+// With this upper bound, if you'll be generating 1k ids/sec, you're going to hit it in ~25 days.
+// This is very likely to happen in real-world applications, hence the limit is enforced.
+// Growing beyond this value will make the id generation slower and cause a deopt.
+// In the worst cases, it will become a float, losing accuracy.
+const maxInt = 2147483647
+let nextReqId = Math.floor(Math.random() * maxInt)
+function genReqId() {
+  nextReqId = (nextReqId + 1) & maxInt
+  return `req-${nextReqId.toString(36)}`
+}
+
+module.exports = (dispatch) => (opts, handler) => {
+  let id = opts.id ?? opts.headers?.['request-id'] ?? opts.headers?.['Request-Id']
+  id = id ? `${id},${genReqId()}` : genReqId()
+
+  return dispatch(
+    {
+      ...opts,
+      id,
+      headers: {
+        ...opts.headers,
+        'request-id': id,
+      },
+    },
+    handler,
+  )
+}

package/lib/interceptor/response-retry.js

@@ -83,6 +83,6 @@ class Handler {
 }
 
 module.exports = (dispatch) => (opts, handler) =>
-  opts.idempotent && opts.retry
+  opts.idempotent && opts.retry && !opts.upgrade
     ? dispatch(opts, new Handler(opts, { handler, dispatch }))
     : dispatch(opts, handler)

package/lib/interceptor/signal.js

@@ -34,12 +34,16 @@ class Handler {
   }
 
   onComplete(rawTrailers) {
-    this.signal.removeEventListener('abort', this.abort)
+    if (this.abort) {
+      this.signal.removeEventListener('abort', this.abort)
+    }
     return this.handler.onComplete(rawTrailers)
   }
 
   onError(err) {
-    this.signal.removeEventListener('abort', this.abort)
+    if (this.abort) {
+      this.signal.removeEventListener('abort', this.abort)
+    }
     return this.handler.onError(err)
   }
 }

package/lib/utils.js

@@ -204,9 +204,43 @@ function isStream(obj) {
   )
 }
 
+// based on https://github.com/node-fetch/fetch-blob/blob/8ab587d34080de94140b54f07168451e7d0b655e/index.js#L229-L241 (MIT License)
+function isBlobLike(object) {
+  return (
+    (Blob && object instanceof Blob) ||
+    (object &&
+      typeof object === 'object' &&
+      (typeof object.stream === 'function' || typeof object.arrayBuffer === 'function') &&
+      /^(Blob|File)$/.test(object[Symbol.toStringTag]))
+  )
+}
+
+function isBuffer(buffer) {
+  // See, https://github.com/mcollina/undici/pull/319
+  return buffer instanceof Uint8Array || Buffer.isBuffer(buffer)
+}
+
+function bodyLength(body) {
+  if (body == null) {
+    return 0
+  } else if (isStream(body)) {
+    const state = body._readableState
+    return state && state.ended === true && Number.isFinite(state.length) ? state.length : null
+  } else if (isBlobLike(body)) {
+    return body.size != null ? body.size : null
+  } else if (isBuffer(body)) {
+    return body.byteLength
+  }
+
+  return null
+}
+
 module.exports = {
   isStream,
+  isBuffer,
+  isBlobLike,
   AbortError,
+  bodyLength,
   parseHeaders,
   isDisturbed,
   parseContentRange,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/nxt-undici",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "main": "lib/index.js",
@@ -18,6 +18,9 @@
     "eslint": "^8.50.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-config-standard": "^17.0.0",
+    "eslint-plugin-import": "^2.28.1",
+    "eslint-plugin-n": "^16.2.0",
+    "eslint-plugin-promise": "^6.1.1",
     "husky": "^8.0.3",
     "lint-staged": "^14.0.1",
     "pinst": "^3.0.0",