@nxtedition/nxt-undici 6.2.7 → 6.2.9

package/lib/interceptor/cache.js

@@ -1,5 +1,5 @@
 import undici from '@nxtedition/undici'
-import { DecoratorHandler, parseCacheControl } from '../utils.js'
+import { DecoratorHandler, parseCacheControl, parseContentRange } from '../utils.js'
 
 const DEFAULT_STORE = new undici.cacheStores.SqliteCacheStore({ location: ':memory:' })
 const DEFAULT_MAX_ENTRY_SIZE = 128 * 1024
@@ -25,11 +25,14 @@ class CacheHandler extends DecoratorHandler {
   onConnect(abort) {
     this.#value = null
 
-    super.onConnect(abort)
+    super.onConnect((reason) => {
+      // TODO (fix): Can we cache partial results?
+      abort(reason)
+    })
   }
 
   onHeaders(statusCode, headers, resume) {
-    if (statusCode !== 307 && statusCode !== 200) {
+    if (statusCode !== 307 && statusCode !== 200 && statusCode !== 206) {
       return super.onHeaders(statusCode, headers, resume)
     }
 
@@ -38,6 +41,16 @@ class CacheHandler extends DecoratorHandler {
       return super.onHeaders(statusCode, headers, resume)
     }
 
+    if (statusCode === 206 && !parseContentRange(headers['content-range'])) {
+      // We don't support caching range responses without content-range...
+      return super.onHeaders(statusCode, headers, resume)
+    }
+
+    if (statusCode === 206) {
+      // TODO (fix): enable range requests
+      return super.onHeaders(statusCode, headers, resume)
+    }
+
     const contentLength = headers['content-length'] ? Number(headers['content-length']) : Infinity
     if (Number.isFinite(contentLength) && contentLength > DEFAULT_MAX_ENTRY_SIZE) {
       // We don't support caching responses with body...
@@ -103,7 +116,7 @@ class CacheHandler extends DecoratorHandler {
       statusMessage: '',
       headers,
       cacheControlDirectives,
-      etag: typeof headers.etag === 'string' && isEtagUsable(headers.etag) ? headers.etag : '',
+      etag: isEtagUsable(headers.etag) ? headers.etag : '',
       vary,
       cachedAt,
       staleAt: 0,
@@ -129,6 +142,7 @@ class CacheHandler extends DecoratorHandler {
   onComplete(trailers) {
     if (this.#value && (!trailers || Object.keys(trailers).length === 0)) {
       this.#store.set(this.#key, this.#value)
+      this.#value = null
     }
 
     super.onComplete(trailers)
@@ -161,11 +175,80 @@ export default () => (dispatch) => (opts, handler) => {
     return dispatch(opts, handler)
   }
 
-  // Dump body...
-  opts.body?.on('error', () => {}).resume()
-
   const store = opts.cache.store ?? DEFAULT_STORE
+
+  // TODO (fix): enable range requests
   const entry = store.get(opts)
+
+  // let entry
+  // if (opts.headers.range) {
+  //   const range = parseRangeHeader(opts.headers.range)
+  //   if (!range) {
+  //     // Invalid range header...
+  //     return dispatch(opts, handler)
+  //   }
+
+  //   // TODO (perf): This is not optimal as all range bodies will be loaded...
+  //   // Make sure it only returns valid ranges by passing/parsing content range...
+  //   const entries = store.getAll(opts)
+
+  //   for (const x of entries) {
+  //     const { statusCode, headers, body } = x
+
+  //     if (!body) {
+  //       continue
+  //     }
+
+  //     let contentRange
+  //     if (statusCode === 200) {
+  //       // TODO (fix): Implement this...
+  //       // contentRange = { start: 0, end: body.byteLength }
+  //       // x = {
+  //       //   ...x,
+  //       //   headers: {
+  //       //     ...x,
+  //       //     'content-md5': undefined
+  //       //     // TODO (fix): What other headers need to be modified? accept-ranges? etag?
+  //       //   }
+  //       // }
+  //     } else if (statusCode === 206) {
+  //       contentRange = parseContentRange(headers?.['content-range'])
+  //     }
+
+  //     if (!contentRange) {
+  //       continue
+  //     }
+
+  //     if (contentRange.start === range.start && contentRange.end === range.end) {
+  //       entry = x
+  //       break
+  //     }
+
+  //     // TODO (fix): Implement this...
+  //     // const start = 0
+  //     // const end = contentRange.end - contentRange.start
+  //     // x = {
+  //     //   ...x,
+  //     //   body: body.subarray(start, end),
+  //     //   headers: {
+  //     //     ...headers,
+  //     //     'content-range': `bytes ${start}-${end - 1}/${contentRange.size ?? '*'}`
+  //     //     'content-md5': undefined
+  //     //      // TODO (fix): What other headers need to be modified? etag?
+  //     //   }
+  //     // }
+  //     // TODO (fix): Pick best entry... i.e. what ever fullfills most of the range
+  //   }
+  // } else {
+  //   entry = store.get(opts)
+
+  //   // TODO (fix): store.get is not optimal as it can return partial (206) responses.
+  //   // Make sure it only returns valid statusCodes.
+  //   if (entry?.statusCode === 206) {
+  //     return dispatch(opts, handler)
+  //   }
+  // }
+
   if (!entry && !cacheControlDirectives['only-if-cached']) {
     return dispatch(
       opts,
@@ -189,6 +272,9 @@ export default () => (dispatch) => (opts, handler) => {
     }
   }
 
+  // Dump body...
+  opts.body?.on('error', () => {}).resume()
+
   try {
     handler.onConnect(abort)
     if (aborted) {
@@ -219,10 +305,14 @@ export default () => (dispatch) => (opts, handler) => {
  *
  * @see https://www.rfc-editor.org/rfc/rfc9110.html#name-etag
  *
- * @param {string} etag
+ * @param {string|any} etag
  * @returns {boolean}
  */
 function isEtagUsable(etag) {
+  if (typeof etag !== 'string') {
+    return false
+  }
+
   if (etag.length <= 2) {
     // Shortest an etag can be is two chars (just ""). This is where we deviate
     //  from the spec requiring a min of 3 chars however

package/lib/sqlite-cache-store.js

@@ -0,0 +1,479 @@
+import { DatabaseSync } from 'node:sqlite'
+
+const VERSION = 4
+
+// 2gb
+const MAX_ENTRY_SIZE = 2 * 1000 * 1000 * 1000
+
+/**
+ * @typedef {import('undici-types/cache-interceptor.d.ts').default.CacheStore} CacheStore
+ * @implements {CacheStore}
+ *
+ * @typedef {{
+ *  id: Readonly<number>,
+ *  body?: Uint8Array
+ *  statusCode: number
+ *  statusMessage: string
+ *  headers?: string
+ *  vary?: string
+ *  etag?: string
+ *  cacheControlDirectives?: string
+ *  cachedAt: number
+ *  staleAt: number
+ *  deleteAt: number
+ * }} SqliteStoreValue
+ */
+export class SqliteCacheStore {
+  #maxEntrySize = MAX_ENTRY_SIZE
+  #maxEntryCount = 16 * 1024
+
+  /**
+   * @type {import('node:sqlite').DatabaseSync}
+   */
+  #db
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #getValuesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #insertValueQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #deleteExpiredValuesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #deleteByUrlQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #countEntriesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync | null}
+   */
+  #deleteOldValuesQuery
+
+  /**
+   * @param {import('undici-types/cache-interceptor.d.ts').default.SqliteCacheStoreOpts & { maxEntryCount?: number} | undefined} opts
+   */
+  constructor(opts) {
+    if (opts) {
+      if (typeof opts !== 'object') {
+        throw new TypeError('SqliteCacheStore options must be an object')
+      }
+
+      if (opts.maxEntrySize !== undefined) {
+        if (
+          typeof opts.maxEntrySize !== 'number' ||
+          !Number.isInteger(opts.maxEntrySize) ||
+          opts.maxEntrySize < 0
+        ) {
+          throw new TypeError(
+            'SqliteCacheStore options.maxEntrySize must be a non-negative integer',
+          )
+        }
+
+        if (opts.maxEntrySize > MAX_ENTRY_SIZE) {
+          throw new TypeError('SqliteCacheStore options.maxEntrySize must be less than 2gb')
+        }
+
+        this.#maxEntrySize = opts.maxEntrySize
+      }
+
+      const maxEntryCount = opts.maxEntryCount ?? opts.maxCount
+      if (maxEntryCount !== undefined) {
+        if (
+          typeof maxEntryCount !== 'number' ||
+          !Number.isInteger(maxEntryCount) ||
+          maxEntryCount < 0
+        ) {
+          throw new TypeError(
+            'SqliteCacheStore options.maxEntryCount must be a non-negative integer',
+          )
+        }
+        this.#maxEntryCount = maxEntryCount
+      }
+    }
+
+    this.#db = new DatabaseSync(opts?.location ?? ':memory:')
+
+    this.#db.exec(`
+      CREATE TABLE IF NOT EXISTS cacheInterceptorV${VERSION} (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        method TEXT NOT NULL,
+        body BUF NULL,
+        deleteAt INTEGER NOT NULL,
+        statusCode INTEGER NOT NULL,
+        statusMessage TEXT NOT NULL,
+        headers TEXT NULL,
+        cacheControlDirectives TEXT NULL,
+        etag TEXT NULL,
+        vary TEXT NULL,
+        cachedAt INTEGER NOT NULL,
+        staleAt INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_url ON cacheInterceptorV${VERSION}(url);
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_method ON cacheInterceptorV${VERSION}(method);
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_deleteAt ON cacheInterceptorV${VERSION}(deleteAt);
+    `)
+
+    this.#getValuesQuery = this.#db.prepare(`
+      SELECT
+        id,
+        body,
+        deleteAt,
+        statusCode,
+        statusMessage,
+        headers,
+        etag,
+        cacheControlDirectives,
+        vary,
+        cachedAt,
+        staleAt
+      FROM cacheInterceptorV${VERSION}
+      WHERE
+        url = ?
+        AND method = ?
+      ORDER BY
+        deleteAt ASC
+    `)
+
+    this.#insertValueQuery = this.#db.prepare(`
+      INSERT INTO cacheInterceptorV${VERSION} (
+        url,
+        method,
+        body,
+        deleteAt,
+        statusCode,
+        statusMessage,
+        headers,
+        etag,
+        cacheControlDirectives,
+        vary,
+        cachedAt,
+        staleAt
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `)
+
+    this.#deleteByUrlQuery = this.#db.prepare(
+      `DELETE FROM cacheInterceptorV${VERSION} WHERE url = ?`,
+    )
+
+    this.#countEntriesQuery = this.#db.prepare(
+      `SELECT COUNT(*) AS total FROM cacheInterceptorV${VERSION}`,
+    )
+
+    this.#deleteExpiredValuesQuery = this.#db.prepare(
+      `DELETE FROM cacheInterceptorV${VERSION} WHERE deleteAt <= ?`,
+    )
+
+    this.#deleteOldValuesQuery =
+      this.#maxEntryCount === Infinity
+        ? null
+        : this.#db.prepare(`
+        DELETE FROM cacheInterceptorV${VERSION}
+        WHERE id IN (
+          SELECT
+            id
+          FROM cacheInterceptorV${VERSION}
+          ORDER BY cachedAt DESC
+          LIMIT ?
+        )
+      `)
+  }
+
+  close() {
+    this.#db.close()
+  }
+
+  /**
+   * @param {import('undici-types/cache-interceptor.d.ts').default.CacheKey} key
+   * @returns {(import('undici-types/cache-interceptor.d.ts').default.GetResult & { body?: Buffer }) | undefined}
+   */
+  get(key) {
+    assertCacheKey(key)
+
+    const value = this.#findValue(key)
+    return value ? makeResult(value) : undefined
+  }
+
+  /**
+   * @param {import('undici-types/cache-interceptor.d.ts').default.CacheKey} key
+   * @returns {(import('undici-types/cache-interceptor.d.ts').default.GetResult & { body?: Buffer })[]}
+   */
+  getAll(key) {
+    assertCacheKey(key)
+
+    return this.#findValues(key).map((value) => makeResult(value))
+  }
+
+  /**
+   * @param {import('undici-types/cache-interceptor.d.ts').default.CacheKey} key
+   * @param {import('undici-types/cache-interceptor.d.ts').default.CacheValue & { body: null | Buffer | Array<Buffer>}} value
+   */
+  set(key, value) {
+    assertCacheKey(key)
+    assertCacheValue(value)
+
+    const body = Array.isArray(value.body) ? Buffer.concat(value.body) : value.body
+    if ((body?.byteLength ?? 0) > this.#maxEntrySize) {
+      return
+    }
+
+    this.#prune()
+
+    this.#insertValueQuery.run(
+      makeValueUrl(key),
+      key.method,
+      body,
+      value.deleteAt,
+      value.statusCode,
+      value.statusMessage,
+      value.headers ? JSON.stringify(value.headers) : null,
+      value.etag ? value.etag : null,
+      value.cacheControlDirectives ? JSON.stringify(value.cacheControlDirectives) : null,
+      value.vary ? JSON.stringify(value.vary) : null,
+      value.cachedAt,
+      value.staleAt,
+    )
+  }
+
+  /**
+   * @param {import('undici-types/cache-interceptor.d.ts').default.CacheKey} key
+   */
+  delete(key) {
+    if (typeof key !== 'object') {
+      throw new TypeError(`expected key to be object, got ${typeof key}`)
+    }
+
+    this.#deleteByUrlQuery.run(makeValueUrl(key))
+  }
+
+  #prune() {
+    if (this.size <= this.#maxEntryCount) {
+      return 0
+    }
+
+    {
+      const removed = this.#deleteExpiredValuesQuery.run(Date.now()).changes
+      if (removed) {
+        return removed
+      }
+    }
+
+    {
+      const removed = this.#deleteOldValuesQuery?.run(
+        Math.max(Math.floor(this.#maxEntryCount * 0.1), 1),
+      ).changes
+      if (removed) {
+        return removed
+      }
+    }
+
+    return 0
+  }
+
+  /**
+   * Counts the number of rows in the cache
+   * @returns {Number}
+   */
+  get size() {
+    const { total } = this.#countEntriesQuery.get()
+    return total
+  }
+
+  /**
+   * @param {import('undici-types/cache-interceptor.d.ts').default.CacheKey} key
+   * @param {boolean} [canBeExpired=false]
+   * @returns {SqliteStoreValue | undefined}
+   */
+  #findValue(key, canBeExpired = false) {
+    const { headers, method } = key
+
+    /**
+     * @type {SqliteStoreValue[]}
+     */
+    const values = this.#getValuesQuery.all(makeValueUrl(key), method)
+
+    if (values.length === 0) {
+      return undefined
+    }
+
+    const now = Date.now()
+    for (const value of values) {
+      if (now >= value.deleteAt && !canBeExpired) {
+        return undefined
+      }
+
+      let matches = true
+
+      if (value.vary) {
+        const vary = JSON.parse(value.vary)
+
+        for (const header in vary) {
+          if (!headerValueEquals(headers?.[header], vary[header])) {
+            matches = false
+            break
+          }
+        }
+      }
+
+      if (matches) {
+        return value
+      }
+    }
+
+    return undefined
+  }
+
+  /**
+   * @param {import('undici-types/cache-interceptor.d.ts').default.CacheKey} key
+   * @param {boolean} [canBeExpired=false]
+   * @returns {SqliteStoreValue[]}
+   */
+  #findValues(key, canBeExpired = false) {
+    const { headers, method } = key
+
+    /**
+     * @type {SqliteStoreValue[]}
+     */
+    const values = this.#getValuesQuery.all(makeValueUrl(key), method)
+    const matches = []
+    const now = Date.now()
+    for (const value of values) {
+      if (now >= value.deleteAt && !canBeExpired) {
+        continue
+      }
+
+      if (value.vary) {
+        const vary = JSON.parse(value.vary)
+
+        for (const header in vary) {
+          if (!headerValueEquals(headers?.[header], vary[header])) {
+            break
+          }
+          matches.push(value)
+        }
+      } else {
+        matches.push(value)
+      }
+    }
+
+    return matches
+  }
+}
+
+/**
+ * @param {string|string[]|null|undefined} lhs
+ * @param {string|string[]|null|undefined} rhs
+ * @returns {boolean}
+ */
+function headerValueEquals(lhs, rhs) {
+  if (lhs == null && rhs == null) {
+    return true
+  }
+
+  if ((lhs == null && rhs != null) || (lhs != null && rhs == null)) {
+    return false
+  }
+
+  if (Array.isArray(lhs) && Array.isArray(rhs)) {
+    if (lhs.length !== rhs.length) {
+      return false
+    }
+
+    return lhs.every((x, i) => x === rhs[i])
+  }
+
+  return lhs === rhs
+}
+
+/**
+ * @param {import('undici-types/cache-interceptor.d.ts').default.CacheKey} key
+ * @returns {string}
+ */
+function makeValueUrl(key) {
+  return `${key.origin}/${key.path}`
+}
+
+function makeResult(value) {
+  return {
+    body: value.body
+      ? Buffer.from(value.body.buffer, value.body.byteOffset, value.body.byteLength)
+      : undefined,
+    statusCode: value.statusCode,
+    statusMessage: value.statusMessage,
+    headers: value.headers ? JSON.parse(value.headers) : undefined,
+    etag: value.etag ? value.etag : undefined,
+    vary: value.vary ? JSON.parse(value.vary) : undefined,
+    cacheControlDirectives: value.cacheControlDirectives
+      ? JSON.parse(value.cacheControlDirectives)
+      : undefined,
+    cachedAt: value.cachedAt,
+    staleAt: value.staleAt,
+    deleteAt: value.deleteAt,
+  }
+}
+
+/**
+ * @param {any} key
+ */
+function assertCacheKey(key) {
+  if (typeof key !== 'object') {
+    throw new TypeError(`expected key to be object, got ${typeof key}`)
+  }
+
+  for (const property of ['origin', 'method', 'path']) {
+    if (typeof key[property] !== 'string') {
+      throw new TypeError(`expected key.${property} to be string, got ${typeof key[property]}`)
+    }
+  }
+
+  if (key.headers !== undefined && typeof key.headers !== 'object') {
+    throw new TypeError(`expected headers to be object, got ${typeof key}`)
+  }
+}
+
+/**
+ * @param {any} value
+ */
+function assertCacheValue(value) {
+  if (typeof value !== 'object') {
+    throw new TypeError(`expected value to be object, got ${typeof value}`)
+  }
+
+  for (const property of ['statusCode', 'cachedAt', 'staleAt', 'deleteAt']) {
+    if (typeof value[property] !== 'number') {
+      throw new TypeError(`expected value.${property} to be number, got ${typeof value[property]}`)
+    }
+  }
+
+  if (typeof value.statusMessage !== 'string') {
+    throw new TypeError(
+      `expected value.statusMessage to be string, got ${typeof value.statusMessage}`,
+    )
+  }
+
+  if (value.headers != null && typeof value.headers !== 'object') {
+    throw new TypeError(`expected value.rawHeaders to be object, got ${typeof value.headers}`)
+  }
+
+  if (value.vary !== undefined && typeof value.vary !== 'object') {
+    throw new TypeError(`expected value.vary to be object, got ${typeof value.vary}`)
+  }
+
+  if (value.etag !== undefined && typeof value.etag !== 'string') {
+    throw new TypeError(`expected value.etag to be string, got ${typeof value.etag}`)
+  }
+}

package/lib/utils.js

@@ -17,23 +17,6 @@ export function parseCacheControl(str) {
   return str ? cacheControlParser.parse(str) : null
 }
 
-// Parsed accordingly to RFC 9110
-// https://www.rfc-editor.org/rfc/rfc9110#field.content-range
-export function parseRangeHeader(range) {
-  if (range == null || range === '') {
-    return { start: 0, end: null, size: null }
-  }
-
-  const m = range ? range.match(/^bytes (\d+)-(\d+)\/(\d+)?$/) : null
-  return m
-    ? {
-        start: parseInt(m[1]),
-        end: m[2] ? parseInt(m[2]) + 1 : null,
-        size: m[3] ? parseInt(m[3]) : null,
-      }
-    : null
-}
-
 export function isDisturbed(body) {
   if (
     body == null ||
@@ -51,32 +34,59 @@ export function isDisturbed(body) {
   return stream.isDisturbed(body)
 }
 
+/**
+ * @typedef {object} RangeHeader
+ * @property {number} start
+ * @property {number | null} end
+ * @property {number | null} size
+ */
+
+/**
+ * @param {string} [range]
+ * @returns {RangeHeader|null|undefined}
+ */
 export function parseContentRange(range) {
-  if (typeof range !== 'string') {
-    return null
+  if (range == null || range === '') {
+    return undefined
   }
 
-  const m = range.match(/^bytes (\d+)-(\d+)?\/(\d+|\*)$/)
-  if (!m) {
+  if (typeof range !== 'string') {
     return null
   }
 
-  const start = m[1] == null ? null : Number(m[1])
-  if (!Number.isFinite(start)) {
-    return null
-  }
+  const m = range.match(/^bytes (\d+)-(\d+)?\/(\d+|\*)$/)
+  return m
+    ? {
+        start: parseInt(m[1], 10),
+        end: m[2] ? parseInt(m[2]) + 1 : null,
+        size: m[3] === '*' ? null : parseInt(m[3]),
+      }
+    : null
+}
 
-  const end = m[2] == null ? null : Number(m[2])
-  if (end !== null && !Number.isFinite(end)) {
-    return null
+// Parsed accordingly to RFC 9110
+// https://www.rfc-editor.org/rfc/rfc9110#field.content-range
+/**
+ * @param {string} [range]
+ * @returns {RangeHeader|null|undefined}
+ */
+export function parseRangeHeader(range) {
+  if (range == null || range === '') {
+    return undefined
   }
 
-  const size = m[2] === '*' ? null : Number(m[2])
-  if (size !== null && !Number.isFinite(size)) {
+  if (typeof range !== 'string') {
     return null
   }
 
-  return { start, end: end ? end + 1 : size, size }
+  const m = range.match(/^bytes (\d+)-(\d+)\/(\d+)?$/)
+  return m
+    ? {
+        start: parseInt(m[1]),
+        end: m[2] ? parseInt(m[2]) + 1 : null,
+        size: m[3] ? parseInt(m[3]) : null,
+      }
+    : null
 }
 
 export function parseURL(url) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/nxt-undici",
-  "version": "6.2.7",
+  "version": "6.2.9",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "main": "lib/index.js",
@@ -9,21 +9,21 @@
     "lib/*"
   ],
   "dependencies": {
-    "@nxtedition/undici": "^10.1.2",
+    "@nxtedition/undici": "^11.0.0",
     "cache-control-parser": "^2.0.6",
     "http-errors": "^2.0.0"
   },
   "devDependencies": {
-    "@types/node": "^22.10.7",
-    "eslint": "^9.16.0",
-    "eslint-plugin-n": "^17.14.0",
+    "@types/node": "^22.13.10",
+    "eslint": "^9.22.0",
+    "eslint-plugin-n": "^17.16.2",
     "husky": "^9.1.7",
-    "lint-staged": "^15.4.1",
+    "lint-staged": "^15.5.0",
     "pinst": "^3.0.0",
-    "prettier": "^3.4.1",
+    "prettier": "^3.5.3",
     "send": "^1.1.0",
-    "tap": "^21.0.1",
-    "undici-types": "^7.2.3"
+    "tap": "^21.1.0",
+    "undici-types": "^7.5.0"
   },
   "scripts": {
     "prepare": "husky",