@nxtedition/nxt-undici 5.1.8 → 5.2.1

package/lib/cache/sqlite-cache-store.js

@@ -0,0 +1,439 @@
+import { assertCacheKey, assertCacheValue } from './util.js'
+
+let DatabaseSync
+try {
+  DatabaseSync = await import('node:sqlite').then((x) => x.DatabaseSync)
+} catch {
+  // Do nothing...
+}
+
+const VERSION = 3
+
+// 2gb
+const MAX_ENTRY_SIZE = 2 * 1000 * 1000 * 1000
+
+/**
+ * @typedef {import('../../types/cache-interceptor.d.ts').default.CacheStore} CacheStore
+ * @implements {CacheStore}
+ *
+ * @typedef {{
+ *  id: Readonly<number>
+ *  headers?: Record<string, string | string[]>
+ *  vary?: string | object
+ *  body: string
+ * } & import('../../types/cache-interceptor.d.ts').default.CacheValue} SqliteStoreValue
+ */
+export class SqliteCacheStore {
+  #maxEntrySize = MAX_ENTRY_SIZE
+  #maxCount = Infinity
+
+  /**
+   * @type {import('node:sqlite').DatabaseSync}
+   */
+  #db
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #getValuesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #updateValueQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #insertValueQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #deleteExpiredValuesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #deleteByUrlQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #countEntriesQuery
+
+  /**
+   * @type {import('node:sqlite').StatementSync}
+   */
+  #deleteOldValuesQuery
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.SqliteCacheStoreOpts | undefined} opts
+   */
+  constructor(opts) {
+    if (!DatabaseSync) {
+      throw new Error('SqliteCacheStore requires node:sqlite')
+    }
+
+    if (opts) {
+      if (typeof opts !== 'object') {
+        throw new TypeError('SqliteCacheStore options must be an object')
+      }
+
+      if (opts.maxEntrySize !== undefined) {
+        if (
+          typeof opts.maxEntrySize !== 'number' ||
+          !Number.isInteger(opts.maxEntrySize) ||
+          opts.maxEntrySize < 0
+        ) {
+          throw new TypeError(
+            'SqliteCacheStore options.maxEntrySize must be a non-negative integer',
+          )
+        }
+
+        if (opts.maxEntrySize > MAX_ENTRY_SIZE) {
+          throw new TypeError('SqliteCacheStore options.maxEntrySize must be less than 2gb')
+        }
+
+        this.#maxEntrySize = opts.maxEntrySize
+      }
+
+      if (opts.maxCount !== undefined) {
+        if (
+          typeof opts.maxCount !== 'number' ||
+          !Number.isInteger(opts.maxCount) ||
+          opts.maxCount < 0
+        ) {
+          throw new TypeError('SqliteCacheStore options.maxCount must be a non-negative integer')
+        }
+        this.#maxCount = opts.maxCount
+      }
+    }
+
+    this.#db = new DatabaseSync(opts?.location ?? ':memory:')
+
+    this.#db.exec(`
+      CREATE TABLE IF NOT EXISTS cacheInterceptorV${VERSION} (
+        -- Data specific to us
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT NOT NULL,
+        method TEXT NOT NULL,
+
+        -- Data returned to the interceptor
+        body BUF NULL,
+        deleteAt INTEGER NOT NULL,
+        statusCode INTEGER NOT NULL,
+        statusMessage TEXT NOT NULL,
+        headers TEXT NULL,
+        cacheControlDirectives TEXT NULL,
+        etag TEXT NULL,
+        vary TEXT NULL,
+        cachedAt INTEGER NOT NULL,
+        staleAt INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_url ON cacheInterceptorV${VERSION}(url);
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_method ON cacheInterceptorV${VERSION}(method);
+      CREATE INDEX IF NOT EXISTS idx_cacheInterceptorV${VERSION}_deleteAt ON cacheInterceptorV${VERSION}(deleteAt);
+    `)
+
+    this.#getValuesQuery = this.#db.prepare(`
+      SELECT
+        id,
+        body,
+        deleteAt,
+        statusCode,
+        statusMessage,
+        headers,
+        etag,
+        cacheControlDirectives,
+        vary,
+        cachedAt,
+        staleAt
+      FROM cacheInterceptorV${VERSION}
+      WHERE
+        url = ?
+        AND method = ?
+      ORDER BY
+        deleteAt ASC
+    `)
+
+    this.#updateValueQuery = this.#db.prepare(`
+      UPDATE cacheInterceptorV${VERSION} SET
+        body = ?,
+        deleteAt = ?,
+        statusCode = ?,
+        statusMessage = ?,
+        headers = ?,
+        etag = ?,
+        cacheControlDirectives = ?,
+        cachedAt = ?,
+        staleAt = ?,
+        deleteAt = ?
+      WHERE
+        id = ?
+    `)
+
+    this.#insertValueQuery = this.#db.prepare(`
+      INSERT INTO cacheInterceptorV${VERSION} (
+        url,
+        method,
+        body,
+        deleteAt,
+        statusCode,
+        statusMessage,
+        headers,
+        etag,
+        cacheControlDirectives,
+        vary,
+        cachedAt,
+        staleAt,
+        deleteAt
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `)
+
+    this.#deleteByUrlQuery = this.#db.prepare(
+      `DELETE FROM cacheInterceptorV${VERSION} WHERE url = ?`,
+    )
+
+    this.#countEntriesQuery = this.#db.prepare(
+      `SELECT COUNT(*) AS total FROM cacheInterceptorV${VERSION}`,
+    )
+
+    this.#deleteExpiredValuesQuery = this.#db.prepare(
+      `DELETE FROM cacheInterceptorV${VERSION} WHERE deleteAt <= ?`,
+    )
+
+    this.#deleteOldValuesQuery =
+      this.#maxCount === Infinity
+        ? null
+        : this.#db.prepare(`
+        DELETE FROM cacheInterceptorV${VERSION}
+        WHERE id IN (
+          SELECT
+            id
+          FROM cacheInterceptorV${VERSION}
+          ORDER BY cachedAt DESC
+          LIMIT ?
+        )
+      `)
+  }
+
+  close() {
+    this.#db.close()
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   * @returns {import('../../types/cache-interceptor.d.ts').default.GetResult | undefined}
+   */
+  get(key) {
+    assertCacheKey(key)
+
+    const value = this.#findValue(key)
+
+    if (!value) {
+      return undefined
+    }
+
+    /**
+     * @type {import('../../types/cache-interceptor.d.ts').default.GetResult}
+     */
+    const result = {
+      body: Buffer.from(value.body),
+      statusCode: value.statusCode,
+      statusMessage: value.statusMessage,
+      headers: value.headers ? JSON.parse(value.headers) : undefined,
+      etag: value.etag ? value.etag : undefined,
+      vary: value.vary ?? undefined,
+      cacheControlDirectives: value.cacheControlDirectives
+        ? JSON.parse(value.cacheControlDirectives)
+        : undefined,
+      cachedAt: value.cachedAt,
+      staleAt: value.staleAt,
+      deleteAt: value.deleteAt,
+    }
+
+    return result
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheValue & { body: Buffer | null }} value
+   */
+  set(key, value) {
+    assertCacheKey(value)
+    assertCacheValue(value)
+
+    const body = value.body
+    const size = body ? body.byteLength : 0
+
+    if (size > this.#maxEntrySize) {
+      return false
+    }
+
+    const url = this.#makeValueUrl(key)
+
+    const existingValue = this.#findValue(key, true)
+    if (existingValue) {
+      // Updating an existing response, let's overwrite it
+      this.#updateValueQuery.run(
+        body,
+        value.deleteAt,
+        value.statusCode,
+        value.statusMessage,
+        value.headers ? JSON.stringify(value.headers) : null,
+        value.etag ? value.etag : null,
+        value.cacheControlDirectives ? JSON.stringify(value.cacheControlDirectives) : null,
+        value.cachedAt,
+        value.staleAt,
+        value.deleteAt,
+        existingValue.id,
+      )
+    } else {
+      this.#prune()
+      // New response, let's insert it
+      this.#insertValueQuery.run(
+        url,
+        key.method,
+        body,
+        value.deleteAt,
+        value.statusCode,
+        value.statusMessage,
+        value.headers ? JSON.stringify(value.headers) : null,
+        value.etag ? value.etag : null,
+        value.cacheControlDirectives ? JSON.stringify(value.cacheControlDirectives) : null,
+        value.vary ? JSON.stringify(value.vary) : null,
+        value.cachedAt,
+        value.staleAt,
+        value.deleteAt,
+      )
+    }
+
+    return true
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   */
+  delete(key) {
+    if (typeof key !== 'object') {
+      throw new TypeError(`expected key to be object, got ${typeof key}`)
+    }
+
+    this.#deleteByUrlQuery.run(this.#makeValueUrl(key))
+  }
+
+  #prune() {
+    if (this.size <= this.#maxCount) {
+      return 0
+    }
+
+    {
+      const removed = this.#deleteExpiredValuesQuery.run(Date.now()).changes
+      if (removed > 0) {
+        return removed
+      }
+    }
+
+    {
+      const removed = this.#deleteOldValuesQuery.run(
+        Math.max(Math.floor(this.#maxCount * 0.1), 1),
+      ).changes
+      if (removed > 0) {
+        return removed
+      }
+    }
+
+    return 0
+  }
+
+  /**
+   * Counts the number of rows in the cache
+   * @returns {Number}
+   */
+  get size() {
+    const { total } = this.#countEntriesQuery.get()
+    return total
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   * @returns {string}
+   */
+  #makeValueUrl(key) {
+    return `${key.origin}/${key.path}`
+  }
+
+  /**
+   * @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} key
+   * @param {boolean} [canBeExpired=false]
+   * @returns {(SqliteStoreValue & { vary?: Record<string, string[]> }) | undefined}
+   */
+  #findValue(key, canBeExpired = false) {
+    const url = this.#makeValueUrl(key)
+    const { headers, method } = key
+
+    /**
+     * @type {SqliteStoreValue[]}
+     */
+    const values = this.#getValuesQuery.all(url, method)
+
+    if (values.length === 0) {
+      return undefined
+    }
+
+    const now = Date.now()
+    for (const value of values) {
+      if (now >= value.deleteAt && !canBeExpired) {
+        return undefined
+      }
+
+      let matches = true
+
+      if (value.vary) {
+        if (!headers) {
+          return undefined
+        }
+
+        value.vary = JSON.parse(value.vary)
+
+        for (const header in value.vary) {
+          if (!headerValueEquals(headers[header], value.vary[header])) {
+            matches = false
+            break
+          }
+        }
+      }
+
+      if (matches) {
+        return value
+      }
+    }
+
+    return undefined
+  }
+}
+
+/**
+ * @param {string|string[]|null|undefined} lhs
+ * @param {string|string[]|null|undefined} rhs
+ * @returns {boolean}
+ */
+function headerValueEquals(lhs, rhs) {
+  if (Array.isArray(lhs) && Array.isArray(rhs)) {
+    if (lhs.length !== rhs.length) {
+      return false
+    }
+
+    for (let i = 0; i < lhs.length; i++) {
+      if (rhs.includes(lhs[i])) {
+        return false
+      }
+    }
+
+    return true
+  }
+
+  return lhs === rhs
+}

package/lib/cache/util.js

@@ -0,0 +1,51 @@
+/**
+ * @param {any} key
+ */
+export function assertCacheKey(key) {
+  if (typeof key !== 'object') {
+    throw new TypeError(`expected key to be object, got ${typeof key}`)
+  }
+
+  for (const property of ['origin', 'method', 'path']) {
+    if (typeof key[property] !== 'string') {
+      throw new TypeError(`expected key.${property} to be string, got ${typeof key[property]}`)
+    }
+  }
+
+  if (key.headers !== undefined && typeof key.headers !== 'object') {
+    throw new TypeError(`expected headers to be object, got ${typeof key}`)
+  }
+}
+
+/**
+ * @param {any} value
+ */
+export function assertCacheValue(value) {
+  if (typeof value !== 'object') {
+    throw new TypeError(`expected value to be object, got ${typeof value}`)
+  }
+
+  for (const property of ['statusCode', 'cachedAt', 'staleAt', 'deleteAt']) {
+    if (typeof value[property] !== 'number') {
+      throw new TypeError(`expected value.${property} to be number, got ${typeof value[property]}`)
+    }
+  }
+
+  if (typeof value.statusMessage !== 'string') {
+    throw new TypeError(
+      `expected value.statusMessage to be string, got ${typeof value.statusMessage}`,
+    )
+  }
+
+  if (value.headers != null && typeof value.headers !== 'object') {
+    throw new TypeError(`expected value.rawHeaders to be object, got ${typeof value.headers}`)
+  }
+
+  if (value.vary !== undefined && typeof value.vary !== 'object') {
+    throw new TypeError(`expected value.vary to be object, got ${typeof value.vary}`)
+  }
+
+  if (value.etag !== undefined && typeof value.etag !== 'string') {
+    throw new TypeError(`expected value.etag to be string, got ${typeof value.etag}`)
+  }
+}

package/lib/index.js

@@ -1,6 +1,7 @@
-import undici from 'undici'
+import undici from '@nxtedition/undici'
 import { parseHeaders } from './utils.js'
 import { request as _request } from './request.js'
+import { SqliteCacheStore } from './cache/sqlite-cache-store.js'
 
 const dispatcherCache = new WeakMap()
 
@@ -18,8 +19,12 @@ export const interceptors = {
   lookup: (await import('./interceptor/lookup.js')).default,
 }
 
+export const cache = {
+  SqliteCacheStore,
+}
+
 export { parseHeaders } from './utils.js'
-export { Client, Pool, Agent, getGlobalDispatcher, setGlobalDispatcher } from 'undici'
+export { Client, Pool, Agent, getGlobalDispatcher, setGlobalDispatcher } from '@nxtedition/undici'
 
 function defaultLookup(origin, opts, callback) {
   callback(null, Array.isArray(origin) ? origin[Math.floor(Math.random() * origin.length)] : origin)
@@ -69,14 +74,12 @@ function wrapDispatch(dispatcher) {
       (dispatch) => (opts, handler) => {
         const headers = parseHeaders(opts.headers)
 
-        const userAgent = opts.userAgent ?? globalThis.userAgent
-        if (userAgent && headers?.['user-agent'] !== userAgent) {
-          headers['user-agent'] = userAgent
-        }
+        // TODO (fix): Move to interceptor?
+        headers['user-agent'] ??= opts.userAgent ?? globalThis.userAgent
 
         return dispatch(
           {
-            id: opts.id,
+            id: opts.id ?? headers['request-id'],
             origin: opts.origin,
             path: opts.path,
             method: opts.method ?? (opts.body ? 'POST' : 'GET'),