@nxtedition/lib 14.2.1 → 15.0.0

package/app.js

@@ -1,10 +1,11 @@
-const { getDockerSecretsSync } = require('./docker-secrets')
-const { getGlobalDispatcher } = require('undici')
+const os = require('node:os')
+const net = require('node:net')
+const assert = require('node:assert')
 const stream = require('node:stream')
 const { Buffer } = require('node:buffer')
-const net = require('net')
+const { getDockerSecretsSync } = require('./docker-secrets')
+const { getGlobalDispatcher } = require('undici')
 const fp = require('lodash/fp.js')
-const assert = require('node:assert')
 
 module.exports = function (appConfig, onTerminate) {
   let ds
@@ -90,14 +91,21 @@ module.exports = function (appConfig, onTerminate) {
 
   const destroyers = []
 
-  const instanceId = process.env.NODE_APP_INSTANCE || process.env.pm_id || ''
-
-  const serviceName = appConfig.name + (instanceId && instanceId !== '0' ? `-${instanceId}` : '')
+  const serviceName = appConfig.name
+  const serviceModule = appConfig.module ?? 'main'
   const serviceVersion = appConfig.version
-
-  const userAgent =
-    appConfig.userAgent ||
-    (serviceName ? `${serviceName}/${serviceVersion || '*'} Node/${process.version}` : null)
+  const serviceInstanceId =
+    // process.env.name is the pm2 name of the process
+    appConfig.instanceId ?? appConfig.containerId ?? process.env.name ?? os.hostname()
+
+  const userAgent = globalThis.userAgent = (
+    appConfig.userAgent ??
+    (serviceName &&
+      `${serviceName}/${
+        serviceVersion || '*'
+      } (module:${serviceModule}; instance:${serviceInstanceId}) Node/${process.version}`) ??
+    null
+  )
 
   const terminate = async (finalLogger) => {
     finalLogger ??= logger
@@ -137,9 +145,10 @@ module.exports = function (appConfig, onTerminate) {
       {
         ...loggerConfig,
         name: serviceName,
+        module: serviceModule,
         base: loggerConfig?.base ? { ...loggerConfig.base } : {},
       },
-      terminate
+      terminate,
     )
   }
 
@@ -166,12 +175,7 @@ module.exports = function (appConfig, onTerminate) {
   }
 
   if (appConfig.perf && process.platform === 'linux') {
-    const os = require('os')
-
-    const containerId = appConfig.containerId ?? os.hostname()
-    const hostname = process.env.NODE_ENV === 'production' ? containerId : serviceName
-
-    const perfName = typeof appConfig.perf === 'string' ? appConfig.perf : hostname
+    const perfName = typeof appConfig.perf === 'string' ? appConfig.perf : serviceInstanceId
 
     try {
       const linuxPerf = require('linux-perf')
@@ -225,7 +229,7 @@ module.exports = function (appConfig, onTerminate) {
         } else {
           return {}
         }
-      })
+      }),
     )
 
     if (couchConfig.url) {
@@ -253,7 +257,7 @@ module.exports = function (appConfig, onTerminate) {
         } else {
           return {}
         }
-      })
+      }),
     )
 
     if (!dsConfig.credentials || !dsConfig.url) {
@@ -307,7 +311,7 @@ module.exports = function (appConfig, onTerminate) {
 
         logger[level](
           { ds: { connectionState, username: userName, url: dsConfig.url } },
-          'Deepstream Connection State Changed.'
+          'Deepstream Connection State Changed.',
         )
 
         prevConnectionState = connectionState
@@ -348,7 +352,7 @@ module.exports = function (appConfig, onTerminate) {
         rx.exhaustMap(() => {
           const ret = appConfig.stats({ ds, couch, logger })
           return ret?.then || ret?.subscribe ? ret : rxjs.of(ret)
-        })
+        }),
       )
     } else if (typeof appConfig.stats === 'object') {
       stats$ = rxjs.timer(0, 10e3).pipe(rx.map(() => appConfig.stats))
@@ -366,12 +370,12 @@ module.exports = function (appConfig, onTerminate) {
       rx.retryWhen((err$) =>
         err$.pipe(
           rx.tap((err) => logger.error({ err }, 'monitor.stats')),
-          rx.delay(10e3)
-        )
+          rx.delay(10e3),
+        ),
       ),
       rx.startWith({}),
       rx.publishReplay(1),
-      rx.refCount()
+      rx.refCount(),
     )
 
     monitorProviders.stats$ = stats$
@@ -390,7 +394,7 @@ module.exports = function (appConfig, onTerminate) {
             heap: v8.getHeapStatistics(),
             ...stats,
           },
-          'STATS'
+          'STATS',
         )
         elu1 = elu2
       }
@@ -419,7 +423,7 @@ module.exports = function (appConfig, onTerminate) {
         })
         .pipe(
           rx.catchError((err) => rxjs.of({ warnings: [err.message] })),
-          rx.repeatWhen(() => rxjs.timer(10e3))
+          rx.repeatWhen(() => rxjs.timer(10e3)),
         )
     } else if (appConfig.status && typeof appConfig.status === 'object') {
       status$ = rxjs.timer(0, 10e3).pipe(rx.exhaustMap(() => appConfig.status))
@@ -445,7 +449,7 @@ module.exports = function (appConfig, onTerminate) {
             }),
             rx.startWith([]),
             rx.distinctUntilChanged(fp.isEqual),
-            rx.repeatWhen((complete$) => complete$.pipe(rx.delay(10e3)))
+            rx.repeatWhen((complete$) => complete$.pipe(rx.delay(10e3))),
           ),
           toobusy
             ? rxjs.timer(0, 1e3).pipe(
@@ -459,10 +463,10 @@ module.exports = function (appConfig, onTerminate) {
                           msg: `lag: ${toobusy.lag()}`,
                         },
                       ]
-                    : []
+                    : [],
                 ),
                 rx.startWith([]),
-                rx.distinctUntilChanged(fp.isEqual)
+                rx.distinctUntilChanged(fp.isEqual),
               )
             : rxjs.of({}),
           couch
@@ -482,7 +486,7 @@ module.exports = function (appConfig, onTerminate) {
                   }
                 }),
                 rx.startWith([]),
-                rx.distinctUntilChanged(fp.isEqual)
+                rx.distinctUntilChanged(fp.isEqual),
               )
             : rxjs.of({}),
           ds
@@ -559,7 +563,7 @@ module.exports = function (appConfig, onTerminate) {
                       }
 
                       return messages
-                    })
+                    }),
                   )
                   .subscribe(o)
 
@@ -570,7 +574,7 @@ module.exports = function (appConfig, onTerminate) {
               }).pipe(rx.startWith([]), rx.distinctUntilChanged(fp.isEqual))
             : rxjs.of({}),
           rxjs.timer(0, 10e3),
-        ].filter(Boolean)
+        ].filter(Boolean),
       )
       .pipe(
         rx.auditTime(1e3),
@@ -594,7 +598,7 @@ module.exports = function (appConfig, onTerminate) {
                     ...message,
                     message: undefined,
                     msg: message.message,
-                  }
+                  },
             )
             .map((message) =>
               message.id
@@ -602,9 +606,9 @@ module.exports = function (appConfig, onTerminate) {
                 : {
                     ...message,
                     id: hashString(
-                      [message.msg, message].find(fp.isString) ?? JSON.stringify(message)
+                      [message.msg, message].find(fp.isString) ?? JSON.stringify(message),
                     ),
-                  }
+                  },
             )
 
           return { ...status, messages, timestamp: Date.now() }
@@ -619,7 +623,7 @@ module.exports = function (appConfig, onTerminate) {
         rx.startWith({}),
         rx.distinctUntilChanged(fp.isEqual),
         rx.publishReplay(1),
-        rx.refCount()
+        rx.refCount(),
       )
 
     const loggerSubscription = status$
@@ -648,21 +652,15 @@ module.exports = function (appConfig, onTerminate) {
     const { isMainThread } = require('node:worker_threads')
 
     if (isMainThread) {
-      const os = require('os')
-      const rx = require('rxjs/operators')
-
-      const containerId = appConfig.containerId ?? os.hostname()
-      const hostname = process.env.NODE_ENV === 'production' ? containerId : serviceName
-
       const unprovide = ds.record.provide(`^([^:]+):monitor\\.([^?]+)[?]?`, (key) => {
         const [, id, prop] = key.match(/^([^:]+):monitor\.([^?]+)[?]?/)
 
         if (id === serviceName) {
           // TODO (fix): If id === serviceName check if there are multiple instances.
-          return monitorProviders[prop + '$']?.pipe(rx.map((value) => ({ [hostname]: value })))
+          return monitorProviders[prop + '$']
         }
 
-        if (id === hostname) {
+        if (id === serviceInstanceId) {
           return monitorProviders[prop + '$']
         }
       })
@@ -750,13 +748,13 @@ module.exports = function (appConfig, onTerminate) {
           },
         ]
           .flat()
-          .filter(Boolean)
+          .filter(Boolean),
       )
 
       server = createServer(
         typeof appConfig.http === 'object' ? appConfig.http : {},
         { ds, couch, config: httpConfig, logger },
-        middleware
+        middleware,
       )
 
       if (httpConfig.keepAlive != null) {
@@ -794,7 +792,9 @@ module.exports = function (appConfig, onTerminate) {
     tracer: trace,
     userAgent,
     serviceName,
+    serviceModule,
     serviceVersion,
+    serviceInstanceId,
     signal: ac.signal,
   }
 }

package/errors.js

@@ -3,8 +3,8 @@ const fp = require('lodash/fp.js')
 const { toString } = Object.prototype
 
 module.exports.AbortError = class AbortError extends Error {
-  constructor() {
-    super('The operation was aborted')
+  constructor(message) {
+    super(message ?? 'The operation was aborted')
     this.code = 'ABORT_ERR'
     this.name = 'AbortError'
   }
@@ -34,7 +34,7 @@ module.exports.parseError = function parseError(error) {
     {
       ...properties,
       cause: cause ? parseError(error.cause) : undefined,
-    }
+    },
   )
 }
 
@@ -101,7 +101,7 @@ module.exports.serializeError = function serializeError(error) {
       data,
       cause,
       errors,
-    })
+    }),
   )
 }
 
@@ -115,7 +115,7 @@ module.exports.makeMessages = function makeMessages(error, options) {
     return fp.pipe(
       fp.flattenDeep,
       fp.flatMap((x) => makeMessages(x, null)),
-      fp.uniqBy('id')
+      fp.uniqBy('id'),
     )(error)
   } else if (Array.isArray(error.messages)) {
     return makeMessages(error.messages, null)
@@ -127,11 +127,11 @@ module.exports.makeMessages = function makeMessages(error, options) {
       const level = parseInt(error.level) || options?.level || 50
       const code =
         [error?.code, options?.codes?.[error?.code]].find(
-          (x) => typeof x === 'string' && x.length > 0
+          (x) => typeof x === 'string' && x.length > 0,
         ) ?? undefined
       const msg =
         [error.msg, error.message, code?.toLowerCase().replace('_', ' ')].find(
-          (x) => typeof x === 'string' && x.length > 0
+          (x) => typeof x === 'string' && x.length > 0,
         ) || 'unknown error'
 
       let data = error.data

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "14.2.1",
+  "version": "15.0.0",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "files": [
@@ -8,6 +8,7 @@
     "ass.js",
     "rxjs/*",
     "util/*",
+    "undici/*",
     "subtract-ranges.js",
     "serializers.js",
     "elasticsearch.js",
@@ -22,9 +23,7 @@
     "app.js",
     "errors.js",
     "worker.js",
-    "proxy.js",
     "stream.js",
-    "undici.js",
     "timeline.js",
     "docker-secrets.js"
   ],
@@ -71,6 +70,7 @@
     "/__tests__"
   ],
   "dependencies": {
+    "cache-control-parser": "^2.0.4",
     "date-fns": "^2.29.3",
     "fast-querystring": "^1.1.1",
     "hasha": "^5.2.2",
@@ -78,6 +78,7 @@
     "json5": "^2.2.3",
     "koa-compose": "^4.1.0",
     "lodash": "^4.17.21",
+    "lru-cache": "^10.0.1",
     "mime": "^3.0.0",
     "moment-timezone": "^0.5.43",
     "nconf": "^0.12.0",
@@ -90,23 +91,25 @@
     "smpte-timecode": "^1.3.3",
     "split-string": "^6.0.0",
     "toobusy-js": "^0.5.1",
-    "undici": "^5.22.0",
+    "undici": "^5.25.2",
     "url-join": "^4.0.0",
     "xuid": "^4.1.2"
   },
   "devDependencies": {
-    "eslint": "^8.38.0",
-    "eslint-config-prettier": "^8.8.0",
+    "@types/node": "^20.6.3",
+    "eslint": "^8.50.0",
+    "eslint-config-prettier": "^9.0.0",
     "eslint-config-standard": "^17.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-import": "^2.28.1",
+    "eslint-plugin-n": "^16.1.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^6.0.0",
     "husky": "^8.0.3",
-    "lint-staged": "^13.2.1",
+    "lint-staged": "^14.0.1",
     "pinst": "^3.0.0",
-    "prettier": "^2.8.7",
+    "prettier": "^3.0.3",
     "rxjs": "^7.5.6",
+    "send": "^0.18.0",
     "tap": "^16.3.4"
   },
   "peerDependencies": {

package/undici/index.js

@@ -0,0 +1,192 @@
+const assert = require('assert')
+const createError = require('http-errors')
+const xuid = require('xuid')
+const undici = require('undici')
+const stream = require('stream')
+const { parseHeaders } = require('../http')
+
+class Readable extends stream.Readable {
+  constructor({ statusCode, statusMessage, headers, ...opts }) {
+    super(opts)
+    this.statusCode = statusCode
+    this.statusMessage = statusMessage
+    this.headers = headers
+    this.body = this
+  }
+
+  async text() {
+    const dec = new TextDecoder()
+    let str = ''
+    for await (const chunk of this) {
+      if (typeof chunk === 'string') {
+        str += chunk
+      } else {
+        str += dec.decode(chunk, { stream: true })
+      }
+    }
+    // Flush the streaming TextDecoder so that any pending
+    // incomplete multibyte characters are handled.
+    str += dec.decode(undefined, { stream: false })
+    return str
+  }
+
+  async json() {
+    return JSON.parse(await this.text())
+  }
+
+  async arrayBuffer() {
+    const buffers = []
+    for await (const chunk of this) {
+      buffers.push(chunk)
+    }
+    return Buffer.concat(buffers)
+  }
+
+  async buffer() {
+    return Buffer.from(await this.arrayBuffer())
+  }
+
+  async dump() {
+    let n = 0
+    try {
+      for await (const chunk of this) {
+        // do nothing
+        n += chunk.length
+        if (n > 128 * 1024) {
+          break
+        }
+      }
+    } catch {
+      this.destroy()
+    }
+  }
+}
+
+const dispatchers = {
+  abort: require('./interceptor/abort.js'),
+  catch: require('./interceptor/catch.js'),
+  content: require('./interceptor/content.js'),
+  responseBodyDump: require('./interceptor/response-body-dump.js'),
+  log: require('./interceptor/log.js'),
+  redirect: require('./interceptor/redirect.js'),
+  responseBodyRetry: require('./interceptor/response-body-retry.js'),
+  responseStatusRetry: require('./interceptor/response-status-retry.js'),
+  responseRetry: require('./interceptor/response-retry.js'),
+  signal: require('./interceptor/signal.js'),
+  proxy: require('./interceptor/proxy.js'),
+}
+
+async function request(urlOrOpts, opts = {}) {
+  let url
+  if (typeof urlOrOpts === 'string') {
+    url = new URL(urlOrOpts)
+  } else if (urlOrOpts instanceof URL) {
+    url = urlOrOpts
+  } else if (typeof urlOrOpts?.origin === 'string' && typeof urlOrOpts?.path === 'string') {
+    url = urlOrOpts
+  } else if (typeof urlOrOpts === 'object' && urlOrOpts != null) {
+    opts = urlOrOpts
+    url = opts.url
+  }
+
+  const method = opts.method ?? (opts.body ? 'POST' : 'GET')
+  const idempotent = opts.idempotent ?? (method === 'GET' || method === 'HEAD')
+  const dump = opts.dump ?? method === 'HEAD'
+
+  let headers
+  if (Array.isArray(opts.headers)) {
+    headers = parseHeaders(opts.headers)
+  } else {
+    headers = opts.headers
+  }
+
+  opts = {
+    url,
+    method,
+    body: opts.body,
+    headers: {
+      'request-id': xuid(),
+      'user-agent': opts.userAgent ?? globalThis.userAgent,
+      ...headers,
+    },
+    origin: opts.origin ?? url.origin,
+    path: opts.path ?? url.search ? `${url.pathname}${url.search ?? ''}` : url.pathname,
+    reset: opts.reset ?? false,
+    headersTimeout: opts.headersTimeout,
+    bodyTimeout: opts.bodyTimeout,
+    idempotent,
+    signal: opts.signal,
+    retry: opts.retry ?? 8,
+    follow: { count: opts.maxRedirections ?? 8, ...opts.redirect, ...opts.follow },
+    dump,
+    logger: opts.logger,
+  }
+
+  const dispatcher = opts.dispatcher ?? undici.getGlobalDispatcher()
+
+  return new Promise((resolve) => {
+    let dispatch = (opts, handler) => dispatcher.dispatch(opts, handler)
+
+    dispatch = dispatchers.catch(dispatch)
+    dispatch = dispatchers.abort(dispatch)
+    dispatch = dispatchers.log(dispatch)
+    dispatch = dispatchers.responseRetry(dispatch)
+    dispatch = dispatchers.responseStatusRetry(dispatch)
+    dispatch = dispatchers.responseBodyRetry(dispatch)
+    dispatch = dispatchers.content(dispatch)
+    dispatch = dispatchers.responseBodyDump(dispatch)
+    dispatch = dispatchers.redirect(dispatch)
+    dispatch = dispatchers.signal(dispatch)
+    dispatch = dispatchers.proxy(dispatch)
+
+    dispatch(opts, {
+      resolve,
+      /** @type {Function | null} */ abort: null,
+      /** @type {stream.Readable | null} */ body: null,
+      onConnect(abort) {
+        this.abort = abort
+      },
+      onHeaders(statusCode, rawHeaders, resume, statusMessage) {
+        assert(this.abort)
+
+        const headers = parseHeaders(rawHeaders)
+
+        if (statusCode >= 400) {
+          this.abort(createError(statusCode, { headers }))
+        } else {
+          assert(statusCode >= 200)
+
+          this.body = new Readable({
+            read: resume,
+            highWaterMark: 128 * 1024,
+            statusCode,
+            statusMessage,
+            headers,
+          })
+
+          this.resolve(this.body)
+          this.resolve = null
+        }
+
+        return false
+      },
+      onData(chunk) {
+        assert(this.body)
+        return this.body.push(chunk)
+      },
+      onComplete() {
+        assert(this.body)
+        this.body.push(null)
+      },
+      onError(err) {
+        if (this.body) {
+          this.body.destroy(err)
+        } else {
+          this.resolve(Promise.reject(err))
+        }
+      },
+    })
+  })
+}
+
+module.exports = { request }

package/undici/interceptor/abort.js

@@ -0,0 +1,61 @@
+const { AbortError } = require('../../errors')
+
+class Handler {
+  constructor(opts, { handler }) {
+    this.handler = handler
+    this.pos = 0
+    this.reason = null
+  }
+
+  onConnect(abort) {
+    this.abort = abort
+    this.handler.onConnect((reason) => {
+      this.reason = reason ?? new AbortError()
+    })
+  }
+
+  onBodySent(chunk) {
+    return this.handler.onBodySent(chunk)
+  }
+
+  onHeaders(statusCode, rawHeaders, resume, statusMessage) {
+    if (this.reason == null) {
+      const ret = this.handler.onHeaders(statusCode, rawHeaders, resume, statusMessage)
+      if (this.reason == null) {
+        return ret
+      }
+    }
+
+    return true
+  }
+
+  onData(chunk) {
+    if (this.reason == null) {
+      const ret = this.handler.onData(chunk)
+      if (this.reason == null) {
+        return ret
+      }
+    }
+
+    this.pos += chunk.length
+    if (this.pos < 128 * 1024) {
+      return true
+    }
+
+    this.abort(this.reason)
+
+    return false
+  }
+
+  onComplete(rawTrailers) {
+    return this.reason == null
+      ? this.handler.onComplete(rawTrailers)
+      : this.handler.onError(this.reason)
+  }
+
+  onError(err) {
+    return this.handler.onError(err)
+  }
+}
+
+module.exports = (dispatch) => (opts, handler) => dispatch(opts, new Handler(opts, { handler }))

package/undici/interceptor/catch.js

@@ -0,0 +1,55 @@
+class Handler {
+  constructor(opts, { handler }) {
+    this.handler = handler
+  }
+
+  onConnect(abort) {
+    this.abort = abort
+    try {
+      return this.handler.onConnect(abort)
+    } catch (err) {
+      this.abort(err)
+    }
+  }
+
+  onBodySent(chunk) {
+    try {
+      return this.handler.onBodySent(chunk)
+    } catch (err) {
+      this.abort(err)
+    }
+  }
+
+  onHeaders(statusCode, rawHeaders, resume, statusMessage) {
+    try {
+      return this.handler.onHeaders(statusCode, rawHeaders, resume, statusMessage)
+    } catch (err) {
+      this.abort(err)
+      return false
+    }
+  }
+
+  onData(chunk) {
+    try {
+      return this.handler.onData(chunk)
+    } catch (err) {
+      this.abort(err)
+      return false
+    }
+  }
+
+  onComplete(rawTrailers) {
+    try {
+      return this.handler.onComplete(rawTrailers)
+    } catch (err) {
+      this.abort(err)
+      return false
+    }
+  }
+
+  onError(err) {
+    return this.handler.onError(err)
+  }
+}
+
+module.exports = (dispatch) => (opts, handler) => dispatch(opts, new Handler(opts, { handler }))