@nxtedition/lib 28.0.20 → 28.0.22

package/app.js

@@ -623,24 +623,42 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
     let stats$
 
     if (typeof appConfig.stats.subscribe === 'function') {
-      stats$ = appConfig.stats
+      stats$ = appConfig.stats.pipe(rxjs.auditTime(1e3))
     } else if (typeof appConfig.stats === 'function') {
-      stats$ = rxjs.timer(0, 10e3).pipe(
+      stats$ = rxjs.timer(0, 1e3).pipe(
         rxjs.exhaustMap(() => {
           const ret = appConfig.stats({ ds, couch, logger })
           return ret?.then || ret?.subscribe ? ret : rxjs.of(ret)
         }),
       )
     } else if (typeof appConfig.stats === 'object') {
-      stats$ = rxjs.timer(0, 10e3).pipe(rxjs.map(() => appConfig.stats))
+      stats$ = rxjs.timer(0, 1e3).pipe(rxjs.map(() => appConfig.stats))
     } else {
-      stats$ = rxjs.timer(0, 10e3).pipe(rxjs.map(() => ({})))
+      stats$ = rxjs.timer(0, 1e3).pipe(rxjs.map(() => ({})))
     }
 
     let statsMap
 
     const startTime = Date.now()
 
+    const makeStatsPipe = (key) =>
+      rxjs.pipe(
+        rxjs.catchError((err) => {
+          logger.error({ err, key }, 'monitor.stats')
+          return rxjs.of([
+            {
+              id: `app:user_monitor_stats_${key}`,
+              level: 50,
+              code: err.code,
+              msg: err.message,
+            },
+          ])
+        }),
+        rxjs.startWith({}),
+        rxjs.distinctUntilChanged(fp.isEqual),
+        rxjs.repeatWhen((complete$) => complete$.pipe(rxjs.delay(10e3))),
+      )
+
     appDestroyers.unshift(
       rxjs
         .timer(0, 1e3)
@@ -711,18 +729,14 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
               undici,
             }
           }),
-          rxjs.withLatestFrom(stats$.pipe(rxjs.startWith({}))),
+          makeStatsPipe('app'),
+          rxjs.withLatestFrom(stats$.pipe(makeStatsPipe('user'))),
           rxjs.map(([appStats, serviceStats]) => ({
             ...appStats,
             // TODO (fix): [serviceName] is not great and can collide.
             [serviceName]: serviceStats,
           })),
-          rxjs.retry({
-            delay(err, retryCount) {
-              logger.error({ err, retryCount }, 'monitor.stats$ failed')
-              return rxjs.timer(10e3)
-            },
-          }),
+          makeStatsPipe('stats'),
         )
         .subscribe(monitorProviders.stats$),
     )
@@ -761,231 +775,250 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
 
   if (appConfig.status) {
     let status$
-    if (appConfig.status.subscribe) {
-      status$ = appConfig.status
+    if (typeof appConfig.status.subscribe === 'function') {
+      status$ = appConfig.status.pipe(rxjs.auditTime(1e3))
     } else if (typeof appConfig.status === 'function') {
-      status$ = rxjs
-        .defer(() => {
-          const ret = appConfig.status({ ds, couch, logger })
-          return ret?.then || ret?.subscribe ? ret : rxjs.of(ret)
-        })
-        .pipe(
-          rxjs.catchError((err) => rxjs.of({ warnings: [err.message] })),
-          rxjs.repeatWhen(() => rxjs.timer(10e3)),
-        )
+      status$ = rxjs.defer(() => {
+        const ret = appConfig.status({ ds, couch, logger })
+        return ret?.then || ret?.subscribe ? ret : rxjs.of(ret)
+      })
     } else if (appConfig.status && typeof appConfig.status === 'object') {
-      status$ = rxjs.timer(0, 10e3).pipe(rxjs.exhaustMap(() => appConfig.status))
+      status$ = rxjs.timer(0, 1e3).pipe(rxjs.exhaustMap(() => appConfig.status))
     } else {
       status$ = rxjs.of({})
     }
 
+    let statusMap
+
+    const makeStatusPipe = (key) =>
+      rxjs.pipe(
+        rxjs.catchError((err) => {
+          logger.error({ err, key }, 'monitor.status')
+          return rxjs.of([
+            {
+              id: `app:user_monitor_status_${key}`,
+              level: 50,
+              code: err.code,
+              msg: err.message,
+            },
+          ])
+        }),
+        rxjs.startWith([]),
+        rxjs.distinctUntilChanged(fp.isEqual),
+        rxjs.repeatWhen((complete$) => complete$.pipe(rxjs.delay(10e3))),
+      )
+
     appDestroyers.unshift(
       rxjs
-        .combineLatest(
-          [
-            status$.pipe(
-              rxjs.filter(Boolean),
-              rxjs.map((xs) => (Array.isArray(xs) ? { messages: xs } : xs)),
-              rxjs.catchError((err) => {
-                logger.error({ err }, 'monitor.status')
-                return rxjs.of([
-                  {
-                    id: 'app:user_monitor_status',
-                    level: 50,
-                    code: err.code,
-                    msg: err.message,
-                  },
-                ])
-              }),
-              rxjs.startWith([]),
-              rxjs.distinctUntilChanged(fp.isEqual),
-              rxjs.repeatWhen((complete$) => complete$.pipe(rxjs.delay(10e3))),
-            ),
-            monitorProviders.stats$?.pipe(
-              rxjs.map(({ memory, heap, utilization, undici, http }) => {
-                const messages = []
-
-                if (memory?.containerLimit) {
-                  const usagePercent = (memory.containerUsage / memory.containerLimit) * 100
-                  messages.push({
-                    id: 'app:container_memory_usage',
-                    level: usagePercent > 90 ? 50 : usagePercent > 70 ? 40 : 30,
-                    msg: `Memory Usage: ${usagePercent.toFixed(2)}%`,
-                  })
+        .combineLatest([
+          rxjs.timer(0, 1e3).pipe(
+            rxjs.map(() => {
+              const messages = []
+              if (statusMap) {
+                for (const status of statusMap.values()) {
+                  if (Array.isArray(status?.messages)) {
+                    messages.push(...status.messages)
+                  } else if (Array.isArray(status)) {
+                    messages.push(...status)
+                  }
                 }
+              }
+              return { messages }
+            }),
+          ),
+          status$.pipe(
+            rxjs.filter(Boolean),
+            rxjs.map((xs) => (Array.isArray(xs) ? { messages: xs } : xs)),
+            makeStatusPipe('stats'),
+          ),
+          monitorProviders.stats$?.pipe(
+            rxjs.map(({ memory, heap, utilization, undici, http }) => {
+              const messages = []
+
+              if (memory?.containerLimit) {
+                const usagePercent = (memory.containerUsage / memory.containerLimit) * 100
+                messages.push({
+                  id: 'app:container_memory_usage',
+                  level: usagePercent > 90 ? 50 : usagePercent > 70 ? 40 : 30,
+                  msg: `Memory Usage: ${usagePercent.toFixed(2)}% (${(memory.containerUsage / 1e9).toFixed(2)} GiB / ${(memory.containerLimit / 1e9).toFixed(2)} GiB)`,
+                })
+              }
 
-                if (heap) {
-                  const usagePercent = (heap.used_heap_size / heap.heap_size_limit) * 100
-                  messages.push({
-                    id: 'app:heap_memory_usage',
-                    level: usagePercent > 90 ? 50 : usagePercent > 70 ? 40 : 30,
-                    msg: `Heap Usage: ${usagePercent.toFixed(2)}%`,
-                  })
-                }
+              if (heap) {
+                const usagePercent = (heap.used_heap_size / heap.heap_size_limit) * 100
+                messages.push({
+                  id: 'app:heap_memory_usage',
+                  level: usagePercent > 90 ? 50 : usagePercent > 70 ? 40 : 30,
+                  msg: `Heap Usage: ${usagePercent.toFixed(2)}%`,
+                })
+              }
 
-                if (utilization) {
-                  const elp = utilization.utilization * 100
-                  messages.push({
-                    id: 'app:event_loop_utilization',
-                    level: elp > 95 ? 50 : elp > 80 ? 40 : 30,
-                    msg: `Event Loop Utilization: ${elp.toFixed(2)}%`,
-                  })
-                }
+              if (utilization) {
+                const elp = utilization.utilization * 100
+                messages.push({
+                  id: 'app:event_loop_utilization',
+                  level: elp > 95 ? 50 : elp > 80 ? 40 : 30,
+                  msg: `Event Loop Utilization: ${elp.toFixed(2)}%`,
+                })
+              }
 
-                if (undici) {
-                  messages.push({
-                    id: 'app:undici_upstream_sockets',
-                    level: undici.totalSockets > 8192 ? 50 : undici.totalSockets > 4096 ? 40 : 30,
-                    msg: `Undici: ${undici.totalSockets} upstream connected`,
-                  })
-                }
+              if (undici?.totalSockets) {
+                messages.push({
+                  id: 'app:undici_upstream_sockets',
+                  level: undici.totalSockets > 8192 ? 50 : undici.totalSockets > 4096 ? 40 : 30,
+                  msg: `Undici: ${undici.totalSockets} upstream connected`,
+                })
+              }
 
-                if (http) {
-                  messages.push({
-                    id: 'app:http_pending_requests',
-                    level: http.totalPending > 8192 ? 50 : http.totalPending > 4096 ? 40 : 30,
-                    msg: `HTTP: ${http.totalPending} pending requests`,
-                  })
-                }
+              if (http?.totalPending) {
+                messages.push({
+                  id: 'app:http_pending_requests',
+                  level: http.totalPending > 8192 ? 50 : http.totalPending > 4096 ? 40 : 30,
+                  msg: `http: ${http.totalPending} pending requests`,
+                })
+              }
 
-                return messages
-              }),
+              return messages
+            }),
+            makeStatusPipe('stats'),
+          ),
+          toobusy?.appLag$.pipe(
+            rxjs.map((lag) =>
+              lag == null
+                ? []
+                : [
+                    {
+                      id: 'app:toobusy_lag',
+                      level: lag > 1e3 ? 50 : lag > toobusy.maxLag ? 40 : 30,
+                      code: 'NXT_LAG',
+                      msg: `Lag: ${lag.toFixed(2)} ms`,
+                    },
+                  ],
+            ),
+            makeStatusPipe('toobusy'),
+          ) ?? rxjs.of([]),
+          underPressure?.pressure$?.pipe(
+            rxjs.map((pressure) =>
+              pressure == null
+                ? []
+                : [
+                    {
+                      id: 'app:under_pressure',
+                      level: 40,
+                      code: 'NXT_PRESSURE',
+                      msg: `Under Pressure`,
+                    },
+                  ],
             ),
-            toobusy?.appLag$.pipe(
-              rxjs.map((lag) =>
-                lag == null
-                  ? []
-                  : [
+            makeStatusPipe('underpressure'),
+          ) ?? rxjs.of([]),
+          couch
+            ? rxjs.timer(0, 10e3).pipe(
+                rxjs.exhaustMap(async () => {
+                  try {
+                    await couch.up()
+                    return [
                       {
-                        id: 'app:toobusy_lag',
-                        level: lag > 1e3 ? 50 : lag > toobusy.maxLag ? 40 : 30,
-                        code: 'NXT_LAG',
-                        msg: `Lag: ${lag.toFixed(2)} ms`,
+                        id: 'app:couch',
+                        level: 30,
+                        msg: 'Couch: connected',
                       },
-                    ],
-              ),
-            ) ?? rxjs.of([]),
-            underPressure?.pressure$?.pipe(
-              rxjs.map((pressure) =>
-                pressure == null
-                  ? []
-                  : [
+                    ]
+                  } catch (err) {
+                    return [
                       {
-                        id: 'app:under_pressure',
+                        id: 'app:couch',
                         level: 40,
-                        code: 'NXT_PRESSURE',
-                        msg: `Under Pressure`,
+                        code: err.code,
+                        msg: 'Couch: ' + err.message,
                       },
-                    ],
-              ),
-            ) ?? rxjs.of([]),
-            couch
-              ? rxjs.timer(0, 10e3).pipe(
-                  rxjs.exhaustMap(async () => {
-                    try {
-                      await couch.up()
-                      return [
+                    ]
+                  }
+                }),
+                makeStatusPipe('couch'),
+              )
+            : rxjs.of([]),
+          ds
+            ? rxjs.fromEvent(ds, 'connectionStateChanged').pipe(
+                rxjs.map((connectionState) =>
+                  connectionState === 'OPEN'
+                    ? [
                         {
-                          id: 'app:couch',
+                          id: 'app:ds_connection_state',
                           level: 30,
-                          msg: 'Couch: connected',
+                          msg: 'Deepstream: connected',
+                          data: { connectionState },
                         },
                       ]
-                    } catch (err) {
-                      return [
+                    : [
                         {
-                          id: 'app:couch',
+                          id: 'app:ds_connection_state',
                           level: 40,
-                          code: err.code,
-                          msg: 'Couch: ' + err.message,
+                          msg: 'Deepstream: connecting',
+                          data: { connectionState },
                         },
-                      ]
-                    }
-                  }),
-                  rxjs.startWith([]),
-                  rxjs.distinctUntilChanged(fp.isEqual),
-                )
-              : rxjs.of({}),
-            ds
-              ? rxjs.fromEvent(ds, 'connectionStateChanged').pipe(
-                  rxjs.map((connectionState) =>
-                    connectionState === 'OPEN'
-                      ? [
-                          {
-                            id: 'app:ds_connection_state',
-                            level: 30,
-                            msg: 'Deepstream: connected',
-                            data: { connectionState },
-                          },
-                        ]
-                      : [
-                          {
-                            id: 'app:ds_connection_state',
-                            level: 40,
-                            msg: 'Deepstream: connecting',
-                            data: { connectionState },
-                          },
-                        ],
-                  ),
-                )
-              : rxjs.of([]),
-            ds
-              ? rxjs.timer(0, 10e3).pipe(
-                  rxjs.exhaustMap(async () => {
-                    const messages = []
-
-                    if (ds.stats.record.records > 100e3) {
-                      messages.push({
-                        id: 'app:ds_record_records',
-                        level: 40,
-                        code: 'NXT_DEEPSTREAM_RECORDS_RECORDS',
-                        msg: 'Deepstream: ' + ds.stats.record.records + ' records',
-                      })
-                    }
-
-                    if (ds.stats.record.pruning > 100e3) {
-                      messages.push({
-                        id: 'app:ds_record_pruning',
-                        level: 40,
-                        code: 'NXT_DEEPSTREAM_RECORDS_PRUNING',
-                        msg: 'Deepstream: ' + ds.stats.record.pruning + ' pruning',
-                      })
-                    }
-
-                    if (ds.stats.record.pending > 10e3) {
-                      messages.push({
-                        id: 'app:ds_record_pending',
-                        level: 40,
-                        code: 'NXT_DEEPSTREAM_RECORDS_PENDING',
-                        msg: 'Deepstream: ' + ds.stats.record.pending + ' pending',
-                      })
-                    }
-
-                    if (ds.stats.record.updating > 10e3) {
-                      messages.push({
-                        id: 'app:ds_record_updating',
-                        level: 40,
-                        code: 'NXT_DEEPSTREAM_RECORDS_UPDATING',
-                        msg: 'Deepstream: ' + ds.stats.record.updating + ' updating',
-                      })
-                    }
-
-                    if (ds.stats.record.patching > 10e3) {
-                      messages.push({
-                        id: 'app:ds_record_patching',
-                        level: 40,
-                        code: 'NXT_DEEPSTREAM_RECORDS_PATCHING',
-                        msg: 'Deepstream: ' + ds.stats.record.patching + ' patching',
-                      })
-                    }
-
-                    return messages
-                  }),
-                )
-              : rxjs.of([]),
-            rxjs.timer(0, 10e3),
-          ].filter(Boolean),
-        )
+                      ],
+                ),
+                makeStatusPipe('ds_connection'),
+              )
+            : rxjs.of([]),
+          ds
+            ? rxjs.timer(0, 10e3).pipe(
+                rxjs.map(() => {
+                  const messages = []
+
+                  if (ds.stats.record.records) {
+                    messages.push({
+                      id: 'app:ds_record_records',
+                      level: ds.stats.record.records > 100e3 ? 40 : 30,
+                      code: 'NXT_DEEPSTREAM_RECORDS_RECORDS',
+                      msg: 'Deepstream: ' + ds.stats.record.records + ' records',
+                    })
+                  }
+
+                  if (ds.stats.record.pruning) {
+                    messages.push({
+                      id: 'app:ds_record_pruning',
+                      level: ds.stats.record.pruning > 100e3 ? 40 : 30,
+                      code: 'NXT_DEEPSTREAM_RECORDS_PRUNING',
+                      msg: 'Deepstream: ' + ds.stats.record.pruning + ' pruning',
+                    })
+                  }
+
+                  if (ds.stats.record.pending) {
+                    messages.push({
+                      id: 'app:ds_record_pending',
+                      level: ds.stats.record.pending > 10e3 ? 40 : 30,
+                      code: 'NXT_DEEPSTREAM_RECORDS_PENDING',
+                      msg: 'Deepstream: ' + ds.stats.record.pending + ' pending',
+                    })
+                  }
+
+                  if (ds.stats.record.updating) {
+                    messages.push({
+                      id: 'app:ds_record_updating',
+                      level: ds.stats.record.updating > 10e3 ? 40 : 30,
+                      code: 'NXT_DEEPSTREAM_RECORDS_UPDATING',
+                      msg: 'Deepstream: ' + ds.stats.record.updating + ' updating',
+                    })
+                  }
+
+                  if (ds.stats.record.patching) {
+                    messages.push({
+                      id: 'app:ds_record_patching',
+                      level: ds.stats.record.patching ? 40 : 30,
+                      code: 'NXT_DEEPSTREAM_RECORDS_PATCHING',
+                      msg: 'Deepstream: ' + ds.stats.record.patching + ' patching',
+                    })
+                  }
+
+                  return messages
+                }),
+                makeStatusPipe('ds'),
+              )
+            : rxjs.of([]),
+          rxjs.timer(0, 10e3),
+        ])
         .pipe(
           rxjs.auditTime(1e3),
           rxjs.map(([status, lag, couch, ds]) => {
@@ -1023,22 +1056,37 @@ export function makeApp(appConfig, onTerminateOrMeta, metaOrNull) {
 
             return { ...status, messages, timestamp: Date.now() }
           }),
-          rxjs.catchError((err) => {
-            logger.error({ err }, 'monitor.status')
-            return rxjs.of({
-              messages: [{ id: 'app:monitor_status', level: 50, code: err.code, msg: err.message }],
-            })
-          }),
-          rxjs.repeatWhen((complete$) => complete$.pipe(rxjs.delay(10e3))),
-          rxjs.startWith({}),
-          rxjs.distinctUntilChanged(fp.isEqual),
+          makeStatusPipe('status'),
         )
         .subscribe(monitorProviders.status$),
     )
 
+    const statussBC = new BroadcastChannel('nxt:app:stats').unref()
+
+    if (isMainThread) {
+      statusMap = new Map()
+      statussBC.onmessage = ({ data: { data, id } }) => {
+        if (data != null) {
+          statusMap.set(id, data)
+        } else {
+          statusMap.delete(id)
+        }
+      }
+    }
+
+    appDestroyers.unshift(
+      monitorProviders.status$
+        .subscribe((stats) => {
+          statussBC.postMessage({ id: threadId, data: stats })
+        })
+        .add(() => {
+          statussBC.postMessage({ id: threadId, data: undefined })
+        }),
+    )
+
     appDestroyers.unshift(
       monitorProviders.status$
-        .pipe(rxjs.auditTime(1e3), rxjs.pluck('messages'), rxjs.startWith([]), rxjs.pairwise())
+        .pipe(rxjs.pluck('messages'), rxjs.startWith([]), rxjs.pairwise())
         .subscribe(([prev, next]) => {
           for (const { level, msg: status, ...message } of fp.differenceBy('id', next, prev)) {
             if (level >= 50) {

package/cache.js

@@ -1,6 +1,7 @@
 import { DatabaseSync } from 'node:sqlite'
 import { LRUCache } from 'lru-cache'
 import { fastNow } from './time.js'
+import { doYield } from '@nxtedition/yield'
 
 function noop() {}
 
@@ -50,6 +51,8 @@ export class AsyncCache {
   #delQuery
   #purgeStaleQuery
 
+  #setQueue = []
+
   /**
    * @param {string} location
    * @param {((...args: any[]) => Promise<V>)|undefined} [valueSelector]
@@ -304,10 +307,23 @@ export class AsyncCache {
 
     this.#lru?.set(key, { ttl, stale, value })
 
-    try {
-      this.#setQuery?.run(key, JSON.stringify(value), ttl, stale)
-    } catch {
-      // Do nothing...
+    this.#setQueue.push({ key, value, ttl, stale })
+    if (this.#setQueue.length === 1) {
+      doYield(this.#flushSetQueue)
+    }
+  }
+
+  #flushSetQueue = () => {
+    for (const { key, value, ttl, stale } of this.#setQueue.splice(0, 64)) {
+      try {
+        this.#setQuery?.run(key, JSON.stringify(value), ttl, stale)
+      } catch {
+        // Do nothing...
+      }
+    }
+
+    if (this.#setQueue.length > 0) {
+      doYield(this.#flushSetQueue)
     }
   }
 

package/memory.js

@@ -1,5 +1,7 @@
 import { readFileSync } from 'node:fs'
 
+const INACTIVE_FILE_REGEX = /^inactive_file\s+(\d+)/m
+
 function readFile(path) {
   try {
     return readFileSync(path, 'utf8').trim()
@@ -30,13 +32,19 @@ export function getContainerMemoryUsage() {
   // cgroups v2
   const v2Usage = readFile('/sys/fs/cgroup/memory.current')
   if (v2Usage) {
-    return Number(v2Usage)
+    const usage = Number(v2Usage)
+    const stat = readFile('/sys/fs/cgroup/memory.stat')
+    const inactiveFile = Number(stat?.match(INACTIVE_FILE_REGEX)?.[1] ?? 0)
+    return Math.max(0, usage - inactiveFile)
   }
 
   // cgroups v1
   const v1Usage = readFile('/sys/fs/cgroup/memory/memory.usage_in_bytes')
   if (v1Usage) {
-    return Number(v1Usage)
+    const usage = Number(v1Usage)
+    const stat = readFile('/sys/fs/cgroup/memory/memory.stat')
+    const inactiveFile = Number(stat?.match(INACTIVE_FILE_REGEX)?.[1] ?? 0)
+    return Math.max(0, usage - inactiveFile)
   }
 
   return undefined

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "28.0.20",
+  "version": "28.0.22",
   "license": "UNLICENSED",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "type": "module",
@@ -54,6 +54,7 @@
     "@nxtedition/sched": "^1.0.2",
     "@nxtedition/template": "^1.0.10",
     "@nxtedition/weak-cache": "^1.0.2",
+    "@nxtedition/yield": "^1.0.2",
     "diff": "5.2.0",
     "fast-querystring": "^1.1.2",
     "flamegraph-middleware": "^1.0.0",
@@ -92,5 +93,5 @@
     "pino": ">=7.0.0",
     "rxjs": "^7.0.0"
   },
-  "gitHead": "b33dfcfc5f68626f846f364805f51a0e6de74cce"
+  "gitHead": "f323a39ce1fe315c68d4d9a277fd63b3416b4d7b"
 }