@data-fair/lib-node 2.3.2 → 2.5.0

package/observer.js

@@ -36,8 +36,7 @@ const serveRegistry = async (res, registry) => {
   res.end()
 }
 // live CPU performance inspection
-const serveCPUProfile = async (res, duration = 2000) => {
-  // @ts-ignore
+const serveCPUProfile = async (res, duration) => {
   const { Session } = await import('node:inspector/promises')
   const session = new Session()
   session.connect()
@@ -45,21 +44,44 @@ const serveCPUProfile = async (res, duration = 2000) => {
   await session.post('Profiler.start')
   await new Promise(resolve => setTimeout(resolve, duration))
   const { profile } = await session.post('Profiler.stop')
+  await session.post('Profiler.disable')
   session.disconnect()
   res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.cpuprofile"`)
   res.write(JSON.stringify(profile))
   res.end()
 }
+// live heap allocation sampler
+const serveHeapSnapshot = async (res, includeCollected, duration) => {
+  const { Session } = await import('node:inspector/promises')
+  const session = new Session()
+  session.connect()
+  await session.post('HeapProfiler.enable')
+  await session.post('HeapProfiler.startSampling', { includeObjectsCollectedByMajorGC: includeCollected, includeObjectsCollectedByMinorGC: true })
+  await new Promise(resolve => setTimeout(resolve, duration))
+  const { profile } = await session.post('HeapProfiler.stopSampling')
+  await session.post('HeapProfiler.disable')
+  session.disconnect()
+  res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.heapprofile"`)
+  res.write(JSON.stringify(profile))
+  res.end()
+}
 let server
 export const startObserver = async (port = 9090) => {
   server = createServer((req, res) => {
-    if (req.method === 'GET' && req.url === '/metrics') {
+    const url = new URL(req.url || '', `http://${req.headers.host}`)
+    if (req.method === 'GET' && url.pathname === '/metrics') {
       serveRegistry(res, register).catch(err => httpErrorHandler(res, err))
-    } else if (req.method === 'GET' && (req.url === '/global-metrics' || req.url === '/service-metrics')) {
+    } else if (req.method === 'GET' && (url.pathname === '/global-metrics' || url.pathname === '/service-metrics')) {
       serveRegistry(res, servicePromRegistry).catch(err => httpErrorHandler(res, err))
-    } else if (req.method === 'GET' && req.url === '/cpu-profile') {
-      // TODO: duration
-      serveCPUProfile(res).catch(err => httpErrorHandler(res, err))
+    } else if (req.method === 'GET' && url.pathname === '/cpu-profile') {
+      const duration = parseInt(url.searchParams.get('duration') || '2000')
+      serveCPUProfile(res, duration).catch(err => httpErrorHandler(res, err))
+    } else if (req.method === 'GET' && url.pathname === '/heap-profile') {
+      const duration = parseInt(url.searchParams.get('duration') || '2000')
+      serveHeapSnapshot(res, false, duration).catch(err => httpErrorHandler(res, err))
+    } else if (req.method === 'GET' && url.pathname === '/heap-profile-all') {
+      const duration = parseInt(url.searchParams.get('duration') || '2000')
+      serveHeapSnapshot(res, true, duration).catch(err => httpErrorHandler(res, err))
     } else {
       res.writeHead(404)
       res.end()
@@ -71,6 +93,8 @@ export const startObserver = async (port = 9090) => {
     GET /metrics -> get prometheus metrics for this instance
     GET /service-metrics -> get prometheus metrics shared accross all instances
     GET /cpu-profile -> generate and fetch a CPU profile
+    GET /heap-snapshot -> generate and fetch a sampled heap allocation snapshot
+    GET /heap-snapshot-all -> generate and fetch a sampled heap allocation snapshot including objects collected by GC
 `)
 }
 export const stopObserver = async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/lib-node",
-  "version": "2.3.2",
+  "version": "2.5.0",
   "description": "Small nodejs related utility functions for the data-fair stack.",
   "main": "index.js",
   "files": [
@@ -15,10 +15,9 @@
   },
   "dependencies": {
     "@data-fair/lib-common-types": "^1.5.0",
+    "@data-fair/lib-validation": "^1.0.0",
     "@data-fair/lib-utils": "^1.1.0",
     "agentkeepalive": "^4.5.0",
-    "ajv-formats": "^3.0.1",
-    "ajv-i18n": "^4.2.0",
     "axios": "^1.7.7",
     "cacheable-lookup": "^7.0.0",
     "semver": "^7.6.3"

package/upgrade-scripts.js

@@ -34,9 +34,9 @@ async function runScripts (db, basePath, isFresh) {
   debug('list scripts from ' + scriptsRoot)
   let scripts = await listScripts(scriptsRoot)
   if (previousPersion) {
-    debug(`Current service version from database : ${previousPersion}`)
+    debug(`current service version from database : ${previousPersion}`)
   } else {
-    debug('No service version found in database, this is the first time the upgrade system is run')
+    debug('no service version found in database, this is the first time the upgrade system is run')
     scripts = scripts.filter(scriptDef => scriptDef.version === 'init')
     if (isFresh) {
       if (await isFresh()) {
@@ -46,7 +46,7 @@ async function runScripts (db, basePath, isFresh) {
         previousPersion = '0.0.0'
       }
     } else {
-      debug('No isFresh function, it could be a fresh install, do not run any script')
+      debug('no isFresh function, it could be a fresh install, do not run any script')
     }
   }
   if (previousPersion) {
@@ -54,7 +54,7 @@ async function runScripts (db, basePath, isFresh) {
       if (semver.gte(scriptDef.version, previousPersion)) {
         for (const scriptName of scriptDef.names) {
           const script = (await import(path.join(scriptsRoot, scriptDef.version, scriptName))).default
-          debug('Apply script %s/%s : %s', scriptDef.version, scriptName, script.description)
+          debug('apply script %s/%s : %s', scriptDef.version, scriptName, script.description)
           await script.exec(db, Debug(`upgrade:${scriptDef.version}:${scriptName}`))
         }
       }
@@ -63,7 +63,7 @@ async function runScripts (db, basePath, isFresh) {
   const coercedVersion = semver.coerce(pjson.version)
   if (!coercedVersion) { throw new Error(`Invalid version number in package.json ${pjson.version}`) }
   const newService = { id: pjson.name, version: coercedVersion.version }
-  debug(`Upgrade scripts are over, save current version number ${newService.version}`)
+  debug(`upgrade scripts are over, save current version number ${newService.version}`)
   await services.updateOne({ id: pjson.name }, { $set: newService }, { upsert: true })
 }
 // Walk the scripts directories

package/validation.d.ts

@@ -1,8 +1 @@
-import type { ErrorObject, ValidateFunction } from 'ajv';
-export declare const errorsText: (errors: ErrorObject[] | null | undefined, varName?: string) => string;
-export type AssertValidOptions = {
-    lang?: string;
-    name?: string;
-    internal?: boolean;
-};
-export declare function assertValid<T>(validate: ValidateFunction, data: any, options?: AssertValidOptions): asserts data is T;
+export * from '@data-fair/lib-validation';

package/validation.js

@@ -1,37 +1 @@
-import ajvFrModule from 'ajv-i18n/localize/fr/index.js'
-import ajvEnModule from 'ajv-i18n/localize/en/index.js'
-import { httpError } from '@data-fair/lib-utils/http-errors.js'
-const ajvFr = ajvFrModule
-const ajvEn = ajvEnModule
-const localize = {
-  fr: ajvFr,
-  en: ajvEn
-}
-// cf https://github.com/ajv-validator/ajv/blob/b3e0cb17d0e095b5c883042b2306571be5ec86b7/lib/core.ts#L650
-export const errorsText = (errors, varName = 'data') => {
-  if (!errors || errors.length === 0) { return 'No errors' }
-  return errors
-    .map((e) => {
-      let msg = `${varName}${e.instancePath} ${e.message}`.trim()
-      const paramKeys = Object.keys(e.params || {}).filter(key => {
-        if (key === 'error') { return false }
-        if (e.keyword === 'type' && key === 'type') { return false }
-        return true
-      })
-      const params = paramKeys
-        .map(key => paramKeys.length > 1 ? `${key}=${e.params[key]}` : e.params[key])
-        .join(', ')
-      if (params) { msg += ` (${params})` }
-      return msg
-    })
-    .reduce((text, msg) => text + ', ' + msg)
-}
-export function assertValid (validate, data, options = {}) {
-  const lang = options.lang ?? 'fr'
-  const name = options.name ?? 'data'
-  if (!validate(data)) {
-    (localize[lang] || localize.fr)(validate.errors)
-    const message = errorsText(validate.errors, name)
-    if (options.internal) { throw new Error(message) } else { throw httpError(400, message) }
-  }
-}
+export * from '@data-fair/lib-validation'