@data-fair/lib-node 0.2.0 → 1.0.0

package/axios-auth.js

@@ -1,34 +1,29 @@
 // build axios instances with sessions on a simple-directory instance
 // used for integration testing
-import { Agent } from 'node:http';
-import { axiosBuilder, axiosInstance } from './axios.js';
-export async function axiosAuth(opts) {
-    const body = { email: opts.email, password: opts.password };
-    if (opts.org)
-        body.org = opts.org;
-    if (opts.dep)
-        body.dep = opts.dep;
-    if (opts.adminMode)
-        body.adminMode = opts.adminMode;
-    const axiosOpts = {
-        httpAgent: new Agent({ keepAlive: false }),
-        ...opts.axiosOpts
-    };
-    const directoryUrl = opts.directoryUrl ?? 'http://localhost:8080';
-    let callbackUrl = (await axiosInstance.post(directoryUrl + '/api/auth/password', body, { params: { redirect: directoryUrl }, maxRedirects: 0 })).data;
-    if (callbackUrl.startsWith(directoryUrl + '/simple-directory')) {
-        callbackUrl = callbackUrl.replace(directoryUrl + '/simple-directory', directoryUrl);
-    }
-    try {
-        await axiosInstance.get(callbackUrl, { maxRedirects: 0 });
-    }
-    catch (err) {
-        if (err.status !== 302)
-            throw err;
-        axiosOpts.headers = axiosOpts.headers ?? {};
-        axiosOpts.headers.common = axiosOpts.headers.common ?? {};
-        axiosOpts.headers.common.Cookie = err.headers['set-cookie'].map((s) => s.split(';')[0]).join(';');
-    }
-    return axiosBuilder(axiosOpts);
+import { Agent } from 'node:http'
+import { axiosBuilder, axiosInstance } from './axios.js'
+export async function axiosAuth (opts) {
+  const body = { email: opts.email, password: opts.password }
+  if (opts.org) { body.org = opts.org }
+  if (opts.dep) { body.dep = opts.dep }
+  if (opts.adminMode) { body.adminMode = opts.adminMode }
+  const axiosOpts = {
+    httpAgent: new Agent({ keepAlive: false }),
+    ...opts.axiosOpts
+  }
+  const directoryUrl = opts.directoryUrl ?? 'http://localhost:8080'
+  let callbackUrl = (await axiosInstance.post(directoryUrl + '/api/auth/password', body, { params: { redirect: directoryUrl }, maxRedirects: 0 })).data
+  if (callbackUrl.startsWith(directoryUrl + '/simple-directory')) {
+    callbackUrl = callbackUrl.replace(directoryUrl + '/simple-directory', directoryUrl)
+  }
+  try {
+    await axiosInstance.get(callbackUrl, { maxRedirects: 0 })
+  } catch (err) {
+    if (err.status !== 302) { throw err }
+    axiosOpts.headers = axiosOpts.headers ?? {}
+    axiosOpts.headers.common = axiosOpts.headers.common ?? {}
+    axiosOpts.headers.common.Cookie = err.headers['set-cookie'].map((s) => s.split(';')[0]).join(';')
+  }
+  return axiosBuilder(axiosOpts)
 }
-export default axiosAuth;
+export default axiosAuth

package/axios.js

@@ -1,53 +1,50 @@
 // prepare an axios instance with improved error management and some performance tuning for nodejs usage
-import axios from 'axios';
-import { httpAgent, httpsAgent } from './http-agents.js';
-const { stackTraceLimit } = Error;
+import axios from 'axios'
+import { httpAgent, httpsAgent } from './http-agents.js'
+const { stackTraceLimit } = Error
 class AxiosRequestError extends Error {
-    constructor() {
-        super();
-        this.name = 'AxiosRequestError';
-    }
+  constructor () {
+    super()
+    this.name = 'AxiosRequestError'
+  }
 }
 const requestInterceptor = (config) => {
-    // better stack traces
-    // see https://github.com/axios/axios/issues/2387#issuecomment-652242713
-    Error.stackTraceLimit = 0;
-    const errorContext = new AxiosRequestError();
-    Error.stackTraceLimit = stackTraceLimit;
-    Error.captureStackTrace(errorContext, requestInterceptor);
-    // @ts-ignore
-    config.errorContext = errorContext;
-    return config;
-};
-export function axiosBuilder(opts = {}) {
-    const ax = axios.create({
-        httpAgent,
-        httpsAgent,
-        ...opts
-    });
-    ax.interceptors.request.use(requestInterceptor);
-    // shorter stack traces
-    ax.interceptors.response.use(response => response, error => {
-        if (!error.response) {
-            if (error.config?.errorContext?.stack)
-                error.stack += '\nRequest context:\n' + error.config?.errorContext?.stack;
-            return Promise.reject(error);
-        }
-        delete error.response.request;
-        error.response.config = { method: error.response.config.method, url: error.response.config.url, data: error.response.config.data };
-        if (error.response.config.data && error.response.config.data._writableState)
-            delete error.response.config.data;
-        if (error.response.data && error.response.data._readableState)
-            delete error.response.data;
-        let messageText = error.response.statusText;
-        if (error.response.data) {
-            messageText = typeof error.response.data === 'string' ? error.response.data : JSON.stringify(error.response.data);
-        }
-        error.response.message = `${error.response.status} - ${messageText}`;
-        Object.assign(error.config.errorContext, error.response);
-        return Promise.reject(error.config.errorContext);
-    });
-    return ax;
+  // better stack traces
+  // see https://github.com/axios/axios/issues/2387#issuecomment-652242713
+  Error.stackTraceLimit = 0
+  const errorContext = new AxiosRequestError()
+  Error.stackTraceLimit = stackTraceLimit
+  Error.captureStackTrace(errorContext, requestInterceptor)
+  // @ts-ignore
+  config.errorContext = errorContext
+  return config
+}
+export function axiosBuilder (opts = {}) {
+  const ax = axios.create({
+    httpAgent,
+    httpsAgent,
+    ...opts
+  })
+  ax.interceptors.request.use(requestInterceptor)
+  // shorter stack traces
+  ax.interceptors.response.use(response => response, error => {
+    if (!error.response) {
+      if (error.config?.errorContext?.stack) { error.stack += '\nRequest context:\n' + error.config?.errorContext?.stack }
+      return Promise.reject(error)
+    }
+    delete error.response.request
+    error.response.config = { method: error.response.config.method, url: error.response.config.url, data: error.response.config.data }
+    if (error.response.config.data && error.response.config.data._writableState) { delete error.response.config.data }
+    if (error.response.data && error.response.data._readableState) { delete error.response.data }
+    let messageText = error.response.statusText
+    if (error.response.data) {
+      messageText = typeof error.response.data === 'string' ? error.response.data : JSON.stringify(error.response.data)
+    }
+    error.response.message = `${error.response.status} - ${messageText}`
+    Object.assign(error.config.errorContext, error.response)
+    return Promise.reject(error.config.errorContext)
+  })
+  return ax
 }
-export const axiosInstance = axiosBuilder();
-export default axiosInstance;
+export const axiosInstance = axiosBuilder()
+export default axiosInstance

package/http-agents.js

@@ -1,55 +1,55 @@
 // use better DNS lookup than nodejs default and try to reduce number of socket openings
-import CacheableLookup from 'cacheable-lookup';
-import HttpAgent, { HttpsAgent } from 'agentkeepalive';
-import { Counter, Gauge } from 'prom-client';
+import CacheableLookup from 'cacheable-lookup'
+import HttpAgent, { HttpsAgent } from 'agentkeepalive'
+import { Counter, Gauge } from 'prom-client'
 // not very high number of sockets but we don't want to saturate our reverse proxies
 // if a higher number is needed the service should probably be scaled anyway
 const keepaliveOpts = {
-    maxSockets: 8,
-    maxFreeSockets: 8,
-    timeout: 60000,
-    freeSocketTimeout: 30000
-};
-export const httpAgent = new HttpAgent(keepaliveOpts);
-export const httpsAgent = new HttpsAgent(keepaliveOpts);
-const cacheableLookup = new CacheableLookup();
-cacheableLookup.install(httpAgent);
-cacheableLookup.install(httpsAgent);
+  maxSockets: 8,
+  maxFreeSockets: 8,
+  timeout: 60000,
+  freeSocketTimeout: 30000
+}
+export const httpAgent = new HttpAgent(keepaliveOpts)
+export const httpsAgent = new HttpsAgent(keepaliveOpts)
+const cacheableLookup = new CacheableLookup()
+cacheableLookup.install(httpAgent)
+cacheableLookup.install(httpsAgent)
 // monitor agent statuses
-const agents = { http: httpAgent, https: httpsAgent };
+const agents = { http: httpAgent, https: httpsAgent }
 const socketsGauge = new Gauge({
-    name: 'df_http_agent_sockets',
-    help: 'Number of open sockets in the http agent',
-    labelNames: ['protocol', 'host']
-});
+  name: 'df_http_agent_sockets',
+  help: 'Number of open sockets in the http agent',
+  labelNames: ['protocol', 'host']
+})
 const freeSocketsGauge = new Gauge({
-    name: 'df_http_agent_free_sockets',
-    help: 'Number of free sockets in the http agent',
-    labelNames: ['protocol', 'host']
-});
+  name: 'df_http_agent_free_sockets',
+  help: 'Number of free sockets in the http agent',
+  labelNames: ['protocol', 'host']
+})
 const createSocketCounter = new Counter({
-    name: 'df_http_agent_create_socket_total',
-    help: 'Total number of sockets created by the http agent',
-    labelNames: ['protocol']
-});
+  name: 'df_http_agent_create_socket_total',
+  help: 'Total number of sockets created by the http agent',
+  labelNames: ['protocol']
+})
 const requestCounter = new Counter({
-    name: 'df_http_agent_request_total',
-    help: 'Total number of requests created by the http agent',
-    labelNames: ['protocol']
-});
+  name: 'df_http_agent_request_total',
+  help: 'Total number of requests created by the http agent',
+  labelNames: ['protocol']
+})
 setInterval(() => {
-    for (const _protocol of ['http', 'https']) {
-        const protocol = _protocol;
-        const agentStatus = agents[protocol].getCurrentStatus();
-        for (const host in agentStatus.sockets) {
-            socketsGauge.set({ protocol, host }, agentStatus.sockets[host]);
-        }
-        for (const host in agentStatus.freeSockets) {
-            freeSocketsGauge.set({ protocol, host }, agentStatus.freeSockets[host]);
-        }
-        createSocketCounter.remove({ protocol });
-        createSocketCounter.inc({ protocol }, agentStatus.createSocketCount);
-        requestCounter.remove({ protocol });
-        requestCounter.inc({ protocol }, agentStatus.requestCount);
+  for (const _protocol of ['http', 'https']) {
+    const protocol = _protocol
+    const agentStatus = agents[protocol].getCurrentStatus()
+    for (const host in agentStatus.sockets) {
+      socketsGauge.set({ protocol, host }, agentStatus.sockets[host])
     }
-}, 60000);
+    for (const host in agentStatus.freeSockets) {
+      freeSocketsGauge.set({ protocol, host }, agentStatus.freeSockets[host])
+    }
+    createSocketCounter.remove({ protocol })
+    createSocketCounter.inc({ protocol }, agentStatus.createSocketCount)
+    requestCounter.remove({ protocol })
+    requestCounter.inc({ protocol }, agentStatus.requestCount)
+  }
+}, 60000)

package/locks.js

@@ -1,72 +1,63 @@
-import { hostname } from 'node:os';
-import { randomBytes } from 'node:crypto';
-import Debug from 'debug';
-const debug = Debug('locks');
-const pid = randomBytes(8).toString('hex');
-debug('locks with pid', pid);
-let interval;
-let _db;
+import { hostname } from 'node:os'
+import { randomBytes } from 'node:crypto'
+import Debug from 'debug'
+const debug = Debug('locks')
+const pid = randomBytes(8).toString('hex')
+debug('locks with pid', pid)
+let interval
+let _db
 const collection = () => {
-    if (!_db)
-        throw new Error('locks utils was not initialized');
-    return _db?.collection('locks');
-};
+  if (!_db) { throw new Error('locks utils was not initialized') }
+  return _db?.collection('locks')
+}
 export const init = async (db, ttl = 60) => {
-    _db = db;
-    const locks = collection();
-    await locks.createIndex({ pid: 1 });
-    try {
-        await locks.createIndex({ updatedAt: 1 }, { expireAfterSeconds: ttl });
-    }
-    catch (err) {
-        console.log('Failure to create TTL index. Probably because the value changed. Try to update it.');
-        db.command({ collMod: 'locks', index: { keyPattern: { updatedAt: 1 }, expireAfterSeconds: ttl } });
-    }
-    // prolongate lock acquired by this process while it is still active
-    interval = setInterval(() => {
-        locks.updateMany({ pid }, { $currentDate: { updatedAt: true } });
-    }, (ttl / 2) * 1000);
-};
+  _db = db
+  const locks = collection()
+  await locks.createIndex({ pid: 1 })
+  try {
+    await locks.createIndex({ updatedAt: 1 }, { expireAfterSeconds: ttl })
+  } catch (err) {
+    console.log('Failure to create TTL index. Probably because the value changed. Try to update it.')
+    db.command({ collMod: 'locks', index: { keyPattern: { updatedAt: 1 }, expireAfterSeconds: ttl } })
+  }
+  // prolongate lock acquired by this process while it is still active
+  interval = setInterval(() => {
+    locks.updateMany({ pid }, { $currentDate: { updatedAt: true } })
+  }, (ttl / 2) * 1000)
+}
 export const stop = async () => {
-    clearInterval(interval);
-    if (_db)
-        await collection().deleteMany({ pid });
-};
-export async function acquire(_id, origin) {
-    if (!_db)
-        throw new Error('locks not initialized');
-    debug('acquire', _id, origin);
-    const locks = collection();
+  clearInterval(interval)
+  if (_db) { await collection().deleteMany({ pid }) }
+}
+export async function acquire (_id, origin) {
+  if (!_db) { throw new Error('locks not initialized') }
+  debug('acquire', _id, origin)
+  const locks = collection()
+  try {
+    await locks.insertOne({ _id, pid, origin, hostname: hostname(), createdAt: new Date() })
     try {
-        await locks.insertOne({ _id, pid, origin, hostname: hostname(), createdAt: new Date() });
-        try {
-            await locks.updateOne({ _id }, { $currentDate: { updatedAt: true } });
-        }
-        catch (err) {
-            await locks.deleteOne({ _id, pid });
-            throw err;
-        }
-        debug('acquire ok', _id);
-        return true;
-    }
-    catch (err) {
-        if (err.code !== 11000)
-            throw err;
-        // duplicate means the lock was already acquired
-        debug('acquire ko', _id);
-        return false;
+      await locks.updateOne({ _id }, { $currentDate: { updatedAt: true } })
+    } catch (err) {
+      await locks.deleteOne({ _id, pid })
+      throw err
     }
+    debug('acquire ok', _id)
+    return true
+  } catch (err) {
+    if (err.code !== 11000) { throw err }
+    // duplicate means the lock was already acquired
+    debug('acquire ko', _id)
+    return false
+  }
 }
 export const release = async (_id, delay = 0) => {
-    if (!_db)
-        throw new Error('locks not initialized');
-    debug('release', _id);
-    const locks = collection();
-    if (delay) {
-        const date = new Date((new Date()).getTime() + delay);
-        await locks.updateOne({ _id, pid }, { $unset: { pid: 1 }, $set: { delayed: true, updatedAt: date } });
-    }
-    else {
-        await locks.deleteOne({ _id, pid });
-    }
-};
+  if (!_db) { throw new Error('locks not initialized') }
+  debug('release', _id)
+  const locks = collection()
+  if (delay) {
+    const date = new Date((new Date()).getTime() + delay)
+    await locks.updateOne({ _id, pid }, { $unset: { pid: 1 }, $set: { delayed: true, updatedAt: date } })
+  } else {
+    await locks.deleteOne({ _id, pid })
+  }
+}

package/mongo-types.js

@@ -1 +1 @@
-export {};
+export {}

package/mongo.js

@@ -1,63 +1,62 @@
-import { MongoClient, MongoError } from 'mongodb';
+import { MongoClient, MongoError } from 'mongodb'
 export class Mongo {
-    _client;
-    get client() {
-        if (!this._client)
-            throw new Error('db was not connected');
-        return this._client;
+  _client
+  get client () {
+    if (!this._client) { throw new Error('db was not connected') }
+    return this._client
+  }
+
+  get db () {
+    return this.client.db()
+  }
+
+  connect = async (mongoUrl, options = {}) => {
+    if (this._client) {
+      console.warn('db already connected')
+      return
     }
-    get db() {
-        return this.client.db();
+    console.log('connecting to mongodb...')
+    this._client = await MongoClient.connect(mongoUrl, { maxPoolSize: 5, ignoreUndefined: true, ...options })
+    console.log('...ok')
+  }
+
+  // Create an index if it does not exist and manage overwriting existing and conflicting index definitions
+  ensureIndex = async (collection, key, options = {}) => {
+    try {
+      await this.db.collection(collection).createIndex(key, options)
+    } catch (err) {
+      if (options.name && err instanceof MongoError && (err.code === 85 || err.code === 86)) {
+        // if the error is a conflict on keys or params of the index we automatically
+        // delete then recreate the index
+        console.log(`drop then recreate index ${collection}/${options.name}`)
+        await this.db.collection(collection).dropIndex(options.name)
+        await this.db.collection(collection).createIndex(key, options)
+      } else {
+        throw err
+      }
     }
-    connect = async (mongoUrl, options = {}) => {
-        if (this._client) {
-            console.warn('db already connected');
-            return;
-        }
-        console.log('connecting to mongodb...');
-        this._client = await MongoClient.connect(mongoUrl, { maxPoolSize: 5, ignoreUndefined: true, ...options });
-        console.log('...ok');
-    };
-    // Create an index if it does not exist and manage overwriting existing and conflicting index definitions
-    ensureIndex = async (collection, key, options = {}) => {
-        try {
-            await this.db.collection(collection).createIndex(key, options);
-        }
-        catch (err) {
-            if (options.name && err instanceof MongoError && (err.code === 85 || err.code === 86)) {
-                // if the error is a conflict on keys or params of the index we automatically
-                // delete then recreate the index
-                console.log(`drop then recreate index ${collection}/${options.name}`);
-                await this.db.collection(collection).dropIndex(options.name);
-                await this.db.collection(collection).createIndex(key, options);
-            }
-            else {
-                throw err;
-            }
-        }
-    };
-    /**
+  }
+
+  /**
      * create a bunch of indexes and overtwrite previous definitions
      * the structure forces giving names to indexes which is better for managing them properly
      */
-    configure = async (indexDefinitions) => {
-        for (const collectionName in indexDefinitions) {
-            for (const indexName in indexDefinitions[collectionName]) {
-                let key, options;
-                if (Array.isArray(indexDefinitions[collectionName][indexName])) {
-                    [key, options] = indexDefinitions[collectionName][indexName];
-                    if (options.name && options.name !== indexName)
-                        throw new Error(`inconsistent index name ${indexName} or ${options.name}`);
-                    options.name = indexName;
-                }
-                else {
-                    key = indexDefinitions[collectionName][indexName];
-                    options = { name: indexName };
-                }
-                await this.ensureIndex(collectionName, key, options);
-            }
+  configure = async (indexDefinitions) => {
+    for (const collectionName in indexDefinitions) {
+      for (const indexName in indexDefinitions[collectionName]) {
+        let key, options
+        if (Array.isArray(indexDefinitions[collectionName][indexName])) {
+          [key, options] = indexDefinitions[collectionName][indexName]
+          if (options.name && options.name !== indexName) { throw new Error(`inconsistent index name ${indexName} or ${options.name}`) }
+          options.name = indexName
+        } else {
+          key = indexDefinitions[collectionName][indexName]
+          options = { name: indexName }
         }
-    };
+        await this.ensureIndex(collectionName, key, options)
+      }
+    }
+  }
 }
-const mongo = new Mongo();
-export default mongo;
+const mongo = new Mongo()
+export default mongo

package/observer.js

@@ -1,83 +1,78 @@
 // very light web server to expose prometheus metrics, export a cpu profile and maybe other incoming obervability features
-import { hostname } from 'node:os';
-import { createServer } from 'node:http';
-import { Registry, Counter, register, collectDefaultMetrics } from 'prom-client';
-import eventPromise from '@data-fair/lib-utils/event-promise.js';
+import { hostname } from 'node:os'
+import { createServer } from 'node:http'
+import { Registry, Counter, register, collectDefaultMetrics } from 'prom-client'
+import eventPromise from '@data-fair/lib-utils/event-promise.js'
 // a special registry for metrics that are global
 // not based on this specific process state but instead on the content of db for example
-export const servicePromRegistry = new Registry();
-collectDefaultMetrics();
+export const servicePromRegistry = new Registry()
+collectDefaultMetrics()
 // local metrics incremented throughout the code
 const internalErrorCounter = new Counter({
-    name: 'df_internal_error',
-    help: 'A counter of errors from any service, worker, etc. Do not use for client errors, only for anomalies that should trigger alerts. Each increment should be accompanied by an error log with matching code.',
-    labelNames: ['errorCode']
-});
+  name: 'df_internal_error',
+  help: 'A counter of errors from any service, worker, etc. Do not use for client errors, only for anomalies that should trigger alerts. Each increment should be accompanied by an error log with matching code.',
+  labelNames: ['errorCode']
+})
 export const internalError = (errorCode, error, ...optionalParams) => {
-    internalErrorCounter.inc({ errorCode });
-    const message = error.message || error;
-    console.error(`[${errorCode}] ${message}`, ...optionalParams);
-    if (error.stack)
-        console.error(error.stack);
-};
+  internalErrorCounter.inc({ errorCode })
+  const message = error.message || error
+  console.error(`[${errorCode}] ${message}`, ...optionalParams)
+  if (error.stack) { console.error(error.stack) }
+}
 /**
  * @param {import('node:http').ServerResponse} res
  * @param {any} err
  */
 const httpErrorHandler = (res, err) => {
-    console.error('failed to server prometheus /metrics', err);
-    res.writeHead(500);
-    res.end();
-};
+  console.error('failed to server prometheus /metrics', err)
+  res.writeHead(500)
+  res.end()
+}
 const serveRegistry = async (res, registry) => {
-    const metrics = await registry.metrics();
-    res.setHeader('Content-Type', registry.contentType);
-    res.writeHead(200);
-    res.write(metrics);
-    res.end();
-};
+  const metrics = await registry.metrics()
+  res.setHeader('Content-Type', registry.contentType)
+  res.writeHead(200)
+  res.write(metrics)
+  res.end()
+}
 // live CPU performance inspection
 const serveCPUProfile = async (res, duration = 2000) => {
-    // @ts-ignore
-    const { Session } = await import('node:inspector/promises');
-    const session = new Session();
-    session.connect();
-    await session.post('Profiler.enable');
-    await session.post('Profiler.start');
-    await new Promise(resolve => setTimeout(resolve, duration));
-    const { profile } = await session.post('Profiler.stop');
-    session.disconnect();
-    res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.cpuprofile"`);
-    res.write(JSON.stringify(profile));
-    res.end();
-};
-let server;
+  // @ts-ignore
+  const { Session } = await import('node:inspector/promises')
+  const session = new Session()
+  session.connect()
+  await session.post('Profiler.enable')
+  await session.post('Profiler.start')
+  await new Promise(resolve => setTimeout(resolve, duration))
+  const { profile } = await session.post('Profiler.stop')
+  session.disconnect()
+  res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.cpuprofile"`)
+  res.write(JSON.stringify(profile))
+  res.end()
+}
+let server
 export const startObserver = async (port = 9090) => {
-    server = createServer((req, res) => {
-        if (req.method === 'GET' && req.url === '/metrics') {
-            serveRegistry(res, register).catch(err => httpErrorHandler(res, err));
-        }
-        else if (req.method === 'GET' && (req.url === '/global-metrics' || req.url === '/service-metrics')) {
-            serveRegistry(res, servicePromRegistry).catch(err => httpErrorHandler(res, err));
-        }
-        else if (req.method === 'GET' && req.url === '/cpu-profile') {
-            // TODO: duration
-            serveCPUProfile(res).catch(err => httpErrorHandler(res, err));
-        }
-        else {
-            res.writeHead(404);
-            res.end();
-        }
-    });
-    server.listen(port);
-    await eventPromise(server, 'listening');
-    console.log(`Observer server available on http://localhost:${port}
+  server = createServer((req, res) => {
+    if (req.method === 'GET' && req.url === '/metrics') {
+      serveRegistry(res, register).catch(err => httpErrorHandler(res, err))
+    } else if (req.method === 'GET' && (req.url === '/global-metrics' || req.url === '/service-metrics')) {
+      serveRegistry(res, servicePromRegistry).catch(err => httpErrorHandler(res, err))
+    } else if (req.method === 'GET' && req.url === '/cpu-profile') {
+      // TODO: duration
+      serveCPUProfile(res).catch(err => httpErrorHandler(res, err))
+    } else {
+      res.writeHead(404)
+      res.end()
+    }
+  })
+  server.listen(port)
+  await eventPromise(server, 'listening')
+  console.log(`Observer server available on http://localhost:${port}
     GET /metrics -> get prometheus metrics for this instance
     GET /service-metrics -> get prometheus metrics shared accross all instances
     GET /cpu-profile -> generate and fetch a CPU profile
-`);
-};
+`)
+}
 export const stopObserver = async () => {
-    if (server)
-        server.close();
-};
+  if (server) { server.close() }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/lib-node",
-  "version": "0.2.0",
+  "version": "1.0.0",
   "description": "Small nodejs related utility functions for the data-fair stack.",
   "main": "index.js",
   "files": [
@@ -9,9 +9,12 @@
   ],
   "author": "",
   "license": "MIT",
+  "scripts": {
+    "prepublishOnly": "cd .. && npm run prepublishOnly"
+  },
   "dependencies": {
-    "@data-fair/lib-common-types": "^0.2.0",
-    "@data-fair/lib-utils": "0.2.0",
+    "@data-fair/lib-common-types": "^1.1.0",
+    "@data-fair/lib-utils": "1.0.0",
     "agentkeepalive": "^4.5.0",
     "ajv-i18n": "^4.2.0",
     "axios": "^1.7.7",
@@ -19,9 +22,9 @@
     "semver": "^7.6.3"
   },
   "peerDependencies": {
-    "mongodb": "^6.9.0",
-    "prom-client": "^15.1.3",
-    "ws": "^8.18.0"
+    "mongodb": "6",
+    "prom-client": "15",
+    "ws": "8"
   },
   "peerDependenciesMeta": {
     "mongodb": {
@@ -34,5 +37,10 @@
       "optional": true
     }
   },
-  "type": "module"
+  "type": "module",
+  "devDependencies": {
+    "mongodb": "^6.9.0",
+    "prom-client": "^15.1.3",
+    "ws": "^8.18.0"
+  }
 }

package/test-module-hooks.js

@@ -1,16 +1,16 @@
 // @ts-nocheck
 // cf https://nodejs.org/api/module.html#customization-hooks
-const regexp = /\/\/ @test:spy\((.*)\)\n/g;
-export async function load(url, context, nextLoad) {
-    const result = await nextLoad(url, context);
-    if (result.format === 'module' || result.format === 'module-typescript') {
-        let source = result.source.toString();
-        const matches = source.match(regexp);
-        if (matches) {
-            source = 'import { emit as __emitTestSpy } from "@data-fair/lib/node/test-spies.js"\n' + source;
-            source = source.replace(regexp, '__emitTestSpy($1)\n');
-            result.source = Buffer.from(source, 'utf-8');
-        }
+const regexp = /\/\/ @test:spy\((.*)\)\n/g
+export async function load (url, context, nextLoad) {
+  const result = await nextLoad(url, context)
+  if (result.format === 'module' || result.format === 'module-typescript') {
+    let source = result.source.toString()
+    const matches = source.match(regexp)
+    if (matches) {
+      source = 'import { emit as __emitTestSpy } from "@data-fair/lib/node/test-spies.js"\n' + source
+      source = source.replace(regexp, '__emitTestSpy($1)\n')
+      result.source = Buffer.from(source, 'utf-8')
     }
-    return result;
+  }
+  return result
 }

package/test-spies.js

@@ -1,28 +1,26 @@
-import { register } from 'node:module';
-export function registerModuleHooks() {
-    register('@data-fair/lib/node/test-module-hooks.js', { parentURL: import.meta.url });
+import { register } from 'node:module'
+export function registerModuleHooks () {
+  register('@data-fair/lib/node/test-module-hooks.js', { parentURL: import.meta.url })
 }
-const pendingPromises = {};
-let i = 0;
-export function waitFor(eventName, timeout = 2000) {
-    i++;
-    const id = i;
-    const p = new Promise((resolve, reject) => {
-        pendingPromises[eventName] = pendingPromises[eventName] ?? [];
-        pendingPromises[eventName].push({ resolve, id });
-        setTimeout(() => reject(new Error(`Timeout waiting for test spy ${eventName}`)), timeout);
-    });
-    p.finally(() => {
-        pendingPromises[eventName] = pendingPromises[eventName].filter(p => p.id !== id);
-        if (pendingPromises[eventName].length === 0)
-            delete pendingPromises[eventName];
-    });
-    return p;
+const pendingPromises = {}
+let i = 0
+export function waitFor (eventName, timeout = 2000) {
+  i++
+  const id = i
+  const p = new Promise((resolve, reject) => {
+    pendingPromises[eventName] = pendingPromises[eventName] ?? []
+    pendingPromises[eventName].push({ resolve, id })
+    setTimeout(() => reject(new Error(`Timeout waiting for test spy ${eventName}`)), timeout)
+  })
+  p.finally(() => {
+    pendingPromises[eventName] = pendingPromises[eventName].filter(p => p.id !== id)
+    if (pendingPromises[eventName].length === 0) { delete pendingPromises[eventName] }
+  })
+  return p
 }
-export function emit(eventName, data) {
-    if (!pendingPromises[eventName])
-        return;
-    for (const p of pendingPromises[eventName]) {
-        p.resolve(data);
-    }
+export function emit (eventName, data) {
+  if (!pendingPromises[eventName]) { return }
+  for (const p of pendingPromises[eventName]) {
+    p.resolve(data)
+  }
 }

package/upgrade-scripts.js

@@ -1,68 +1,65 @@
-import semver from 'semver';
-import path from 'path';
-import { readdirSync, readFileSync } from 'fs';
-import Debug from 'debug';
-import { acquire, release } from './locks.js';
-const debug = Debug('upgrade');
+import semver from 'semver'
+import path from 'path'
+import { readdirSync, readFileSync } from 'fs'
+import Debug from 'debug'
+import { acquire, release } from './locks.js'
+const debug = Debug('upgrade')
 // chose the proper scripts to execute, then run them
 export default async function (db, basePath = './') {
-    const ack = await acquire('upgrade');
-    if (!ack) {
-        console.warn('upgrade scripts lock is already acquired, skip them');
-        // IMPORTANT: this behaviour of running the process when the upgrade scripts are still running on another one
-        // implies that they cannot be considered a pre-requisite..
-        // if we want to consider the upgrade scripts as a pre-requisite we should implement a wait on all
-        // containers for the scripts that are running in only 1 (while loop on "acquire" ?) and a healthcheck
-        // are not considered "up" and the previous versions keep running in the mean time
-    }
-    else {
-        await runScripts(db, basePath);
-        await release('upgrade');
-    }
+  const ack = await acquire('upgrade')
+  if (!ack) {
+    console.warn('upgrade scripts lock is already acquired, skip them')
+    // IMPORTANT: this behaviour of running the process when the upgrade scripts are still running on another one
+    // implies that they cannot be considered a pre-requisite..
+    // if we want to consider the upgrade scripts as a pre-requisite we should implement a wait on all
+    // containers for the scripts that are running in only 1 (while loop on "acquire" ?) and a healthcheck
+    // are not considered "up" and the previous versions keep running in the mean time
+  } else {
+    await runScripts(db, basePath)
+    await release('upgrade')
+  }
 }
-async function runScripts(db, basePath) {
-    const dir = path.resolve(basePath);
-    const pjsonPath = path.join(dir, 'package.json');
-    debug('read service info from ' + pjsonPath);
-    const pjson = JSON.parse(readFileSync(pjsonPath, 'utf8'));
-    const scriptsRoot = path.join(dir, 'upgrade');
-    debug(`service=${pjson.name} version=${pjson.version}`);
-    const services = db.collection('services');
-    const service = await services.findOne({ id: pjson.name });
-    const version = service?.version;
-    debug('list scripts from ' + scriptsRoot);
-    let scripts = await listScripts(scriptsRoot);
-    if (!version) {
-        debug('No service version found in database, this is probably a fresh install');
-        scripts = scripts.filter(scriptDef => scriptDef.version === 'init');
-    }
-    else {
-        debug(`Current service version from database : ${version}`);
-        scripts = scripts.filter(scriptDef => scriptDef.version !== 'init');
+async function runScripts (db, basePath) {
+  const dir = path.resolve(basePath)
+  const pjsonPath = path.join(dir, 'package.json')
+  debug('read service info from ' + pjsonPath)
+  const pjson = JSON.parse(readFileSync(pjsonPath, 'utf8'))
+  const scriptsRoot = path.join(dir, 'upgrade')
+  debug(`service=${pjson.name} version=${pjson.version}`)
+  const services = db.collection('services')
+  const service = await services.findOne({ id: pjson.name })
+  const version = service?.version
+  debug('list scripts from ' + scriptsRoot)
+  let scripts = await listScripts(scriptsRoot)
+  if (!version) {
+    debug('No service version found in database, this is probably a fresh install')
+    scripts = scripts.filter(scriptDef => scriptDef.version === 'init')
+  } else {
+    debug(`Current service version from database : ${version}`)
+    scripts = scripts.filter(scriptDef => scriptDef.version !== 'init')
+  }
+  for (const scriptDef of scripts) {
+    if (semver.gte(scriptDef.version, version)) {
+      for (const scriptName of scriptDef.names) {
+        const script = (await import(path.join(scriptsRoot, scriptDef.version, scriptName))).default
+        debug('Apply script %s/%s : %s', scriptDef.version, scriptName, script.description)
+        await script.exec(db, Debug(`upgrade:${scriptDef.version}:${scriptName}`))
+      }
     }
-    for (const scriptDef of scripts) {
-        if (semver.gte(scriptDef.version, version)) {
-            for (const scriptName of scriptDef.names) {
-                const script = (await import(path.join(scriptsRoot, scriptDef.version, scriptName))).default;
-                debug('Apply script %s/%s : %s', scriptDef.version, scriptName, script.description);
-                await script.exec(db, Debug(`upgrade:${scriptDef.version}:${scriptName}`));
-            }
-        }
-    }
-    const coercedVersion = semver.coerce(pjson.version);
-    if (!coercedVersion)
-        throw new Error(`Invalid version number in package.json ${pjson.version}`);
-    const newService = { id: pjson.name, version: coercedVersion.version };
-    debug(`Upgrade scripts are over, save current version number ${newService.version}`);
-    await services.updateOne({ id: pjson.name }, { $set: newService }, { upsert: true });
+  }
+  const coercedVersion = semver.coerce(pjson.version)
+  if (!coercedVersion) { throw new Error(`Invalid version number in package.json ${pjson.version}`) }
+  const newService = { id: pjson.name, version: coercedVersion.version }
+  debug(`Upgrade scripts are over, save current version number ${newService.version}`)
+  await services.updateOne({ id: pjson.name }, { $set: newService }, { upsert: true })
 }
 // Walk the scripts directories
-async function listScripts(scriptsRoot) {
-    const dirs = readdirSync(scriptsRoot).sort();
-    return dirs.map(dir => {
-        return {
-            version: dir,
-            names: readdirSync(path.join(scriptsRoot, dir)).sort()
-        };
-    });
+async function listScripts (scriptsRoot) {
+  const dirs = readdirSync(scriptsRoot).sort()
+  return dirs.map(dir => {
+    return {
+      version: dir,
+      names: readdirSync(path.join(scriptsRoot, dir)).sort()
+    }
+  })
 }

package/validation.js

@@ -1,44 +1,37 @@
-import ajvFrModule from 'ajv-i18n/localize/fr/index.js';
-import ajvEnModule from 'ajv-i18n/localize/en/index.js';
-import { httpError } from '@data-fair/lib-utils/http-errors.js';
-const ajvFr = ajvFrModule;
-const ajvEn = ajvEnModule;
+import ajvFrModule from 'ajv-i18n/localize/fr/index.js'
+import ajvEnModule from 'ajv-i18n/localize/en/index.js'
+import { httpError } from '@data-fair/lib-utils/http-errors.js'
+const ajvFr = ajvFrModule
+const ajvEn = ajvEnModule
 const localize = {
-    fr: ajvFr,
-    en: ajvEn
-};
+  fr: ajvFr,
+  en: ajvEn
+}
 // cf https://github.com/ajv-validator/ajv/blob/b3e0cb17d0e095b5c883042b2306571be5ec86b7/lib/core.ts#L650
 export const errorsText = (errors, varName = 'data') => {
-    if (!errors || errors.length === 0)
-        return 'No errors';
-    return errors
-        .map((e) => {
-        let msg = `${varName}${e.instancePath} ${e.message}`.trim();
-        const paramKeys = Object.keys(e.params || {}).filter(key => {
-            if (key === 'error')
-                return false;
-            if (e.keyword === 'type' && key === 'type')
-                return false;
-            return true;
-        });
-        const params = paramKeys
-            .map(key => paramKeys.length > 1 ? `${key}=${e.params[key]}` : e.params[key])
-            .join(', ');
-        if (params)
-            msg += ` (${params})`;
-        return msg;
+  if (!errors || errors.length === 0) { return 'No errors' }
+  return errors
+    .map((e) => {
+      let msg = `${varName}${e.instancePath} ${e.message}`.trim()
+      const paramKeys = Object.keys(e.params || {}).filter(key => {
+        if (key === 'error') { return false }
+        if (e.keyword === 'type' && key === 'type') { return false }
+        return true
+      })
+      const params = paramKeys
+        .map(key => paramKeys.length > 1 ? `${key}=${e.params[key]}` : e.params[key])
+        .join(', ')
+      if (params) { msg += ` (${params})` }
+      return msg
     })
-        .reduce((text, msg) => text + ', ' + msg);
-};
-export function assertValid(validate, data, options = {}) {
-    const lang = options.lang ?? 'fr';
-    const name = options.name ?? 'data';
-    if (!validate(data)) {
-        (localize[lang] || localize.fr)(validate.errors);
-        const message = errorsText(validate.errors, name);
-        if (options.internal)
-            throw new Error(message);
-        else
-            throw httpError(400, message);
-    }
+    .reduce((text, msg) => text + ', ' + msg)
+}
+export function assertValid (validate, data, options = {}) {
+  const lang = options.lang ?? 'fr'
+  const name = options.name ?? 'data'
+  if (!validate(data)) {
+    (localize[lang] || localize.fr)(validate.errors)
+    const message = errorsText(validate.errors, name)
+    if (options.internal) { throw new Error(message) } else { throw httpError(400, message) }
+  }
 }

package/ws-client.js

@@ -1,97 +1,93 @@
 // a wrapper around a websocket connection with auto reconnect
 // JSON parsing and channel subscriptions
-import EventEmitter from 'node:events';
-import WebSocket from 'ws';
-import Debug from 'debug';
-const debug = Debug('ws');
+import EventEmitter from 'node:events'
+import WebSocket from 'ws'
+import Debug from 'debug'
+const debug = Debug('ws')
 export class WsClient extends EventEmitter {
-    _channels;
-    _ws;
-    opts;
-    constructor(opts) {
-        super();
-        this._channels = [];
-        this.opts = { log: console, ...opts };
+  _channels
+  _ws
+  opts
+  constructor (opts) {
+    super()
+    this._channels = []
+    this.opts = { log: console, ...opts }
+  }
+
+  async _connect () {
+    return await new Promise((resolve, reject) => {
+      const wsUrl = this.opts.url.replace('http://', 'ws://').replace('https://', 'wss://') + '/'
+      debug(`connect Web Socket to ${wsUrl}`)
+      const ws = new WebSocket(wsUrl, { headers: this.opts.headers ?? {} })
+      this._ws = ws
+      ws.on('error', (err) => {
+        debug('WS encountered an error', err.message)
+        this._reconnect()
+        reject(err)
+      })
+      ws.once('open', () => {
+        debug('WS is opened')
+        resolve(ws)
+      })
+      ws.on('message', (msg) => {
+        const message = JSON.parse(msg)
+        debug('received message', message)
+        this.emit('message', message)
+      })
+    })
+  }
+
+  async _reconnect () {
+    if (!this._ws) { return }
+    debug('reconnect')
+    this._ws.terminate()
+    await this._connect()
+    for (const channel of this._channels) {
+      await this.subscribe(channel, true)
     }
-    async _connect() {
-        return await new Promise((resolve, reject) => {
-            const wsUrl = this.opts.url.replace('http://', 'ws://').replace('https://', 'wss://') + '/';
-            debug(`connect Web Socket to ${wsUrl}`);
-            const ws = new WebSocket(wsUrl, { headers: this.opts.headers ?? {} });
-            this._ws = ws;
-            ws.on('error', (err) => {
-                debug('WS encountered an error', err.message);
-                this._reconnect();
-                reject(err);
-            });
-            ws.once('open', () => {
-                debug('WS is opened');
-                resolve(ws);
-            });
-            ws.on('message', (msg) => {
-                const message = JSON.parse(msg);
-                debug('received message', message);
-                this.emit('message', message);
-            });
-        });
-    }
-    async _reconnect() {
-        if (!this._ws)
-            return;
-        debug('reconnect');
-        this._ws.terminate();
-        await this._connect();
-        for (const channel of this._channels) {
-            await this.subscribe(channel, true);
+  }
+
+  async subscribe (channel, force = false, timeout = 2000) {
+    if (this._channels.includes(channel) && !force) { return }
+    const ws = this._ws ?? await this._connect()
+    debug('subscribe to channel', channel)
+    const subscribeMessage = { type: 'subscribe', channel }
+    if (this.opts.apiKey) { subscribeMessage.apiKey = this.opts.apiKey }
+    if (this.opts.adminMode && this.opts.account) { subscribeMessage.account = JSON.stringify(this.opts.account) }
+    ws.send(JSON.stringify(subscribeMessage))
+    const event = await this.waitFor(channel, (e) => {
+      return e.type === 'subscribe-confirm' || e.type === 'error'
+    }, timeout, true, true)
+    if (event.type === 'error') { throw new Error(event.data) }
+    if (this._channels.includes(channel)) { this._channels.push(channel) }
+  }
+
+  async waitFor (channel, filter, timeout = 300000, skipSubscribe = false, fullMessage = false) {
+    if (!skipSubscribe) { await this.subscribe(channel) }
+    return await new Promise((resolve, reject) => {
+      const _timeout = setTimeout(() => { reject(new Error('timeout')) }, timeout)
+      const messageCb = (message) => {
+        if (message.channel === channel && (!filter || filter(fullMessage ? message : message.data))) {
+          clearTimeout(_timeout)
+          this.off('message', messageCb)
+          resolve(fullMessage ? message : message.data)
         }
-    }
-    async subscribe(channel, force = false, timeout = 2000) {
-        if (this._channels.includes(channel) && !force)
-            return;
-        const ws = this._ws ?? await this._connect();
-        debug('subscribe to channel', channel);
-        const subscribeMessage = { type: 'subscribe', channel };
-        if (this.opts.apiKey)
-            subscribeMessage.apiKey = this.opts.apiKey;
-        if (this.opts.adminMode && this.opts.account)
-            subscribeMessage.account = JSON.stringify(this.opts.account);
-        ws.send(JSON.stringify(subscribeMessage));
-        const event = await this.waitFor(channel, (e) => {
-            return e.type === 'subscribe-confirm' || e.type === 'error';
-        }, timeout, true, true);
-        if (event.type === 'error')
-            throw new Error(event.data);
-        if (this._channels.includes(channel))
-            this._channels.push(channel);
-    }
-    async waitFor(channel, filter, timeout = 300000, skipSubscribe = false, fullMessage = false) {
-        if (!skipSubscribe)
-            await this.subscribe(channel);
-        return await new Promise((resolve, reject) => {
-            const _timeout = setTimeout(() => { reject(new Error('timeout')); }, timeout);
-            const messageCb = (message) => {
-                if (message.channel === channel && (!filter || filter(fullMessage ? message : message.data))) {
-                    clearTimeout(_timeout);
-                    this.off('message', messageCb);
-                    resolve(fullMessage ? message : message.data);
-                }
-            };
-            this.on('message', messageCb);
-        });
-    }
-    close() {
-        if (this._ws)
-            this._ws.terminate();
-    }
+      }
+      this.on('message', messageCb)
+    })
+  }
+
+  close () {
+    if (this._ws) { this._ws.terminate() }
+  }
 }
 export class DataFairWsClient extends WsClient {
-    async waitForJournal(datasetId, eventType, timeout = 300000) {
-        await this.opts.log.info(`wait for event "${eventType}" on dataset "${datasetId}"`);
-        const event = await this.waitFor(`datasets/${datasetId}/journal`, (e) => {
-            return e.type === eventType || e.type === 'error';
-        }, timeout);
-        if (event.type === 'error')
-            throw new Error(event.data);
-        return event;
-    }
+  async waitForJournal (datasetId, eventType, timeout = 300000) {
+    await this.opts.log.info(`wait for event "${eventType}" on dataset "${datasetId}"`)
+    const event = await this.waitFor(`datasets/${datasetId}/journal`, (e) => {
+      return e.type === eventType || e.type === 'error'
+    }, timeout)
+    if (event.type === 'error') { throw new Error(event.data) }
+    return event
+  }
 }

package/ws-emitter.js

@@ -1,18 +1,16 @@
 // write messages to a mongodb collection/queue for publication by ws-server.js and consumption by ws.js
 // return a capped mongodb collection that acts as a simple queue
 export const initMessagesCollection = async (db) => {
-    const collection = (await db.listCollections({ name: 'ws-messages' }).toArray())[0];
-    if (!collection)
-        await db.createCollection('ws-messages', { capped: true, size: 100000, max: 1000 });
-    return db.collection('ws-messages');
-};
-let messagesCollection;
+  const collection = (await db.listCollections({ name: 'ws-messages' }).toArray())[0]
+  if (!collection) { await db.createCollection('ws-messages', { capped: true, size: 100000, max: 1000 }) }
+  return db.collection('ws-messages')
+}
+let messagesCollection
 export const init = async (db) => {
-    messagesCollection = await initMessagesCollection(db);
-    await messagesCollection.insertOne({ type: 'init' });
-};
+  messagesCollection = await initMessagesCollection(db)
+  await messagesCollection.insertOne({ type: 'init' })
+}
 export const emit = async (channel, data) => {
-    if (!messagesCollection)
-        throw new Error('wsEmitter.init was not called before sending data');
-    await messagesCollection.insertOne({ type: 'message', channel, data, date: new Date().toISOString() });
-};
+  if (!messagesCollection) { throw new Error('wsEmitter.init was not called before sending data') }
+  await messagesCollection.insertOne({ type: 'message', channel, data, date: new Date().toISOString() })
+}