@data-fair/lib-node 2.8.2 → 2.9.1

package/locks.js

@@ -1,75 +1,77 @@
-import { hostname } from 'node:os'
-import { randomBytes } from 'node:crypto'
-import Debug from 'debug'
-const debug = Debug('locks')
+import { hostname } from 'node:os';
+import { randomBytes } from 'node:crypto';
+import Debug from 'debug';
+const debug = Debug('locks');
 export class Locks {
-  pid
-  interval
-  _db
-  get db () {
-    if (!this._db) { throw new Error('locks utils was not initialized') }
-    return this._db
-  }
-
-  get collection () {
-    return this.db.collection('locks')
-  }
-
-  constructor () {
-    this.pid = randomBytes(8).toString('hex')
-    debug('locks with pid', this.pid)
-  }
-
-  start = async (db, ttl = 60) => {
-    if (this._db) { throw new Error('locks utils was already initialized') }
-    this._db = db
-    await this.collection.createIndex({ pid: 1 })
-    try {
-      await this.collection.createIndex({ updatedAt: 1 }, { expireAfterSeconds: ttl })
-    } catch (err) {
-      console.log('Failure to create TTL index. Probably because the value changed. Try to update it.')
-      db.command({ collMod: 'locks', index: { keyPattern: { updatedAt: 1 }, expireAfterSeconds: ttl } })
+    pid;
+    interval;
+    _db;
+    get db() {
+        if (!this._db)
+            throw new Error('locks utils was not initialized');
+        return this._db;
     }
-    // prolongate lock acquired by this process while it is still active
-    this.interval = setInterval(() => {
-      this.collection.updateMany({ pid: this.pid }, { $currentDate: { updatedAt: true } })
-    }, (ttl / 2) * 1000)
-  }
-
-  stop = async () => {
-    clearInterval(this.interval)
-    if (this._db) { await this.collection.deleteMany({ pid: this.pid }) }
-  }
-
-  acquire = async (_id, origin) => {
-    debug('acquire', _id, origin)
-    try {
-      await this.collection.insertOne({ _id, pid: this.pid, origin, hostname: hostname(), createdAt: new Date() })
-      try {
-        await this.collection.updateOne({ _id }, { $currentDate: { updatedAt: true } })
-      } catch (err) {
-        await this.collection.deleteOne({ _id, pid: this.pid })
-        throw err
-      }
-      debug('acquire ok', _id)
-      return true
-    } catch (err) {
-      if (err.code !== 11000) { throw err }
-      // duplicate means the lock was already acquired
-      debug('acquire ko', _id)
-      return false
+    get collection() {
+        return this.db.collection('locks');
     }
-  }
-
-  release = async (_id, delay = 0) => {
-    debug('release', _id)
-    if (delay) {
-      const date = new Date((new Date()).getTime() + delay)
-      await this.collection.updateOne({ _id, pid: this.pid }, { $unset: { pid: 1 }, $set: { delayed: true, updatedAt: date } })
-    } else {
-      await this.collection.deleteOne({ _id, pid: this.pid })
+    constructor() {
+        this.pid = randomBytes(8).toString('hex');
+        debug('locks with pid', this.pid);
     }
-  }
+    start = async (db, ttl = 60) => {
+        if (this._db)
+            throw new Error('locks utils was already initialized');
+        this._db = db;
+        await this.collection.createIndex({ pid: 1 });
+        try {
+            await this.collection.createIndex({ updatedAt: 1 }, { expireAfterSeconds: ttl });
+        }
+        catch (err) {
+            console.log('Failure to create TTL index. Probably because the value changed. Try to update it.');
+            db.command({ collMod: 'locks', index: { keyPattern: { updatedAt: 1 }, expireAfterSeconds: ttl } });
+        }
+        // prolongate lock acquired by this process while it is still active
+        this.interval = setInterval(() => {
+            this.collection.updateMany({ pid: this.pid }, { $currentDate: { updatedAt: true } });
+        }, (ttl / 2) * 1000);
+    };
+    stop = async () => {
+        clearInterval(this.interval);
+        if (this._db)
+            await this.collection.deleteMany({ pid: this.pid });
+    };
+    acquire = async (_id, origin) => {
+        debug('acquire', _id, origin);
+        try {
+            await this.collection.insertOne({ _id, pid: this.pid, origin, hostname: hostname(), createdAt: new Date() });
+            try {
+                await this.collection.updateOne({ _id }, { $currentDate: { updatedAt: true } });
+            }
+            catch (err) {
+                await this.collection.deleteOne({ _id, pid: this.pid });
+                throw err;
+            }
+            debug('acquire ok', _id);
+            return true;
+        }
+        catch (err) {
+            if (err.code !== 11000)
+                throw err;
+            // duplicate means the lock was already acquired
+            debug('acquire ko', _id);
+            return false;
+        }
+    };
+    release = async (_id, delay = 0) => {
+        debug('release', _id);
+        if (delay) {
+            const date = new Date((new Date()).getTime() + delay);
+            await this.collection.updateOne({ _id, pid: this.pid }, { $unset: { pid: 1 }, $set: { delayed: true, updatedAt: date } });
+        }
+        else {
+            await this.collection.deleteOne({ _id, pid: this.pid });
+        }
+    };
 }
-const locks = new Locks()
-export default locks
+const locks = new Locks();
+export default locks;

package/mongo.js

@@ -1,76 +1,80 @@
-import { MongoClient, MongoError } from 'mongodb'
+import { MongoClient, MongoError } from 'mongodb';
 export class Mongo {
-  _client
-  _db
-  get client () {
-    if (!this._client) { throw new Error('db was not connected') }
-    return this._client
-  }
-
-  get db () {
-    if (!this._db) { throw new Error('db was not connected') }
-    return this._db
-  }
-
-  connect = async (mongoUrl, options = {}) => {
-    if (this._client) {
-      console.warn('db already connected')
-      return
+    _client;
+    _db;
+    get client() {
+        if (!this._client)
+            throw new Error('db was not connected');
+        return this._client;
     }
-    console.log('connecting to mongodb...')
-    this._client = await MongoClient.connect(mongoUrl, { maxPoolSize: 5, ignoreUndefined: true, ...options })
-    this._db = this._client.db()
-    console.log('...ok')
-  }
-
-  // Create an index if it does not exist and manage overwriting existing and conflicting index definitions
-  ensureIndex = async (collection, key, options = {}) => {
-    try {
-      await this.db.collection(collection).createIndex(key, options)
-    } catch (err) {
-      if (options.name && err instanceof MongoError && (err.code === 85 || err.code === 86)) {
-        // if the error is a conflict on keys or params of the index we automatically
-        // delete then recreate the index
-        console.log(`drop then recreate index ${collection}/${options.name}`)
-        await this.db.collection(collection).dropIndex(options.name)
-        await this.db.collection(collection).createIndex(key, options)
-      } else {
-        throw err
-      }
+    get db() {
+        if (!this._db)
+            throw new Error('db was not connected');
+        return this._db;
     }
-  }
-
-  /**
+    connect = async (mongoUrl, options = {}) => {
+        if (this._client) {
+            console.warn('db already connected');
+            return;
+        }
+        console.log('connecting to mongodb...');
+        this._client = await MongoClient.connect(mongoUrl, { maxPoolSize: 5, ignoreUndefined: true, ...options });
+        this._db = this._client.db();
+        console.log('...ok');
+    };
+    // Create an index if it does not exist and manage overwriting existing and conflicting index definitions
+    ensureIndex = async (collection, key, options = {}) => {
+        try {
+            await this.db.collection(collection).createIndex(key, options);
+        }
+        catch (err) {
+            if (options.name && err instanceof MongoError && (err.code === 85 || err.code === 86)) {
+                // if the error is a conflict on keys or params of the index we automatically
+                // delete then recreate the index
+                console.log(`drop then recreate index ${collection}/${options.name}`);
+                await this.db.collection(collection).dropIndex(options.name);
+                await this.db.collection(collection).createIndex(key, options);
+            }
+            else {
+                throw err;
+            }
+        }
+    };
+    /**
      * create a bunch of indexes and overtwrite previous definitions
      * the structure forces giving names to indexes which is better for managing them properly
      */
-  configure = async (indexDefinitions) => {
-    for (const collectionName in indexDefinitions) {
-      try {
-        await this.db.createCollection(collectionName)
-      } catch (err) {
-        // nothing TODO
-      }
-      for (const indexName in indexDefinitions[collectionName]) {
-        let key, options
-        const indexDefinition = indexDefinitions[collectionName][indexName]
-        if (indexDefinition === null) {
-          if (await this.db.collection(collectionName).indexExists(indexName)) {
-            await this.db.collection(collectionName).dropIndex(indexName)
-          }
-          continue
-        } else if (Array.isArray(indexDefinition)) {
-          [key, options] = indexDefinition
-          if (options.name && options.name !== indexName) { throw new Error(`inconsistent index name ${indexName} or ${options.name}`) }
-          options.name = indexName
-        } else {
-          key = indexDefinition
-          options = { name: indexName }
+    configure = async (indexDefinitions) => {
+        for (const collectionName in indexDefinitions) {
+            try {
+                await this.db.createCollection(collectionName);
+            }
+            catch (err) {
+                // nothing TODO
+            }
+            for (const indexName in indexDefinitions[collectionName]) {
+                let key, options;
+                const indexDefinition = indexDefinitions[collectionName][indexName];
+                if (indexDefinition === null) {
+                    if (await this.db.collection(collectionName).indexExists(indexName)) {
+                        await this.db.collection(collectionName).dropIndex(indexName);
+                    }
+                    continue;
+                }
+                else if (Array.isArray(indexDefinition)) {
+                    [key, options] = indexDefinition;
+                    if (options.name && options.name !== indexName)
+                        throw new Error(`inconsistent index name ${indexName} or ${options.name}`);
+                    options.name = indexName;
+                }
+                else {
+                    key = indexDefinition;
+                    options = { name: indexName };
+                }
+                await this.ensureIndex(collectionName, key, options);
+            }
         }
-        await this.ensureIndex(collectionName, key, options)
-      }
-    }
-  }
+    };
 }
-const mongo = new Mongo()
-export default mongo
+const mongo = new Mongo();
+export default mongo;

package/observer.js

@@ -1,102 +1,109 @@
 // very light web server to expose prometheus metrics, export a cpu profile and maybe other incoming obervability features
-import { hostname } from 'node:os'
-import { createServer } from 'node:http'
-import { Registry, Counter, register, collectDefaultMetrics } from 'prom-client'
-import eventPromise from '@data-fair/lib-utils/event-promise.js'
+import { hostname } from 'node:os';
+import { createServer } from 'node:http';
+import { Registry, Counter, register, collectDefaultMetrics } from 'prom-client';
+import eventPromise from '@data-fair/lib-utils/event-promise.js';
 // a special registry for metrics that are global
 // not based on this specific process state but instead on the content of db for example
-export const servicePromRegistry = new Registry()
-collectDefaultMetrics()
+export const servicePromRegistry = new Registry();
+collectDefaultMetrics();
 // local metrics incremented throughout the code
 const internalErrorCounter = new Counter({
-  name: 'df_internal_error',
-  help: 'A counter of errors from any service, worker, etc. Do not use for client errors, only for anomalies that should trigger alerts. Each increment should be accompanied by an error log with matching code.',
-  labelNames: ['errorCode']
-})
+    name: 'df_internal_error',
+    help: 'A counter of errors from any service, worker, etc. Do not use for client errors, only for anomalies that should trigger alerts. Each increment should be accompanied by an error log with matching code.',
+    labelNames: ['errorCode']
+});
 export const internalError = (errorCode, error, ...optionalParams) => {
-  internalErrorCounter.inc({ errorCode })
-  const message = error.message || error
-  console.error(`[${errorCode}] ${message}`, ...optionalParams)
-  if (error.stack) { console.error(error.stack) }
-}
+    internalErrorCounter.inc({ errorCode });
+    const message = error.message || error;
+    console.error(`[${errorCode}] ${message}`, ...optionalParams);
+    if (error.stack)
+        console.error(error.stack);
+};
 /**
  * @param {import('node:http').ServerResponse} res
  * @param {any} err
  */
 const httpErrorHandler = (res, err) => {
-  console.error('failed to server prometheus /metrics', err)
-  res.writeHead(500)
-  res.end()
-}
+    console.error('failed to server prometheus /metrics', err);
+    res.writeHead(500);
+    res.end();
+};
 const serveRegistry = async (res, registry) => {
-  const metrics = await registry.metrics()
-  res.setHeader('Content-Type', registry.contentType)
-  res.writeHead(200)
-  res.write(metrics)
-  res.end()
-}
+    const metrics = await registry.metrics();
+    res.setHeader('Content-Type', registry.contentType);
+    res.writeHead(200);
+    res.write(metrics);
+    res.end();
+};
 // live CPU performance inspection
 const serveCPUProfile = async (res, duration) => {
-  const { Session } = await import('node:inspector/promises')
-  const session = new Session()
-  session.connect()
-  await session.post('Profiler.enable')
-  await session.post('Profiler.start')
-  await new Promise(resolve => setTimeout(resolve, duration))
-  const { profile } = await session.post('Profiler.stop')
-  await session.post('Profiler.disable')
-  session.disconnect()
-  res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.cpuprofile"`)
-  res.write(JSON.stringify(profile))
-  res.end()
-}
+    const { Session } = await import('node:inspector/promises');
+    const session = new Session();
+    session.connect();
+    await session.post('Profiler.enable');
+    await session.post('Profiler.start');
+    await new Promise(resolve => setTimeout(resolve, duration));
+    const { profile } = await session.post('Profiler.stop');
+    await session.post('Profiler.disable');
+    session.disconnect();
+    res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.cpuprofile"`);
+    res.write(JSON.stringify(profile));
+    res.end();
+};
 // live heap allocation sampler
 const serveHeapSnapshot = async (res, includeCollected, duration) => {
-  const { Session } = await import('node:inspector/promises')
-  const session = new Session()
-  session.connect()
-  await session.post('HeapProfiler.enable')
-  await session.post('HeapProfiler.startSampling', { includeObjectsCollectedByMajorGC: includeCollected, includeObjectsCollectedByMinorGC: true })
-  await new Promise(resolve => setTimeout(resolve, duration))
-  const { profile } = await session.post('HeapProfiler.stopSampling')
-  await session.post('HeapProfiler.disable')
-  session.disconnect()
-  res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.heapprofile"`)
-  res.write(JSON.stringify(profile))
-  res.end()
-}
-let server
+    const { Session } = await import('node:inspector/promises');
+    const session = new Session();
+    session.connect();
+    await session.post('HeapProfiler.enable');
+    await session.post('HeapProfiler.startSampling', { includeObjectsCollectedByMajorGC: includeCollected, includeObjectsCollectedByMinorGC: true });
+    await new Promise(resolve => setTimeout(resolve, duration));
+    const { profile } = await session.post('HeapProfiler.stopSampling');
+    await session.post('HeapProfiler.disable');
+    session.disconnect();
+    res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.heapprofile"`);
+    res.write(JSON.stringify(profile));
+    res.end();
+};
+let server;
 export const startObserver = async (port = 9090) => {
-  server = createServer((req, res) => {
-    const url = new URL(req.url || '', `http://${req.headers.host}`)
-    if (req.method === 'GET' && url.pathname === '/metrics') {
-      serveRegistry(res, register).catch(err => httpErrorHandler(res, err))
-    } else if (req.method === 'GET' && (url.pathname === '/global-metrics' || url.pathname === '/service-metrics')) {
-      serveRegistry(res, servicePromRegistry).catch(err => httpErrorHandler(res, err))
-    } else if (req.method === 'GET' && url.pathname === '/cpu-profile') {
-      const duration = parseInt(url.searchParams.get('duration') || '2000')
-      serveCPUProfile(res, duration).catch(err => httpErrorHandler(res, err))
-    } else if (req.method === 'GET' && url.pathname === '/heap-profile') {
-      const duration = parseInt(url.searchParams.get('duration') || '2000')
-      serveHeapSnapshot(res, false, duration).catch(err => httpErrorHandler(res, err))
-    } else if (req.method === 'GET' && url.pathname === '/heap-profile-all') {
-      const duration = parseInt(url.searchParams.get('duration') || '2000')
-      serveHeapSnapshot(res, true, duration).catch(err => httpErrorHandler(res, err))
-    } else {
-      res.writeHead(404)
-      res.end()
-    }
-  })
-  server.listen(port)
-  await eventPromise(server, 'listening')
-  console.log(`Observer server available on http://localhost:${port}
+    server = createServer((req, res) => {
+        const url = new URL(req.url || '', `http://${req.headers.host}`);
+        if (req.method === 'GET' && url.pathname === '/metrics') {
+            serveRegistry(res, register).catch(err => httpErrorHandler(res, err));
+        }
+        else if (req.method === 'GET' && (url.pathname === '/global-metrics' || url.pathname === '/service-metrics')) {
+            serveRegistry(res, servicePromRegistry).catch(err => httpErrorHandler(res, err));
+        }
+        else if (req.method === 'GET' && url.pathname === '/cpu-profile') {
+            const duration = parseInt(url.searchParams.get('duration') || '2000');
+            serveCPUProfile(res, duration).catch(err => httpErrorHandler(res, err));
+        }
+        else if (req.method === 'GET' && url.pathname === '/heap-profile') {
+            const duration = parseInt(url.searchParams.get('duration') || '2000');
+            serveHeapSnapshot(res, false, duration).catch(err => httpErrorHandler(res, err));
+        }
+        else if (req.method === 'GET' && url.pathname === '/heap-profile-all') {
+            const duration = parseInt(url.searchParams.get('duration') || '2000');
+            serveHeapSnapshot(res, true, duration).catch(err => httpErrorHandler(res, err));
+        }
+        else {
+            res.writeHead(404);
+            res.end();
+        }
+    });
+    server.listen(port);
+    await eventPromise(server, 'listening');
+    console.log(`Observer server available on http://localhost:${port}
     GET /metrics -> get prometheus metrics for this instance
     GET /service-metrics -> get prometheus metrics shared accross all instances
     GET /cpu-profile -> generate and fetch a CPU profile
     GET /heap-profile -> generate and fetch a sampled heap allocation snapshot
     GET /heap-profile-all -> generate and fetch a sampled heap allocation snapshot including objects collected by GC
-`)
-}
+`);
+};
 export const stopObserver = async () => {
-  if (server) { server.close() }
-}
+    if (server)
+        server.close();
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/lib-node",
-  "version": "2.8.2",
+  "version": "2.9.1",
   "description": "Small nodejs related utility functions for the data-fair stack.",
   "main": "index.js",
   "files": [

package/session.d.ts

@@ -0,0 +1,10 @@
+import type { SessionState } from '@data-fair/lib-common-types/session/index.js';
+import JwksClient from 'jwks-rsa';
+export declare class SessionHandler {
+    private _jwksClient;
+    defaultLang: string;
+    get jwksClient(): JwksClient.JwksClient;
+    initJWKS(directoryUrl?: string): void;
+    verifyToken(token: string): Promise<any>;
+    readStateFromCookie(cookieStr: string | undefined, onlyDecode?: boolean, clearBrokenSession?: () => void): Promise<SessionState>;
+}

package/session.js

@@ -0,0 +1,110 @@
+import jwt from 'jsonwebtoken';
+import JwksClient from 'jwks-rsa';
+import cookie from 'cookie';
+import { httpError } from '@data-fair/lib-utils/http-errors.js';
+export class SessionHandler {
+    _jwksClient;
+    defaultLang = 'fr';
+    get jwksClient() {
+        if (!this._jwksClient)
+            throw new Error('session management was not initialized');
+        return this._jwksClient;
+    }
+    initJWKS(directoryUrl = 'http://simple-directory:8080') {
+        this._jwksClient = JwksClient({ jwksUri: directoryUrl + '/.well-known/jwks.json' });
+    }
+    // used for the main session token, but can also be used to use the jwks to verify other types of tokens
+    async verifyToken(token) {
+        const decoded = jwt.decode(token, { complete: true });
+        if (!decoded)
+            throw new Error('failed to decode token');
+        const signingKey = await this.jwksClient.getSigningKey(decoded.header.kid);
+        return jwt.verify(token, signingKey.getPublicKey());
+    }
+    async readStateFromCookie(cookieStr, onlyDecode, clearBrokenSession) {
+        const session = { lang: this.defaultLang };
+        if (!cookieStr)
+            return session;
+        const cookies = cookie.parse(cookieStr);
+        session.dark = cookies.theme_dark === '1' || cookies.theme_dark === 'true';
+        if (cookies.i18n_lang)
+            session.lang = cookies.i18n_lang;
+        if (!cookies.id_token || !cookies.id_token_sign)
+            return session;
+        const token = cookies.id_token + '.' + cookies.id_token_sign;
+        let user;
+        if (onlyDecode) {
+            user = jwt.decode(token);
+        }
+        else {
+            try {
+                user = await this.verifyToken(token);
+            }
+            catch (err) {
+                console.warn(err);
+                if (err.name === 'JwksError') {
+                    // happens in case of temporary unavailability if SD
+                    // better not to disconnect user in this case
+                    throw httpError(500, 'Session token public keys not initialized');
+                }
+                else {
+                    if (clearBrokenSession)
+                        clearBrokenSession();
+                    throw httpError(401);
+                }
+            }
+        }
+        if (!user)
+            return session;
+        // this is to prevent null values that are put by SD versions that do not strictly respect their schema
+        for (const org of user.organizations) {
+            if (!org.department) {
+                delete org.department;
+                delete org.departmentName;
+            }
+        }
+        session.user = user;
+        const organizationId = cookies.id_token_org;
+        const departmentId = cookies.id_token_dep;
+        const switchedRole = cookies.id_token_role;
+        if (organizationId) {
+            session.organization = user.organizations.find(o => {
+                if (o.id !== organizationId)
+                    return false;
+                if (departmentId && departmentId !== o.department)
+                    return false;
+                if (switchedRole && switchedRole !== o.role)
+                    return false;
+                return true;
+            });
+        }
+        if (session.organization) {
+            session.account = {
+                type: 'organization',
+                id: session.organization.id,
+                name: session.organization.name,
+                department: session.organization.department,
+                departmentName: session.organization.departmentName
+            };
+            session.accountRole = session.organization.role;
+        }
+        else {
+            session.account = {
+                type: 'user',
+                id: user.id,
+                name: user.name
+            };
+            session.accountRole = 'admin';
+            return session;
+        }
+        if (session.user?.siteOwner) {
+            if (session.user.siteOwner.type === 'user' && session.user.siteOwner.id === session.user.id) {
+                session.siteRole = 'admin';
+            }
+            if (session.user.siteOwner.type === 'organization' && session.user.siteOwner.id === session.organization?.id) {
+                session.siteRole = session.organization.role;
+            }
+        }
+        return session;
+    }
+}

package/test-module-hooks.js

@@ -1,16 +1,16 @@
 // @ts-nocheck
 // cf https://nodejs.org/api/module.html#customization-hooks
-const regexp = /\/\/ @test:spy\((.*)\)\n/g
-export async function load (url, context, nextLoad) {
-  const result = await nextLoad(url, context)
-  if (result.format === 'module' || result.format === 'module-typescript') {
-    let source = result.source.toString()
-    const matches = source.match(regexp)
-    if (matches) {
-      source = 'import { emit as __emitTestSpy } from "@data-fair/lib-node/test-spies.js"\n' + source
-      source = source.replace(regexp, '__emitTestSpy($1)\n')
-      result.source = Buffer.from(source, 'utf-8')
+const regexp = /\/\/ @test:spy\((.*)\)\n/g;
+export async function load(url, context, nextLoad) {
+    const result = await nextLoad(url, context);
+    if (result.format === 'module' || result.format === 'module-typescript') {
+        let source = result.source.toString();
+        const matches = source.match(regexp);
+        if (matches) {
+            source = 'import { emit as __emitTestSpy } from "@data-fair/lib-node/test-spies.js"\n' + source;
+            source = source.replace(regexp, '__emitTestSpy($1)\n');
+            result.source = Buffer.from(source, 'utf-8');
+        }
     }
-  }
-  return result
+    return result;
 }