@data-fair/lib-node 0.1.0

package/axios-auth.d.ts

@@ -0,0 +1,12 @@
+import type { AxiosInstance } from 'axios';
+export interface AxiosAuthOptions {
+    email: string;
+    password: string;
+    org?: string;
+    dep?: string;
+    adminMode?: boolean;
+    directoryUrl?: string;
+    axiosOpts?: any;
+}
+export declare function axiosAuth(opts: AxiosAuthOptions): Promise<AxiosInstance>;
+export default axiosAuth;

package/axios-auth.js

@@ -0,0 +1,34 @@
+// build axios instances with sessions on a simple-directory instance
+// used for integration testing
+import { Agent } from 'node:http';
+import { axiosBuilder, axiosInstance } from './axios.js';
+export async function axiosAuth(opts) {
+    const body = { email: opts.email, password: opts.password };
+    if (opts.org)
+        body.org = opts.org;
+    if (opts.dep)
+        body.dep = opts.dep;
+    if (opts.adminMode)
+        body.adminMode = opts.adminMode;
+    const axiosOpts = {
+        httpAgent: new Agent({ keepAlive: false }),
+        ...opts.axiosOpts
+    };
+    const directoryUrl = opts.directoryUrl ?? 'http://localhost:8080';
+    let callbackUrl = (await axiosInstance.post(directoryUrl + '/api/auth/password', body, { params: { redirect: directoryUrl }, maxRedirects: 0 })).data;
+    if (callbackUrl.startsWith(directoryUrl + '/simple-directory')) {
+        callbackUrl = callbackUrl.replace(directoryUrl + '/simple-directory', directoryUrl);
+    }
+    try {
+        await axiosInstance.get(callbackUrl, { maxRedirects: 0 });
+    }
+    catch (err) {
+        if (err.status !== 302)
+            throw err;
+        axiosOpts.headers = axiosOpts.headers ?? {};
+        axiosOpts.headers.common = axiosOpts.headers.common ?? {};
+        axiosOpts.headers.common.Cookie = err.headers['set-cookie'].map((s) => s.split(';')[0]).join(';');
+    }
+    return axiosBuilder(axiosOpts);
+}
+export default axiosAuth;

package/axios.d.ts

@@ -0,0 +1,4 @@
+import type { AxiosInstance } from 'axios';
+export declare function axiosBuilder(opts?: object): AxiosInstance;
+export declare const axiosInstance: AxiosInstance;
+export default axiosInstance;

package/axios.js

@@ -0,0 +1,53 @@
+// prepare an axios instance with improved error management and some performance tuning for nodejs usage
+import axios from 'axios';
+import { httpAgent, httpsAgent } from './http-agents.js';
+const { stackTraceLimit } = Error;
+class AxiosRequestError extends Error {
+    constructor() {
+        super();
+        this.name = 'AxiosRequestError';
+    }
+}
+const requestInterceptor = (config) => {
+    // better stack traces
+    // see https://github.com/axios/axios/issues/2387#issuecomment-652242713
+    Error.stackTraceLimit = 0;
+    const errorContext = new AxiosRequestError();
+    Error.stackTraceLimit = stackTraceLimit;
+    Error.captureStackTrace(errorContext, requestInterceptor);
+    // @ts-ignore
+    config.errorContext = errorContext;
+    return config;
+};
+export function axiosBuilder(opts = {}) {
+    const ax = axios.create({
+        httpAgent,
+        httpsAgent,
+        ...opts
+    });
+    ax.interceptors.request.use(requestInterceptor);
+    // shorter stack traces
+    ax.interceptors.response.use(response => response, error => {
+        if (!error.response) {
+            if (error.config?.errorContext?.stack)
+                error.stack += '\nRequest context:\n' + error.config?.errorContext?.stack;
+            return Promise.reject(error);
+        }
+        delete error.response.request;
+        error.response.config = { method: error.response.config.method, url: error.response.config.url, data: error.response.config.data };
+        if (error.response.config.data && error.response.config.data._writableState)
+            delete error.response.config.data;
+        if (error.response.data && error.response.data._readableState)
+            delete error.response.data;
+        let messageText = error.response.statusText;
+        if (error.response.data) {
+            messageText = typeof error.response.data === 'string' ? error.response.data : JSON.stringify(error.response.data);
+        }
+        error.response.message = `${error.response.status} - ${messageText}`;
+        Object.assign(error.config.errorContext, error.response);
+        return Promise.reject(error.config.errorContext);
+    });
+    return ax;
+}
+export const axiosInstance = axiosBuilder();
+export default axiosInstance;

package/http-agents.d.ts

@@ -0,0 +1,3 @@
+import HttpAgent from 'agentkeepalive';
+export declare const httpAgent: HttpAgent;
+export declare const httpsAgent: HttpAgent.HttpsAgent;

package/http-agents.js

@@ -0,0 +1,55 @@
+// use better DNS lookup than nodejs default and try to reduce number of socket openings
+import CacheableLookup from 'cacheable-lookup';
+import HttpAgent, { HttpsAgent } from 'agentkeepalive';
+import { Counter, Gauge } from 'prom-client';
+// not very high number of sockets but we don't want to saturate our reverse proxies
+// if a higher number is needed the service should probably be scaled anyway
+const keepaliveOpts = {
+    maxSockets: 8,
+    maxFreeSockets: 8,
+    timeout: 60000,
+    freeSocketTimeout: 30000
+};
+export const httpAgent = new HttpAgent(keepaliveOpts);
+export const httpsAgent = new HttpsAgent(keepaliveOpts);
+const cacheableLookup = new CacheableLookup();
+cacheableLookup.install(httpAgent);
+cacheableLookup.install(httpsAgent);
+// monitor agent statuses
+const agents = { http: httpAgent, https: httpsAgent };
+const socketsGauge = new Gauge({
+    name: 'df_http_agent_sockets',
+    help: 'Number of open sockets in the http agent',
+    labelNames: ['protocol', 'host']
+});
+const freeSocketsGauge = new Gauge({
+    name: 'df_http_agent_free_sockets',
+    help: 'Number of free sockets in the http agent',
+    labelNames: ['protocol', 'host']
+});
+const createSocketCounter = new Counter({
+    name: 'df_http_agent_create_socket_total',
+    help: 'Total number of sockets created by the http agent',
+    labelNames: ['protocol']
+});
+const requestCounter = new Counter({
+    name: 'df_http_agent_request_total',
+    help: 'Total number of requests created by the http agent',
+    labelNames: ['protocol']
+});
+setInterval(() => {
+    for (const _protocol of ['http', 'https']) {
+        const protocol = _protocol;
+        const agentStatus = agents[protocol].getCurrentStatus();
+        for (const host in agentStatus.sockets) {
+            socketsGauge.set({ protocol, host }, agentStatus.sockets[host]);
+        }
+        for (const host in agentStatus.freeSockets) {
+            freeSocketsGauge.set({ protocol, host }, agentStatus.freeSockets[host]);
+        }
+        createSocketCounter.remove({ protocol });
+        createSocketCounter.inc({ protocol }, agentStatus.createSocketCount);
+        requestCounter.remove({ protocol });
+        requestCounter.inc({ protocol }, agentStatus.requestCount);
+    }
+}, 60000);

package/locks.d.ts

@@ -0,0 +1,5 @@
+import type { Db } from 'mongodb';
+export declare const init: (db: Db, ttl?: number) => Promise<void>;
+export declare const stop: () => Promise<void>;
+export declare function acquire(_id: string, origin?: string): Promise<boolean>;
+export declare const release: (_id: string, delay?: number) => Promise<void>;

package/locks.js

@@ -0,0 +1,72 @@
+import { hostname } from 'node:os';
+import { randomBytes } from 'node:crypto';
+import Debug from 'debug';
+const debug = Debug('locks');
+const pid = randomBytes(8).toString('hex');
+debug('locks with pid', pid);
+let interval;
+let _db;
+const collection = () => {
+    if (!_db)
+        throw new Error('locks utils was not initialized');
+    return _db?.collection('locks');
+};
+export const init = async (db, ttl = 60) => {
+    _db = db;
+    const locks = collection();
+    await locks.createIndex({ pid: 1 });
+    try {
+        await locks.createIndex({ updatedAt: 1 }, { expireAfterSeconds: ttl });
+    }
+    catch (err) {
+        console.log('Failure to create TTL index. Probably because the value changed. Try to update it.');
+        db.command({ collMod: 'locks', index: { keyPattern: { updatedAt: 1 }, expireAfterSeconds: ttl } });
+    }
+    // prolongate lock acquired by this process while it is still active
+    interval = setInterval(() => {
+        locks.updateMany({ pid }, { $currentDate: { updatedAt: true } });
+    }, (ttl / 2) * 1000);
+};
+export const stop = async () => {
+    clearInterval(interval);
+    if (_db)
+        await collection().deleteMany({ pid });
+};
+export async function acquire(_id, origin) {
+    if (!_db)
+        throw new Error('locks not initialized');
+    debug('acquire', _id, origin);
+    const locks = collection();
+    try {
+        await locks.insertOne({ _id, pid, origin, hostname: hostname(), createdAt: new Date() });
+        try {
+            await locks.updateOne({ _id }, { $currentDate: { updatedAt: true } });
+        }
+        catch (err) {
+            await locks.deleteOne({ _id, pid });
+            throw err;
+        }
+        debug('acquire ok', _id);
+        return true;
+    }
+    catch (err) {
+        if (err.code !== 11000)
+            throw err;
+        // duplicate means the lock was already acquired
+        debug('acquire ko', _id);
+        return false;
+    }
+}
+export const release = async (_id, delay = 0) => {
+    if (!_db)
+        throw new Error('locks not initialized');
+    debug('release', _id);
+    const locks = collection();
+    if (delay) {
+        const date = new Date((new Date()).getTime() + delay);
+        await locks.updateOne({ _id, pid }, { $unset: { pid: 1 }, $set: { delayed: true, updatedAt: date } });
+    }
+    else {
+        await locks.deleteOne({ _id, pid });
+    }
+};

package/mongo-types.d.ts

@@ -0,0 +1 @@
+export {};

package/mongo-types.js

@@ -0,0 +1 @@
+export {};

package/mongo.d.ts

@@ -0,0 +1,17 @@
+import type { IndexSpecification, CreateIndexesOptions, MongoClientOptions } from 'mongodb';
+import { MongoClient } from 'mongodb';
+export type IndexDefinitions = Record<string, Record<string, IndexSpecification | [IndexSpecification, CreateIndexesOptions]>>;
+export declare class Mongo {
+    private _client;
+    get client(): MongoClient;
+    get db(): import("mongodb").Db;
+    connect: (mongoUrl: string, options?: MongoClientOptions) => Promise<void>;
+    ensureIndex: (collection: string, key: IndexSpecification, options?: CreateIndexesOptions) => Promise<void>;
+    /**
+     * create a bunch of indexes and overtwrite previous definitions
+     * the structure forces giving names to indexes which is better for managing them properly
+     */
+    configure: (indexDefinitions: IndexDefinitions) => Promise<void>;
+}
+declare const mongo: Mongo;
+export default mongo;

package/mongo.js

@@ -0,0 +1,63 @@
+import { MongoClient, MongoError } from 'mongodb';
+export class Mongo {
+    _client;
+    get client() {
+        if (!this._client)
+            throw new Error('db was not connected');
+        return this._client;
+    }
+    get db() {
+        return this.client.db();
+    }
+    connect = async (mongoUrl, options = {}) => {
+        if (this._client) {
+            console.warn('db already connected');
+            return;
+        }
+        console.log('connecting to mongodb...');
+        this._client = await MongoClient.connect(mongoUrl, { maxPoolSize: 5, ignoreUndefined: true, ...options });
+        console.log('...ok');
+    };
+    // Create an index if it does not exist and manage overwriting existing and conflicting index definitions
+    ensureIndex = async (collection, key, options = {}) => {
+        try {
+            await this.db.collection(collection).createIndex(key, options);
+        }
+        catch (err) {
+            if (options.name && err instanceof MongoError && (err.code === 85 || err.code === 86)) {
+                // if the error is a conflict on keys or params of the index we automatically
+                // delete then recreate the index
+                console.log(`drop then recreate index ${collection}/${options.name}`);
+                await this.db.collection(collection).dropIndex(options.name);
+                await this.db.collection(collection).createIndex(key, options);
+            }
+            else {
+                throw err;
+            }
+        }
+    };
+    /**
+     * create a bunch of indexes and overtwrite previous definitions
+     * the structure forces giving names to indexes which is better for managing them properly
+     */
+    configure = async (indexDefinitions) => {
+        for (const collectionName in indexDefinitions) {
+            for (const indexName in indexDefinitions[collectionName]) {
+                let key, options;
+                if (Array.isArray(indexDefinitions[collectionName][indexName])) {
+                    [key, options] = indexDefinitions[collectionName][indexName];
+                    if (options.name && options.name !== indexName)
+                        throw new Error(`inconsistent index name ${indexName} or ${options.name}`);
+                    options.name = indexName;
+                }
+                else {
+                    key = indexDefinitions[collectionName][indexName];
+                    options = { name: indexName };
+                }
+                await this.ensureIndex(collectionName, key, options);
+            }
+        }
+    };
+}
+const mongo = new Mongo();
+export default mongo;

package/observer.d.ts

@@ -0,0 +1,5 @@
+import { Registry } from 'prom-client';
+export declare const servicePromRegistry: Registry<"text/plain; version=0.0.4; charset=utf-8">;
+export declare const internalError: (errorCode: string, error: any, ...optionalParams: any) => void;
+export declare const startObserver: (port?: number) => Promise<void>;
+export declare const stopObserver: () => Promise<void>;

package/observer.js

@@ -0,0 +1,83 @@
+// very light web server to expose prometheus metrics, export a cpu profile and maybe other incoming obervability features
+import { hostname } from 'node:os';
+import { createServer } from 'node:http';
+import { Registry, Counter, register, collectDefaultMetrics } from 'prom-client';
+import eventPromise from '@data-fair/lib-utils/event-promise.js';
+// a special registry for metrics that are global
+// not based on this specific process state but instead on the content of db for example
+export const servicePromRegistry = new Registry();
+collectDefaultMetrics();
+// local metrics incremented throughout the code
+const internalErrorCounter = new Counter({
+    name: 'df_internal_error',
+    help: 'A counter of errors from any service, worker, etc. Do not use for client errors, only for anomalies that should trigger alerts. Each increment should be accompanied by an error log with matching code.',
+    labelNames: ['errorCode']
+});
+export const internalError = (errorCode, error, ...optionalParams) => {
+    internalErrorCounter.inc({ errorCode });
+    const message = error.message || error;
+    console.error(`[${errorCode}] ${message}`, ...optionalParams);
+    if (error.stack)
+        console.error(error.stack);
+};
+/**
+ * @param {import('node:http').ServerResponse} res
+ * @param {any} err
+ */
+const httpErrorHandler = (res, err) => {
+    console.error('failed to server prometheus /metrics', err);
+    res.writeHead(500);
+    res.end();
+};
+const serveRegistry = async (res, registry) => {
+    const metrics = await registry.metrics();
+    res.setHeader('Content-Type', registry.contentType);
+    res.writeHead(200);
+    res.write(metrics);
+    res.end();
+};
+// live CPU performance inspection
+const serveCPUProfile = async (res, duration = 2000) => {
+    // @ts-ignore
+    const { Session } = await import('node:inspector/promises');
+    const session = new Session();
+    session.connect();
+    await session.post('Profiler.enable');
+    await session.post('Profiler.start');
+    await new Promise(resolve => setTimeout(resolve, duration));
+    const { profile } = await session.post('Profiler.stop');
+    session.disconnect();
+    res.setHeader('Content-Disposition', `attachment; filename="data-fair-${hostname()}-${new Date().toISOString()}.cpuprofile"`);
+    res.write(JSON.stringify(profile));
+    res.end();
+};
+let server;
+export const startObserver = async (port = 9090) => {
+    server = createServer((req, res) => {
+        if (req.method === 'GET' && req.url === '/metrics') {
+            serveRegistry(res, register).catch(err => httpErrorHandler(res, err));
+        }
+        else if (req.method === 'GET' && (req.url === '/global-metrics' || req.url === '/service-metrics')) {
+            serveRegistry(res, servicePromRegistry).catch(err => httpErrorHandler(res, err));
+        }
+        else if (req.method === 'GET' && req.url === '/cpu-profile') {
+            // TODO: duration
+            serveCPUProfile(res).catch(err => httpErrorHandler(res, err));
+        }
+        else {
+            res.writeHead(404);
+            res.end();
+        }
+    });
+    server.listen(port);
+    await eventPromise(server, 'listening');
+    console.log(`Observer server available on http://localhost:${port}
+    GET /metrics -> get prometheus metrics for this instance
+    GET /service-metrics -> get prometheus metrics shared accross all instances
+    GET /cpu-profile -> generate and fetch a CPU profile
+`);
+};
+export const stopObserver = async () => {
+    if (server)
+        server.close();
+};

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@data-fair/lib-node",
+  "version": "0.1.0",
+  "description": "Small nodejs related utility functions for the data-fair stack.",
+  "main": "index.js",
+  "files": [
+    "**/*.js",
+    "**/*.d.ts"
+  ],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@data-fair/lib-utils": "0.2.0",
+    "agentkeepalive": "^4.5.0",
+    "ajv-i18n": "^4.2.0",
+    "axios": "^1.7.7",
+    "cacheable-lookup": "^7.0.0",
+    "semver": "^7.6.3"
+  },
+  "peerDependencies": {
+    "mongodb": "^6.9.0",
+    "prom-client": "^15.1.3",
+    "ws": "^8.18.0"
+  },
+  "peerDependenciesMeta": {
+    "mongodb": {
+      "optional": true
+    },
+    "prom-client": {
+      "optional": true
+    },
+    "ws": {
+      "optional": true
+    }
+  },
+  "type": "module"
+}

package/test-module-hooks.d.ts

@@ -0,0 +1 @@
+export declare function load(url: any, context: any, nextLoad: any): Promise<any>;

package/test-module-hooks.js

@@ -0,0 +1,16 @@
+// @ts-nocheck
+// cf https://nodejs.org/api/module.html#customization-hooks
+const regexp = /\/\/ @test:spy\((.*)\)\n/g;
+export async function load(url, context, nextLoad) {
+    const result = await nextLoad(url, context);
+    if (result.format === 'module' || result.format === 'module-typescript') {
+        let source = result.source.toString();
+        const matches = source.match(regexp);
+        if (matches) {
+            source = 'import { emit as __emitTestSpy } from "@data-fair/lib/node/test-spies.js"\n' + source;
+            source = source.replace(regexp, '__emitTestSpy($1)\n');
+            result.source = Buffer.from(source, 'utf-8');
+        }
+    }
+    return result;
+}

package/test-spies.d.ts

@@ -0,0 +1,3 @@
+export declare function registerModuleHooks(): void;
+export declare function waitFor(eventName: string, timeout?: number): Promise<unknown>;
+export declare function emit(eventName: string, data: any): void;

package/test-spies.js

@@ -0,0 +1,28 @@
+import { register } from 'node:module';
+export function registerModuleHooks() {
+    register('@data-fair/lib/node/test-module-hooks.js', { parentURL: import.meta.url });
+}
+const pendingPromises = {};
+let i = 0;
+export function waitFor(eventName, timeout = 2000) {
+    i++;
+    const id = i;
+    const p = new Promise((resolve, reject) => {
+        pendingPromises[eventName] = pendingPromises[eventName] ?? [];
+        pendingPromises[eventName].push({ resolve, id });
+        setTimeout(() => reject(new Error(`Timeout waiting for test spy ${eventName}`)), timeout);
+    });
+    p.finally(() => {
+        pendingPromises[eventName] = pendingPromises[eventName].filter(p => p.id !== id);
+        if (pendingPromises[eventName].length === 0)
+            delete pendingPromises[eventName];
+    });
+    return p;
+}
+export function emit(eventName, data) {
+    if (!pendingPromises[eventName])
+        return;
+    for (const p of pendingPromises[eventName]) {
+        p.resolve(data);
+    }
+}

package/upgrade-scripts.d.ts

@@ -0,0 +1,7 @@
+import type { Db } from 'mongodb';
+import type { Debugger } from 'debug';
+export interface UpgradeScript {
+    description: string;
+    exec: (db: Db, debug: Debugger) => Promise<void>;
+}
+export default function (db: Db, basePath?: string): Promise<void>;

package/upgrade-scripts.js

@@ -0,0 +1,68 @@
+import semver from 'semver';
+import path from 'path';
+import { readdirSync, readFileSync } from 'fs';
+import Debug from 'debug';
+import { acquire, release } from './locks.js';
+const debug = Debug('upgrade');
+// chose the proper scripts to execute, then run them
+export default async function (db, basePath = './') {
+    const ack = await acquire('upgrade');
+    if (!ack) {
+        console.warn('upgrade scripts lock is already acquired, skip them');
+        // IMPORTANT: this behaviour of running the process when the upgrade scripts are still running on another one
+        // implies that they cannot be considered a pre-requisite..
+        // if we want to consider the upgrade scripts as a pre-requisite we should implement a wait on all
+        // containers for the scripts that are running in only 1 (while loop on "acquire" ?) and a healthcheck
+        // are not considered "up" and the previous versions keep running in the mean time
+    }
+    else {
+        await runScripts(db, basePath);
+        await release('upgrade');
+    }
+}
+async function runScripts(db, basePath) {
+    const dir = path.resolve(basePath);
+    const pjsonPath = path.join(dir, 'package.json');
+    debug('read service info from ' + pjsonPath);
+    const pjson = JSON.parse(readFileSync(pjsonPath, 'utf8'));
+    const scriptsRoot = path.join(dir, 'upgrade');
+    debug(`service=${pjson.name} version=${pjson.version}`);
+    const services = db.collection('services');
+    const service = await services.findOne({ id: pjson.name });
+    const version = service?.version;
+    debug('list scripts from ' + scriptsRoot);
+    let scripts = await listScripts(scriptsRoot);
+    if (!version) {
+        debug('No service version found in database, this is probably a fresh install');
+        scripts = scripts.filter(scriptDef => scriptDef.version === 'init');
+    }
+    else {
+        debug(`Current service version from database : ${version}`);
+        scripts = scripts.filter(scriptDef => scriptDef.version !== 'init');
+    }
+    for (const scriptDef of scripts) {
+        if (semver.gte(scriptDef.version, version)) {
+            for (const scriptName of scriptDef.names) {
+                const script = (await import(path.join(scriptsRoot, scriptDef.version, scriptName))).default;
+                debug('Apply script %s/%s : %s', scriptDef.version, scriptName, script.description);
+                await script.exec(db, Debug(`upgrade:${scriptDef.version}:${scriptName}`));
+            }
+        }
+    }
+    const coercedVersion = semver.coerce(pjson.version);
+    if (!coercedVersion)
+        throw new Error(`Invalid version number in package.json ${pjson.version}`);
+    const newService = { id: pjson.name, version: coercedVersion.version };
+    debug(`Upgrade scripts are over, save current version number ${newService.version}`);
+    await services.updateOne({ id: pjson.name }, { $set: newService }, { upsert: true });
+}
+// Walk the scripts directories
+async function listScripts(scriptsRoot) {
+    const dirs = readdirSync(scriptsRoot).sort();
+    return dirs.map(dir => {
+        return {
+            version: dir,
+            names: readdirSync(path.join(scriptsRoot, dir)).sort()
+        };
+    });
+}

package/validation.d.ts

@@ -0,0 +1,8 @@
+import type { ErrorObject, ValidateFunction } from 'ajv';
+export declare const errorsText: (errors: ErrorObject[] | null | undefined, varName?: string) => string;
+export type AssertValidOptions = {
+    lang?: string;
+    name?: string;
+    internal?: boolean;
+};
+export declare function assertValid<T>(validate: ValidateFunction, data: any, options?: AssertValidOptions): asserts data is T;

package/validation.js

@@ -0,0 +1,44 @@
+import ajvFrModule from 'ajv-i18n/localize/fr/index.js';
+import ajvEnModule from 'ajv-i18n/localize/en/index.js';
+import { httpError } from '@data-fair/lib-utils/http-errors.js';
+const ajvFr = ajvFrModule;
+const ajvEn = ajvEnModule;
+const localize = {
+    fr: ajvFr,
+    en: ajvEn
+};
+// cf https://github.com/ajv-validator/ajv/blob/b3e0cb17d0e095b5c883042b2306571be5ec86b7/lib/core.ts#L650
+export const errorsText = (errors, varName = 'data') => {
+    if (!errors || errors.length === 0)
+        return 'No errors';
+    return errors
+        .map((e) => {
+        let msg = `${varName}${e.instancePath} ${e.message}`.trim();
+        const paramKeys = Object.keys(e.params || {}).filter(key => {
+            if (key === 'error')
+                return false;
+            if (e.keyword === 'type' && key === 'type')
+                return false;
+            return true;
+        });
+        const params = paramKeys
+            .map(key => paramKeys.length > 1 ? `${key}=${e.params[key]}` : e.params[key])
+            .join(', ');
+        if (params)
+            msg += ` (${params})`;
+        return msg;
+    })
+        .reduce((text, msg) => text + ', ' + msg);
+};
+export function assertValid(validate, data, options = {}) {
+    const lang = options.lang ?? 'fr';
+    const name = options.name ?? 'data';
+    if (!validate(data)) {
+        (localize[lang] || localize.fr)(validate.errors);
+        const message = errorsText(validate.errors, name);
+        if (options.internal)
+            throw new Error(message);
+        else
+            throw httpError(400, message);
+    }
+}

package/ws-client.d.ts

@@ -0,0 +1,32 @@
+import { type Message } from '@data-fair/lib-common-types/ws.js';
+import { type Account } from '@data-fair/lib-common-types/session/index.js';
+import EventEmitter from 'node:events';
+type logFn = (msg: string, ...args: any[]) => void;
+export interface WsClientOpts {
+    log?: {
+        info: logFn;
+        error: logFn;
+        debug: logFn;
+    };
+    url: string;
+    headers?: Record<string, string>;
+    apiKey?: string;
+    adminMode?: boolean;
+    account?: Account;
+}
+export type FullWsClientOpts = WsClientOpts & Required<Pick<WsClientOpts, 'log'>>;
+export declare class WsClient extends EventEmitter {
+    private _channels;
+    private _ws;
+    opts: FullWsClientOpts;
+    constructor(opts: WsClientOpts);
+    private _connect;
+    private _reconnect;
+    subscribe(channel: string, force?: boolean, timeout?: number): Promise<void>;
+    waitFor(channel: string, filter?: (message: Message) => boolean, timeout?: number, skipSubscribe?: boolean, fullMessage?: boolean): Promise<Message>;
+    close(): void;
+}
+export declare class DataFairWsClient extends WsClient {
+    waitForJournal(datasetId: string, eventType: string, timeout?: number): Promise<Message>;
+}
+export {};

package/ws-client.js

@@ -0,0 +1,97 @@
+// a wrapper around a websocket connection with auto reconnect
+// JSON parsing and channel subscriptions
+import EventEmitter from 'node:events';
+import WebSocket from 'ws';
+import Debug from 'debug';
+const debug = Debug('ws');
+export class WsClient extends EventEmitter {
+    _channels;
+    _ws;
+    opts;
+    constructor(opts) {
+        super();
+        this._channels = [];
+        this.opts = { log: console, ...opts };
+    }
+    async _connect() {
+        return await new Promise((resolve, reject) => {
+            const wsUrl = this.opts.url.replace('http://', 'ws://').replace('https://', 'wss://') + '/';
+            debug(`connect Web Socket to ${wsUrl}`);
+            const ws = new WebSocket(wsUrl, { headers: this.opts.headers ?? {} });
+            this._ws = ws;
+            ws.on('error', (err) => {
+                debug('WS encountered an error', err.message);
+                this._reconnect();
+                reject(err);
+            });
+            ws.once('open', () => {
+                debug('WS is opened');
+                resolve(ws);
+            });
+            ws.on('message', (msg) => {
+                const message = JSON.parse(msg);
+                debug('received message', message);
+                this.emit('message', message);
+            });
+        });
+    }
+    async _reconnect() {
+        if (!this._ws)
+            return;
+        debug('reconnect');
+        this._ws.terminate();
+        await this._connect();
+        for (const channel of this._channels) {
+            await this.subscribe(channel, true);
+        }
+    }
+    async subscribe(channel, force = false, timeout = 2000) {
+        if (this._channels.includes(channel) && !force)
+            return;
+        const ws = this._ws ?? await this._connect();
+        debug('subscribe to channel', channel);
+        const subscribeMessage = { type: 'subscribe', channel };
+        if (this.opts.apiKey)
+            subscribeMessage.apiKey = this.opts.apiKey;
+        if (this.opts.adminMode && this.opts.account)
+            subscribeMessage.account = JSON.stringify(this.opts.account);
+        ws.send(JSON.stringify(subscribeMessage));
+        const event = await this.waitFor(channel, (e) => {
+            return e.type === 'subscribe-confirm' || e.type === 'error';
+        }, timeout, true, true);
+        if (event.type === 'error')
+            throw new Error(event.data);
+        if (this._channels.includes(channel))
+            this._channels.push(channel);
+    }
+    async waitFor(channel, filter, timeout = 300000, skipSubscribe = false, fullMessage = false) {
+        if (!skipSubscribe)
+            await this.subscribe(channel);
+        return await new Promise((resolve, reject) => {
+            const _timeout = setTimeout(() => { reject(new Error('timeout')); }, timeout);
+            const messageCb = (message) => {
+                if (message.channel === channel && (!filter || filter(fullMessage ? message : message.data))) {
+                    clearTimeout(_timeout);
+                    this.off('message', messageCb);
+                    resolve(fullMessage ? message : message.data);
+                }
+            };
+            this.on('message', messageCb);
+        });
+    }
+    close() {
+        if (this._ws)
+            this._ws.terminate();
+    }
+}
+export class DataFairWsClient extends WsClient {
+    async waitForJournal(datasetId, eventType, timeout = 300000) {
+        await this.opts.log.info(`wait for event "${eventType}" on dataset "${datasetId}"`);
+        const event = await this.waitFor(`datasets/${datasetId}/journal`, (e) => {
+            return e.type === eventType || e.type === 'error';
+        }, timeout);
+        if (event.type === 'error')
+            throw new Error(event.data);
+        return event;
+    }
+}

package/ws-emitter.d.ts

@@ -0,0 +1,4 @@
+import type { Db, Collection } from 'mongodb';
+export declare const initMessagesCollection: (db: Db) => Promise<Collection<import("bson").Document>>;
+export declare const init: (db: Db) => Promise<void>;
+export declare const emit: (channel: string, data: any) => Promise<void>;

package/ws-emitter.js

@@ -0,0 +1,18 @@
+// write messages to a mongodb collection/queue for publication by ws-server.js and consumption by ws.js
+// return a capped mongodb collection that acts as a simple queue
+export const initMessagesCollection = async (db) => {
+    const collection = (await db.listCollections({ name: 'ws-messages' }).toArray())[0];
+    if (!collection)
+        await db.createCollection('ws-messages', { capped: true, size: 100000, max: 1000 });
+    return db.collection('ws-messages');
+};
+let messagesCollection;
+export const init = async (db) => {
+    messagesCollection = await initMessagesCollection(db);
+    await messagesCollection.insertOne({ type: 'init' });
+};
+export const emit = async (channel, data) => {
+    if (!messagesCollection)
+        throw new Error('wsEmitter.init was not called before sending data');
+    await messagesCollection.insertOne({ type: 'message', channel, data, date: new Date().toISOString() });
+};