@data-fair/lib-node 2.8.1 → 2.9.0

package/test-spies.js

@@ -1,41 +1,42 @@
-import { register } from 'node:module'
-import debugModule from 'debug'
-const debug = debugModule('test-spies')
-export function registerModuleHooks () {
-  register('@data-fair/lib-node/test-module-hooks.js', { parentURL: import.meta.url })
+import { register } from 'node:module';
+import debugModule from 'debug';
+const debug = debugModule('test-spies');
+export function registerModuleHooks() {
+    register('@data-fair/lib-node/test-module-hooks.js', { parentURL: import.meta.url });
 }
-const { stackTraceLimit } = Error
-const pendingPromises = {}
-let i = 0
-export function waitFor (eventName, timeout = 2000) {
-  i++
-  const id = i
-  debug(`Waiting for test spy ${eventName} (id=${id}, timeout: ${timeout}ms)`)
-  // better stack traces
-  // see https://github.com/axios/axios/issues/2387#issuecomment-652242713
-  Error.stackTraceLimit = 0
-  const timeoutError = new Error(`Timeout waiting for test spy ${eventName}`)
-  Error.stackTraceLimit = stackTraceLimit
-  Error.captureStackTrace(timeoutError, waitFor)
-  const p = new Promise((resolve, reject) => {
-    pendingPromises[eventName] = pendingPromises[eventName] ?? []
-    pendingPromises[eventName].push({ resolve, id })
-    setTimeout(() => reject(timeoutError), timeout)
-  })
-  p.finally(() => {
-    debug(`Cleaning up test spy ${eventName} (id=${id})`)
-    pendingPromises[eventName] = pendingPromises[eventName].filter(p => p.id !== id)
-    if (pendingPromises[eventName].length === 0) { delete pendingPromises[eventName] }
-  })
-  return p
+const { stackTraceLimit } = Error;
+const pendingPromises = {};
+let i = 0;
+export function waitFor(eventName, timeout = 2000) {
+    i++;
+    const id = i;
+    debug(`Waiting for test spy ${eventName} (id=${id}, timeout: ${timeout}ms)`);
+    // better stack traces
+    // see https://github.com/axios/axios/issues/2387#issuecomment-652242713
+    Error.stackTraceLimit = 0;
+    const timeoutError = new Error(`Timeout waiting for test spy ${eventName}`);
+    Error.stackTraceLimit = stackTraceLimit;
+    Error.captureStackTrace(timeoutError, waitFor);
+    const p = new Promise((resolve, reject) => {
+        pendingPromises[eventName] = pendingPromises[eventName] ?? [];
+        pendingPromises[eventName].push({ resolve, id });
+        setTimeout(() => reject(timeoutError), timeout);
+    });
+    p.finally(() => {
+        debug(`Cleaning up test spy ${eventName} (id=${id})`);
+        pendingPromises[eventName] = pendingPromises[eventName].filter(p => p.id !== id);
+        if (pendingPromises[eventName].length === 0)
+            delete pendingPromises[eventName];
+    });
+    return p;
 }
-export function emit (eventName, data) {
-  debug(`Emitting test spy ${eventName} (${pendingPromises[eventName]?.length} listeners)`)
-  if (!pendingPromises[eventName]?.length) {
-    console.debug(`No listener for test spy ${eventName}`)
-    return
-  }
-  for (const p of pendingPromises[eventName]) {
-    p.resolve(data)
-  }
+export function emit(eventName, data) {
+    debug(`Emitting test spy ${eventName} (${pendingPromises[eventName]?.length} listeners)`);
+    if (!pendingPromises[eventName]?.length) {
+        console.debug(`No listener for test spy ${eventName}`);
+        return;
+    }
+    for (const p of pendingPromises[eventName]) {
+        p.resolve(data);
+    }
 }

package/upgrade-scripts.js

@@ -1,80 +1,86 @@
-import semver from 'semver'
-import path from 'path'
-import { readdirSync, readFileSync, statSync } from 'fs'
-import Debug from 'debug'
-const debug = Debug('upgrade')
+import semver from 'semver';
+import path from 'path';
+import { readdirSync, readFileSync, statSync } from 'fs';
+import Debug from 'debug';
+const debug = Debug('upgrade');
 // chose the proper scripts to execute, then run them
 export default async function (db, locks, basePath = './', isFresh) {
-  const ack = await locks.acquire('upgrade')
-  if (!ack) {
-    console.warn('upgrade scripts lock is already acquired, skip them')
-    // IMPORTANT: this behaviour of running the process when the upgrade scripts are still running on another one
-    // implies that they cannot be considered a pre-requisite..
-    // if we want to consider the upgrade scripts as a pre-requisite we should implement a wait on all
-    // containers for the scripts that are running in only 1 (while loop on "acquire" ?) and a healthcheck
-    // are not considered "up" and the previous versions keep running in the mean time
-  } else {
-    try {
-      await runScripts(db, basePath, isFresh)
-    } finally {
-      await locks.release('upgrade')
+    const ack = await locks.acquire('upgrade');
+    if (!ack) {
+        console.warn('upgrade scripts lock is already acquired, skip them');
+        // IMPORTANT: this behaviour of running the process when the upgrade scripts are still running on another one
+        // implies that they cannot be considered a pre-requisite..
+        // if we want to consider the upgrade scripts as a pre-requisite we should implement a wait on all
+        // containers for the scripts that are running in only 1 (while loop on "acquire" ?) and a healthcheck
+        // are not considered "up" and the previous versions keep running in the mean time
+    }
+    else {
+        try {
+            await runScripts(db, basePath, isFresh);
+        }
+        finally {
+            await locks.release('upgrade');
+        }
     }
-  }
 }
-async function runScripts (db, basePath, isFresh) {
-  const dir = path.resolve(basePath)
-  const pjsonPath = path.join(dir, 'package.json')
-  debug('read service info from ' + pjsonPath)
-  const pjson = JSON.parse(readFileSync(pjsonPath, 'utf8'))
-  const scriptsRoot = path.join(dir, 'upgrade')
-  debug(`service=${pjson.name} version=${pjson.version}`)
-  const services = db.collection('services')
-  const service = await services.findOne({ id: pjson.name })
-  let previousPersion = service?.version
-  debug('list scripts from ' + scriptsRoot)
-  let scripts = await listScripts(scriptsRoot)
-  if (previousPersion) {
-    debug(`current service version from database : ${previousPersion}`)
-  } else {
-    debug('no service version found in database, this is the first time the upgrade system is run')
-    scripts = scripts.filter(scriptDef => scriptDef.version === 'init')
-    if (isFresh) {
-      if (await isFresh()) {
-        debug('isFresh function returned true, this is a fresh install, do not run any script')
-      } else {
-        debug('isFresh function returned false, this is not a fresh install, run all init scripts')
-        previousPersion = '0.0.0'
-      }
-    } else {
-      debug('no isFresh function, it could be a fresh install, do not run any script')
+async function runScripts(db, basePath, isFresh) {
+    const dir = path.resolve(basePath);
+    const pjsonPath = path.join(dir, 'package.json');
+    debug('read service info from ' + pjsonPath);
+    const pjson = JSON.parse(readFileSync(pjsonPath, 'utf8'));
+    const scriptsRoot = path.join(dir, 'upgrade');
+    debug(`service=${pjson.name} version=${pjson.version}`);
+    const services = db.collection('services');
+    const service = await services.findOne({ id: pjson.name });
+    let previousPersion = service?.version;
+    debug('list scripts from ' + scriptsRoot);
+    let scripts = await listScripts(scriptsRoot);
+    if (previousPersion) {
+        debug(`current service version from database : ${previousPersion}`);
     }
-  }
-  if (previousPersion) {
-    for (const scriptDef of scripts) {
-      if (semver.gte(scriptDef.version, previousPersion)) {
-        for (const scriptName of scriptDef.names) {
-          const script = (await import(path.join(scriptsRoot, scriptDef.version, scriptName))).default
-          debug('apply script %s/%s : %s', scriptDef.version, scriptName, script.description)
-          await script.exec(db, Debug(`upgrade:${scriptDef.version}:${scriptName}`))
+    else {
+        debug('no service version found in database, this is the first time the upgrade system is run');
+        scripts = scripts.filter(scriptDef => scriptDef.version === 'init');
+        if (isFresh) {
+            if (await isFresh()) {
+                debug('isFresh function returned true, this is a fresh install, do not run any script');
+            }
+            else {
+                debug('isFresh function returned false, this is not a fresh install, run all init scripts');
+                previousPersion = '0.0.0';
+            }
+        }
+        else {
+            debug('no isFresh function, it could be a fresh install, do not run any script');
         }
-      }
     }
-  }
-  const coercedVersion = semver.coerce(pjson.version)
-  if (!coercedVersion) { throw new Error(`Invalid version number in package.json ${pjson.version}`) }
-  const newService = { id: pjson.name, version: coercedVersion.version }
-  debug(`upgrade scripts are over, save current version number ${newService.version}`)
-  await services.updateOne({ id: pjson.name }, { $set: newService }, { upsert: true })
+    if (previousPersion) {
+        for (const scriptDef of scripts) {
+            if (semver.gte(scriptDef.version, previousPersion)) {
+                for (const scriptName of scriptDef.names) {
+                    const script = (await import(path.join(scriptsRoot, scriptDef.version, scriptName))).default;
+                    debug('apply script %s/%s : %s', scriptDef.version, scriptName, script.description);
+                    await script.exec(db, Debug(`upgrade:${scriptDef.version}:${scriptName}`));
+                }
+            }
+        }
+    }
+    const coercedVersion = semver.coerce(pjson.version);
+    if (!coercedVersion)
+        throw new Error(`Invalid version number in package.json ${pjson.version}`);
+    const newService = { id: pjson.name, version: coercedVersion.version };
+    debug(`upgrade scripts are over, save current version number ${newService.version}`);
+    await services.updateOne({ id: pjson.name }, { $set: newService }, { upsert: true });
 }
 // Walk the scripts directories
-async function listScripts (scriptsRoot) {
-  const dirs = readdirSync(scriptsRoot)
-    .filter(dir => statSync(path.join(scriptsRoot, dir)).isDirectory())
-    .sort()
-  return dirs.map(dir => {
-    return {
-      version: dir,
-      names: readdirSync(path.join(scriptsRoot, dir)).sort()
-    }
-  })
+async function listScripts(scriptsRoot) {
+    const dirs = readdirSync(scriptsRoot)
+        .filter(dir => statSync(path.join(scriptsRoot, dir)).isDirectory())
+        .sort();
+    return dirs.map(dir => {
+        return {
+            version: dir,
+            names: readdirSync(path.join(scriptsRoot, dir)).sort()
+        };
+    });
 }

package/validation.js

@@ -1 +1 @@
-export * from '@data-fair/lib-validation'
+export * from '@data-fair/lib-validation';

package/ws-client.js

@@ -1,93 +1,97 @@
 // a wrapper around a websocket connection with auto reconnect
 // JSON parsing and channel subscriptions
-import EventEmitter from 'node:events'
-import WebSocket from 'ws'
-import Debug from 'debug'
-const debug = Debug('ws')
+import EventEmitter from 'node:events';
+import WebSocket from 'ws';
+import Debug from 'debug';
+const debug = Debug('ws');
 export class WsClient extends EventEmitter {
-  _channels
-  _ws
-  opts
-  constructor (opts) {
-    super()
-    this._channels = []
-    this.opts = { log: console, ...opts }
-  }
-
-  async _connect () {
-    return await new Promise((resolve, reject) => {
-      const wsUrl = this.opts.url.replace('http://', 'ws://').replace('https://', 'wss://') + '/'
-      debug(`connect Web Socket to ${wsUrl}`)
-      const ws = new WebSocket(wsUrl, { headers: this.opts.headers ?? {} })
-      this._ws = ws
-      ws.on('error', (err) => {
-        debug('WS encountered an error', err.message)
-        this._reconnect()
-        reject(err)
-      })
-      ws.once('open', () => {
-        debug('WS is opened')
-        resolve(ws)
-      })
-      ws.on('message', (msg) => {
-        const message = JSON.parse(msg)
-        debug('received message', message)
-        this.emit('message', message)
-      })
-    })
-  }
-
-  async _reconnect () {
-    if (!this._ws) { return }
-    debug('reconnect')
-    this._ws.terminate()
-    await this._connect()
-    for (const channel of this._channels) {
-      await this.subscribe(channel, true)
+    _channels;
+    _ws;
+    opts;
+    constructor(opts) {
+        super();
+        this._channels = [];
+        this.opts = { log: console, ...opts };
     }
-  }
-
-  async subscribe (channel, force = false, timeout = 2000) {
-    if (this._channels.includes(channel) && !force) { return }
-    const ws = this._ws ?? await this._connect()
-    debug('subscribe to channel', channel)
-    const subscribeMessage = { type: 'subscribe', channel }
-    if (this.opts.apiKey) { subscribeMessage.apiKey = this.opts.apiKey }
-    if (this.opts.adminMode && this.opts.account) { subscribeMessage.account = JSON.stringify(this.opts.account) }
-    ws.send(JSON.stringify(subscribeMessage))
-    const event = await this.waitFor(channel, (e) => {
-      return e.type === 'subscribe-confirm' || e.type === 'error'
-    }, timeout, true, true)
-    if (event.type === 'error') { throw new Error(event.data) }
-    if (this._channels.includes(channel)) { this._channels.push(channel) }
-  }
-
-  async waitFor (channel, filter, timeout = 300000, skipSubscribe = false, fullMessage = false) {
-    if (!skipSubscribe) { await this.subscribe(channel) }
-    return await new Promise((resolve, reject) => {
-      const _timeout = setTimeout(() => { reject(new Error('timeout')) }, timeout)
-      const messageCb = (message) => {
-        if (message.channel === channel && (!filter || filter(fullMessage ? message : message.data))) {
-          clearTimeout(_timeout)
-          this.off('message', messageCb)
-          resolve(fullMessage ? message : message.data)
+    async _connect() {
+        return await new Promise((resolve, reject) => {
+            const wsUrl = this.opts.url.replace('http://', 'ws://').replace('https://', 'wss://') + '/';
+            debug(`connect Web Socket to ${wsUrl}`);
+            const ws = new WebSocket(wsUrl, { headers: this.opts.headers ?? {} });
+            this._ws = ws;
+            ws.on('error', (err) => {
+                debug('WS encountered an error', err.message);
+                this._reconnect();
+                reject(err);
+            });
+            ws.once('open', () => {
+                debug('WS is opened');
+                resolve(ws);
+            });
+            ws.on('message', (msg) => {
+                const message = JSON.parse(msg);
+                debug('received message', message);
+                this.emit('message', message);
+            });
+        });
+    }
+    async _reconnect() {
+        if (!this._ws)
+            return;
+        debug('reconnect');
+        this._ws.terminate();
+        await this._connect();
+        for (const channel of this._channels) {
+            await this.subscribe(channel, true);
         }
-      }
-      this.on('message', messageCb)
-    })
-  }
-
-  close () {
-    if (this._ws) { this._ws.terminate() }
-  }
+    }
+    async subscribe(channel, force = false, timeout = 2000) {
+        if (this._channels.includes(channel) && !force)
+            return;
+        const ws = this._ws ?? await this._connect();
+        debug('subscribe to channel', channel);
+        const subscribeMessage = { type: 'subscribe', channel };
+        if (this.opts.apiKey)
+            subscribeMessage.apiKey = this.opts.apiKey;
+        if (this.opts.adminMode && this.opts.account)
+            subscribeMessage.account = JSON.stringify(this.opts.account);
+        ws.send(JSON.stringify(subscribeMessage));
+        const event = await this.waitFor(channel, (e) => {
+            return e.type === 'subscribe-confirm' || e.type === 'error';
+        }, timeout, true, true);
+        if (event.type === 'error')
+            throw new Error(event.data);
+        if (this._channels.includes(channel))
+            this._channels.push(channel);
+    }
+    async waitFor(channel, filter, timeout = 300000, skipSubscribe = false, fullMessage = false) {
+        if (!skipSubscribe)
+            await this.subscribe(channel);
+        return await new Promise((resolve, reject) => {
+            const _timeout = setTimeout(() => { reject(new Error('timeout')); }, timeout);
+            const messageCb = (message) => {
+                if (message.channel === channel && (!filter || filter(fullMessage ? message : message.data))) {
+                    clearTimeout(_timeout);
+                    this.off('message', messageCb);
+                    resolve(fullMessage ? message : message.data);
+                }
+            };
+            this.on('message', messageCb);
+        });
+    }
+    close() {
+        if (this._ws)
+            this._ws.terminate();
+    }
 }
 export class DataFairWsClient extends WsClient {
-  async waitForJournal (datasetId, eventType, timeout = 300000) {
-    await this.opts.log.info(`wait for event "${eventType}" on dataset "${datasetId}"`)
-    const event = await this.waitFor(`datasets/${datasetId}/journal`, (e) => {
-      return e.type === eventType || e.type === 'error'
-    }, timeout)
-    if (event.type === 'error') { throw new Error(event.data) }
-    return event
-  }
+    async waitForJournal(datasetId, eventType, timeout = 300000) {
+        await this.opts.log.info(`wait for event "${eventType}" on dataset "${datasetId}"`);
+        const event = await this.waitFor(`datasets/${datasetId}/journal`, (e) => {
+            return e.type === eventType || e.type === 'error';
+        }, timeout);
+        if (event.type === 'error')
+            throw new Error(event.data);
+        return event;
+    }
 }

package/ws-emitter.js

@@ -1,16 +1,18 @@
 // write messages to a mongodb collection/queue for publication by ws-server.js and consumption by ws.js
 // return a capped mongodb collection that acts as a simple queue
 export const initMessagesCollection = async (db) => {
-  const collection = (await db.listCollections({ name: 'ws-messages' }).toArray())[0]
-  if (!collection) { await db.createCollection('ws-messages', { capped: true, size: 100000, max: 1000 }) }
-  return db.collection('ws-messages')
-}
-let messagesCollection
+    const collection = (await db.listCollections({ name: 'ws-messages' }).toArray())[0];
+    if (!collection)
+        await db.createCollection('ws-messages', { capped: true, size: 100000, max: 1000 });
+    return db.collection('ws-messages');
+};
+let messagesCollection;
 export const init = async (db) => {
-  messagesCollection = await initMessagesCollection(db)
-  await messagesCollection.insertOne({ type: 'init' })
-}
+    messagesCollection = await initMessagesCollection(db);
+    await messagesCollection.insertOne({ type: 'init' });
+};
 export const emit = async (channel, data) => {
-  if (!messagesCollection) { throw new Error('wsEmitter.init was not called before sending data') }
-  await messagesCollection.insertOne({ type: 'message', channel, data, date: new Date().toISOString() })
-}
+    if (!messagesCollection)
+        throw new Error('wsEmitter.init was not called before sending data');
+    await messagesCollection.insertOne({ type: 'message', channel, data, date: new Date().toISOString() });
+};