@data-fair/dev-server 2.4.1 → 2.5.0

package/README.md

@@ -7,6 +7,41 @@ A development server for optimal development experience of data-fair application
 See [data-fair-charts](https://github.com/koumoul-dev/data-fair-charts/blob/master/package.json) for an example with nuxt.
 See [data-fair-minimal](https://github.com/koumoul-dev/data-fair-minimal/blob/master/package.json) for an example with a simple http server.
 
+## Ports de développement
+
+`df-dev-env` génère un `.env` git-ignoré portant trois ports libres consécutifs,
+tirés dans 20000–29999, pour que plusieurs applications tournent en parallèle :
+
+```
+APP_PORT=24730          # le serveur de dev de l'application (Vite)
+DEV_SERVER_PORT=24731   # df-dev-server
+E2E_PORT=24732          # le webServer de Playwright
+APP_PATH=/app/          # chemin sous lequel l'application est servie
+```
+
+Le fichier est généré une fois, au premier `npm run dev`, puis laissé tel quel.
+`df-dev-env --force` retire de nouveaux ports en cas de collision, en
+conservant l'`APP_PATH` déjà choisi : le remède à une collision de ports ne
+doit pas replacer en silence sous `/app/` une application servie à la racine.
+`--app-path` explicite prime sur la valeur conservée.
+
+Un `.env` déjà présent est laissé intact — mais `df-dev-env` prévient quand ce
+silence n'est pas ce qu'on attendait : quand le fichier ne porte pas `APP_PORT`
+(un `.env` écrit pour une autre raison, qui ne recevrait donc aucun port), et
+quand un `--app-path` explicite diffère de celui déjà stocké.
+
+`df-dev-server` lit ce `.env` (`dotenv`) : `DEV_SERVER_PORT` fixe son port, et
+`app.url` est dérivée de `APP_PORT` + `APP_PATH`. `APP_URL` reste prioritaire
+pour une application qui n'est pas servie sur `localhost`.
+
+`npm run dev-test-app-minimal` relit `.env` lui-même (via `dotenv-cli`) pour
+faire écouter `http-server` sur le bon `APP_PORT`, mais ne peut pas agir sur
+`APP_PATH` : ce script et le processus `df-dev-server` déjà démarré ne
+partagent rien d'autre que le fichier `.env`, écrit une seule fois par
+`df-dev-env` avec `/app/` par défaut. Pour tester `test-apps/minimal` (servie
+à la racine) il faut régénérer `.env` avant de lancer le dev-server :
+`node src/dev-env.js --force --app-path=`.
+
 ## Development
 
 Run development server :

package/config/custom-environment-variables.js

@@ -1,4 +1,10 @@
 export default {
+  port: {
+    __name: 'DEV_SERVER_PORT',
+    // __format, otherwise the config module hands over the raw string: the declared type is a
+    // number, and ajv cannot coerce it back since the config object is frozen before assertValid.
+    __format: 'json'
+  },
   dataFair: {
     url: 'DATAFAIR_URL',
     owner: {

package/config/default.js

@@ -10,7 +10,12 @@ export default {
     apiKey: null
   },
   app: {
-    url: 'http://localhost:3000',
+    // Derived from the app port so that a shifted dev setup cannot desynchronize the two.
+    // Path defaults to /app/: 27 of the 37 applications are Vite apps served under it, and
+    // this is the only value that works in a fresh clone that never ran df-dev-env. An
+    // application served at the root opts out with APP_PATH= (empty string is not nullish,
+    // so ?? keeps it). APP_URL still overrides it entirely, for apps not served on localhost.
+    url: `http://localhost:${process.env.APP_PORT ?? 3000}${process.env.APP_PATH ?? '/app/'}`,
     proxyPaths: ['/_nuxt/']
   },
   site: {

package/package.json

@@ -1,21 +1,23 @@
 {
   "name": "@data-fair/dev-server",
-  "version": "2.4.1",
+  "version": "2.5.0",
   "description": "A development server for optimal development experience of data-fair applications.",
   "main": "src/index.js",
   "bin": {
-    "df-dev-server": "src/index.js"
+    "df-dev-server": "src/index.js",
+    "df-dev-env": "src/dev-env.js"
   },
   "type": "module",
   "scripts": {
     "build": "tsc",
     "build-watch": "tsc --watch",
     "dev": "NODE_ENV=development node --watch src/index.js",
-    "dev-test-app-minimal": "http-server test-apps/minimal --port=3000 --cors --silent",
+    "dev-test-app-minimal": "dotenv -- sh -c 'http-server test-apps/minimal --port=${APP_PORT:-3000} --cors --silent'",
     "dev-test-app-vue3": "cd test-apps/vue3 && npm run dev",
     "dev-test-app-modern": "cd test-apps/modern && npm run dev",
-    "dev-zellij": "zellij --layout .zellij.kdl",
-    "prepublishOnly": "npm run lint && npm run build && npm -w ui run build",
+    "dev-zellij": "npm run build && node src/dev-env.js && dotenv -- zellij --layout .zellij.kdl",
+    "test": "node --test test/*.test.ts",
+    "prepublishOnly": "npm run lint && npm test && npm run build && npm -w ui run build",
     "lint": "eslint . && npm -w ui run lint",
     "lint-fix": "eslint --fix . && npm -w ui run lint-fix",
     "build-types": "df-build-types ."
@@ -46,6 +48,7 @@
     "@types/express": "^5.0.6",
     "@types/node": "^26.3.0",
     "@types/ws": "^8.18.1",
+    "dotenv-cli": "^11.0.0",
     "eslint": "^9.39.5",
     "eslint-plugin-vue": "^10.10.0",
     "eslint-plugin-vuetify": "^2.7.2",

package/src/app.js

@@ -1,6 +1,7 @@
 // Express app for TaxMan own API and UI
 import config from './config.js';
 import uiConfig from './ui-config.js';
+import { localizeConfig } from './localize.js';
 import { WebSocket, WebSocketServer } from 'ws';
 import { createServer } from 'node:http';
 import express from 'express';
@@ -91,7 +92,7 @@ app.get('/config', (req, res, next) => {
 // exact same data as the application receives in window.APPLICATION
 app.get('/config/enriched', async (req, res) => {
     try {
-        res.send(await refreshConfigDatasets(readDevConfig()));
+        res.send(await prepareConfig(readDevConfig()));
     }
     catch (err) {
         res.status(500).send({ error: err.message });
@@ -168,14 +169,11 @@ const enrichDataset = async (dataset) => {
     }
     try {
         const fresh = await remoteFetch('/datasets/' + encodeURIComponent(dataset.id));
-        const localBase = `http://localhost:${config.port}/data-fair`;
         const data = {};
         for (const prop of INJECTED_DATASET_PROPS) {
             if (fresh[prop] !== undefined)
                 data[prop] = fresh[prop];
         }
-        if (typeof data.href === 'string')
-            data.href = data.href.replace(config.dataFair.url, localBase);
         data.userPermissions = fresh.userPermissions ?? [];
         datasetsCache.set(dataset.id, { data, fetchedAt: Date.now() });
         return { ...dataset, ...data };
@@ -188,12 +186,15 @@ const enrichDataset = async (dataset) => {
         return dataset;
     }
 };
-const refreshConfigDatasets = async (configuration) => {
+// Enrich the datasets from the remote data-fair, then rewrite every remote origin to ours.
+// The rewrite is applied even when there is no dataset to enrich: a configuration can carry
+// remote urls anywhere (logos, links, tileserver styles), not only in datasets[].href.
+const prepareConfig = async (configuration) => {
     const datasets = configuration?.datasets?.filter((d) => !!d);
-    if (!datasets?.length)
-        return configuration;
-    const enriched = await Promise.all(datasets.map(enrichDataset));
-    return { ...configuration, datasets: enriched };
+    const enriched = datasets?.length
+        ? { ...configuration, datasets: await Promise.all(datasets.map(enrichDataset)) }
+        : configuration;
+    return localizeConfig(enriched, new URL(config.dataFair.url).origin, `http://localhost:${config.port}`);
 };
 // read the .dev-config.json file of the app under development
 const readDevConfig = () => existsSync('.dev-config.json') ? JSON.parse(readFileSync('.dev-config.json', 'utf8')) : {};
@@ -354,14 +355,13 @@ app.get('/configurations', async (req, res) => {
         res.status(500).send({ ...response, error: err.message });
     }
 });
-// copy a configuration from the remote data-fair, replacing all references to the
-// remote origin by the local one so that data goes through the local proxies
+// copy a configuration from the remote data-fair, keeping its remote origins so that
+// .dev-config.json stays portable — see localize.ts
 app.get('/configurations/:id', async (req, res) => {
     try {
-        const configuration = await remoteFetch('/applications/' + encodeURIComponent(req.params.id) + '/configuration');
-        const remoteOrigin = new URL(config.dataFair.url).origin;
-        const localOrigin = `http://localhost:${config.port}`;
-        res.send(JSON.parse(JSON.stringify(configuration).replaceAll(remoteOrigin, localOrigin)));
+        // Sent as-is, with its remote origins: the UI stores this straight into .dev-config.json,
+        // which must stay portable. Origins are rewritten on the read path, in prepareConfig.
+        res.send(await remoteFetch('/applications/' + encodeURIComponent(req.params.id) + '/configuration'));
     }
     catch (err) {
         res.status(500).send({ error: err.message });
@@ -407,7 +407,7 @@ app.use('/app', createProxyMiddleware({
                     let output = rawBody.toString();
                     if (output.includes('%APPLICATION%')) {
                         try {
-                            configuration = await refreshConfigDatasets(configuration);
+                            configuration = await prepareConfig(configuration);
                         }
                         catch (err) {
                             console.warn('[dev-server] failed to enrich configuration datasets', err);

package/src/dev-env.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/src/dev-env.js

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { createServer } from 'node:net';
+import { parseArgs } from 'node:util';
+import { MIN_BASE, MAX_BASE, renderEnv, findBase, readEnvVar } from './env-file.js';
+const { values } = parseArgs({
+    options: {
+        force: { type: 'boolean', default: false },
+        // No default value: an absent --app-path must stay distinguishable from an explicit one, so
+        // that --force can redraw the ports without resetting a path the developer chose.
+        'app-path': { type: 'string' }
+    }
+});
+const existing = existsSync('.env') ? readFileSync('.env', 'utf8') : undefined;
+const storedAppPath = existing === undefined ? undefined : readEnvVar(existing, 'APP_PATH');
+if (existing !== undefined && !values.force) {
+    // An existing .env is left untouched — but say so when leaving it untouched is not what the
+    // caller expected, rather than letting the application start on the wrong port or path.
+    if (readEnvVar(existing, 'APP_PORT') === undefined) {
+        console.warn('.env existe déjà mais ne porte pas APP_PORT : y ajouter les lignes de port à la main, ou df-dev-env --force');
+    }
+    else if (values['app-path'] !== undefined && values['app-path'] !== storedAppPath) {
+        console.warn(`.env existe déjà avec APP_PATH=${storedAppPath} : --app-path=${values['app-path']} est ignoré, df-dev-env --force pour l'appliquer`);
+    }
+    process.exit(0);
+}
+const isFree = (port) => new Promise(resolve => {
+    const server = createServer();
+    server.once('error', () => { resolve(false); });
+    server.once('listening', () => { server.close(() => { resolve(true); }); });
+    server.listen(port, '127.0.0.1');
+});
+const draw = () => MIN_BASE + Math.floor(Math.random() * (MAX_BASE - MIN_BASE + 1));
+// --force is the documented remedy for a port collision: it must not silently move an
+// application served at the root back under /app/ along the way.
+const appPath = values['app-path'] ?? storedAppPath ?? '/app/';
+const base = await findBase(isFree, draw);
+writeFileSync('.env', renderEnv(base, appPath));
+console.log(`.env généré — app ${base}, dev-server ${base + 1}, e2e ${base + 2}, chemin ${appPath || '/ (racine)'}`);

package/src/env-file.d.ts

@@ -0,0 +1,5 @@
+export declare const MIN_BASE = 20000;
+export declare const MAX_BASE = 29997;
+export declare const renderEnv: (base: number, appPath: string) => string;
+export declare const readEnvVar: (content: string, name: string) => string | undefined;
+export declare const findBase: (isFree: (port: number) => Promise<boolean>, draw: () => number, attempts?: number) => Promise<number>;

package/src/env-file.js

@@ -0,0 +1,28 @@
+// Development ports live below the kernel ephemeral range (32768-60999 on Linux), where a port
+// can already be held by an outgoing connection, and above the 191xx block used by the
+// data-fair docker compose.
+export const MIN_BASE = 20000;
+// The base takes three consecutive ports, so it may not go past 29999.
+export const MAX_BASE = 29997;
+export const renderEnv = (base, appPath) => `# généré par df-dev-env — ne pas commiter
+APP_PORT=${base}
+DEV_SERVER_PORT=${base + 1}
+E2E_PORT=${base + 2}
+APP_PATH=${appPath}
+`;
+// Reads one variable out of an already written .env. Used to tell a .env that carries our ports
+// from one written for another purpose, and to keep the APP_PATH a developer chose when --force
+// redraws the ports. Deliberately naive: the file we read is the one we wrote.
+export const readEnvVar = (content, name) => {
+    const line = content.split('\n').find(l => l.startsWith(name + '='));
+    return line === undefined ? undefined : line.slice(name.length + 1).trim();
+};
+export const findBase = async (isFree, draw, attempts = 10) => {
+    for (let i = 0; i < attempts; i++) {
+        const base = draw();
+        const free = await Promise.all([base, base + 1, base + 2].map(isFree));
+        if (free.every(Boolean))
+            return base;
+    }
+    throw new Error(`no free port range found after ${attempts} attempts`);
+};

package/src/localize.d.ts

@@ -0,0 +1 @@
+export declare const localizeConfig: <T>(configuration: T, remoteOrigin: string, localOrigin: string) => T;

package/src/localize.js

@@ -0,0 +1,16 @@
+// Configurations are stored with their remote (koumoul.com) origins so that .dev-config.json
+// stays portable across developers, machines and regenerated ports. Origins are rewritten here,
+// on the read path, on the way to the application — never on the write path.
+// Paths the dev-server re-exposes under its own origin.
+const PROXIED_PATHS = ['/data-fair', '/simple-directory', '/tileserver'];
+// Transition rewrite: .dev-config.json files written before this change carry a hardcoded
+// http://localhost:5888 origin. Only rewrite one when it stands on a path we actually proxy,
+// and only when that path is complete — a deliberate http://localhost:8080/data-fair-lookalike
+// typed by a developer must survive untouched.
+const legacyLocalOrigin = new RegExp(`http://localhost:\\d+(?=(?:${PROXIED_PATHS.join('|')})(?:/|"))`, 'g');
+export const localizeConfig = (configuration, remoteOrigin, localOrigin) => {
+    const json = JSON.stringify(configuration);
+    if (json === undefined)
+        return configuration;
+    return JSON.parse(json.replaceAll(remoteOrigin, localOrigin).replace(legacyLocalOrigin, localOrigin));
+};