@data-fair/dev-server 2.6.4 → 2.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/dev-server",
-  "version": "2.6.4",
+  "version": "2.7.0",
   "description": "A development server for optimal development experience of data-fair applications.",
   "main": "src/index.js",
   "bin": {

package/src/app.js

@@ -10,7 +10,7 @@ import { WebSocket, WebSocketServer } from 'ws';
 import { createServer } from 'node:http';
 import express from 'express';
 import cors from 'cors';
-import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync, watchFile } from 'node:fs';
 import debugModule from 'debug';
 import { createProxyMiddleware, fixRequestBody } from 'http-proxy-middleware';
 import * as parse5 from 'parse5';
@@ -111,22 +111,46 @@ app.get('/config', (req, res, next) => {
     debug('read dev config', devConfig);
     res.send(devConfig);
 });
-// the config with datasets enriched from the remote data-fair, so the dev-server UI
-// can display schemas/concepts and build concept & dataset filter params with the
-// exact same data as the application receives in window.APPLICATION
+// the config with datasets enriched from the remote data-fair, as the application receives it in
+// window.APPLICATION ; ?select= adds dataset properties for a dev-server tool that needs them
+// whatever the application selects (the filter tester and its schema fields)
 app.get('/config/enriched', async (req, res) => {
     try {
-        res.send(await prepareConfig(readDevConfig()));
+        const extraSelect = typeof req.query.select === 'string' ? req.query.select.split(',') : undefined;
+        res.send(await prepareConfig(readDevConfig(), extraSelect));
     }
     catch (err) {
         res.status(500).send({ error: err.message });
     }
 });
+// what the UI last wrote, to tell its own saves from a hand edit of the file
+let lastWrittenConfig;
 app.put('/config', (req, res, next) => {
     debug('save dev config', req.body);
-    writeFileSync('.dev-config.json', JSON.stringify(req.body, null, 2));
+    lastWrittenConfig = JSON.stringify(req.body, null, 2);
+    writeFileSync('.dev-config.json', lastWrittenConfig);
     res.send(req.body);
 });
+// A hand edit of .dev-config.json (an editor, an agent) reloads the UI, otherwise the form state
+// would overwrite it on its next save. Polled rather than fs.watch'ed: editors that save through
+// a rename would detach a watcher. A file that does not parse yet is a save in progress, skipped.
+watchFile('.dev-config.json', { interval: 500 }, () => {
+    if (!existsSync('.dev-config.json'))
+        return;
+    const content = readFileSync('.dev-config.json', 'utf8');
+    if (content === lastWrittenConfig)
+        return;
+    try {
+        JSON.parse(content);
+    }
+    catch {
+        return;
+    }
+    lastWrittenConfig = content;
+    debug('.dev-config.json edited outside of the UI');
+    for (const ws of devServerWS)
+        ws.send(JSON.stringify({ type: 'config-file-changed' }));
+});
 app.post('/config/error', (req, res) => {
     console.log('Application sent an error', req.body);
     for (const ws of devServerWS) {
@@ -186,8 +210,16 @@ const localAppInfo = async () => {
 };
 // Enrich the datasets from the remote data-fair and rewrite every remote origin to ours
 // (pure helpers in enrich.ts, wired here to our remote api and local origins).
-const prepareConfig = (configuration) => prepareRemoteConfig(configuration, {
+const prepareConfig = (configuration, extraSelect) => prepareRemoteConfig(configuration, {
     fetchJson: remoteFetch,
+    // the local file, as data-fair reads the one of the base application to deduce its selects
+    fetchConfigSchema: async () => {
+        const res = await fetch(appUrl.origin + appPrefix + '/config-schema.json');
+        if (!res.ok)
+            throw new Error('config-schema.json ' + res.status);
+        return res.json();
+    },
+    extraSelect,
     localize: localizeConfig,
     remoteOrigin: new URL(config.dataFair.url).origin,
     localOrigin: `http://localhost:${config.port}`
@@ -318,6 +350,22 @@ const captureSimulationScript = (opts) => {
 })()
 `;
 };
+// Count what the previewed app writes to the console as error / warning and report it to the
+// dev-server UI, which shows the counts as chips nudging toward the DevTools console.
+// Failed requests are counted from their HTTP status (what Chrome logs as "Failed to load
+// resource", caught by the app or not), not from the resource's error event, which also fires
+// on a decoding failure Chrome never logs (an image answered by Vite's SPA fallback). Kept on a
+// single line so that the inline source map (one "AAAA" segment) covers all of it and its
+// x_google_ignoreList hides the wrapper: DevTools then locates console.error / console.warn at
+// the app's call site instead of here.
+const CONSOLE_COUNTER_SOURCE_MAP = Buffer.from(JSON.stringify({
+    version: 3, sources: ['df-dev-server-console-counter.js'], mappings: 'AAAA', x_google_ignoreList: [0]
+})).toString('base64');
+const CONSOLE_COUNTER_SCRIPT = [
+    ";(function () { if (window.parent === window) return; var post = function (level) { window.parent.postMessage({ type: 'df-dev-server:console', level: level }, '*') }; post('reset'); ['error', 'warn'].forEach(function (method) { var original = console[method]; console[method] = function () { original.apply(console, arguments); post(method === 'warn' ? 'warning' : 'error') } }); window.addEventListener('error', function (e) { if (!/^ResizeObserver loop/.test(e.message)) post('error') }); new PerformanceObserver(function (list) { list.getEntries().forEach(function (e) { if (e.responseStatus >= 400) post('error') }) }).observe({ type: 'resource', buffered: true }); window.addEventListener('unhandledrejection', function () { post('error') }) })()",
+    '//# sourceURL=df-dev-server-console-counter.js',
+    '//# sourceMappingURL=data:application/json;base64,' + CONSOLE_COUNTER_SOURCE_MAP
+].join('\n');
 // extract the major.minor part of a version, versions on remote base apps look like "1.3"
 const minorVersion = (version) => version.split('.').slice(0, 2).join('.');
 // list configurations of the app under development that exist on the remote data-fair
@@ -505,6 +553,14 @@ app.use('/app', createProxyMiddleware({
                             // reliable here as in production.
                             headNode.childNodes.unshift(script);
                         }
+                        // right after <meta charset>, so that the byte offset of the charset (checked by the
+                        // UI on this same proxied document) stays unaffected, and before the app's own scripts
+                        // so that errors raised while it boots are counted too (vite's client script placed
+                        // above is a module, it runs later anyway)
+                        const consoleScript = createElement('script', { type: 'text/javascript' });
+                        appendChild(consoleScript, createTextNode(CONSOLE_COUNTER_SCRIPT));
+                        const charsetIndex = headNode.childNodes.findIndex(c => isElementNode(c) && c.tagName === 'meta' && c.attrs.some(a => a.name === 'charset'));
+                        headNode.childNodes.splice(charsetIndex + 1, 0, consoleScript);
                         // companion script that lets the embedded app report its height / sync params
                         // to the parent <d-frame> ; injected for every mode so the UI mode toggle works
                         // without requiring a df:overflow meta (same variant & version as data-fair prod)
@@ -562,9 +618,12 @@ app.use('/data-fair', createProxyMiddleware({
         proxyRes(proxyRes, req, res) {
             // console.log('DF RES', proxyRes)
             if (proxyRes.headers['content-type'] && proxyRes.headers['content-type'].startsWith('application/json')) {
-                let body = '';
-                proxyRes.on('data', (data) => { body += data.toString(); });
+                // decode the whole body once: decoding chunk by chunk turns a multi-byte character
+                // split across two chunks into U+FFFD (« Repr��sentation »)
+                const chunks = [];
+                proxyRes.on('data', (data) => { chunks.push(data); });
                 proxyRes.on('end', () => {
+                    const body = Buffer.concat(chunks).toString();
                     const output = body.replace(new RegExp(escapeStringRegexp(config.dataFair.url), 'g'), `http://localhost:${config.port}/data-fair`);
                     // proxyRes.headers['content-length'] = output.length
                     delete proxyRes.headers['content-length'];
@@ -596,9 +655,12 @@ app.use('/tileserver', createProxyMiddleware({
         },
         proxyRes(proxyRes, req, res) {
             if (proxyRes.headers['content-type'] && proxyRes.headers['content-type'].startsWith('application/json')) {
-                let body = '';
-                proxyRes.on('data', (data) => { body += data.toString(); });
+                // decode the whole body once: decoding chunk by chunk turns a multi-byte character
+                // split across two chunks into U+FFFD (« Repr��sentation »)
+                const chunks = [];
+                proxyRes.on('data', (data) => { chunks.push(data); });
                 proxyRes.on('end', () => {
+                    const body = Buffer.concat(chunks).toString();
                     // make all references to the tileserver url point to the local proxy
                     const output = body.replace(new RegExp(escapeStringRegexp(tileserverUrl), 'g'), `http://localhost:${config.port}/tileserver`);
                     delete proxyRes.headers['content-length'];

package/src/dev-env.js

@@ -2,7 +2,7 @@
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { createServer } from 'node:net';
 import { parseArgs } from 'node:util';
-import { MIN_BASE, MAX_BASE, renderEnv, findBase, readEnvVar } from './env-file.js';
+import { MIN_BASE, MAX_BASE, renderEnv, mergeEnv, findBase, readEnvVar } from './env-file.js';
 const { values } = parseArgs({
     options: {
         force: { type: 'boolean', default: false },
@@ -35,5 +35,6 @@ const draw = () => MIN_BASE + Math.floor(Math.random() * (MAX_BASE - MIN_BASE +
 // application served at the root back under /app/ along the way.
 const appPath = values['app-path'] ?? storedAppPath ?? '/app/';
 const base = await findBase(isFree, draw);
-writeFileSync('.env', renderEnv(base, appPath));
+const generated = renderEnv(base, appPath);
+writeFileSync('.env', existing === undefined ? generated : mergeEnv(existing, generated));
 console.log(`.env généré — app ${base}, dev-server ${base + 1}, e2e ${base + 2}, chemin ${appPath || '/ (racine)'}`);

package/src/enrich.d.ts

@@ -1,7 +1,13 @@
-export declare const INJECTED_DATASET_PROPS: readonly ["id", "href", "page", "title", "slug", "finalizedAt", "schema", "isRest", "userPermissions", "bbox", "timePeriod", "attachmentsAsImage", "count"];
-export declare const enrichDataset: (dataset: any, fetchJson: (path: string) => Promise<any>) => Promise<any>;
+export interface DatasetFilter {
+    select?: string[];
+    properties?: Record<string, any>;
+}
+export declare const datasetsFiltersFromSchema: (configSchema: any) => DatasetFilter[];
+export declare const enrichDataset: (dataset: any, fetchJson: (path: string) => Promise<any>, filter?: DatasetFilter) => Promise<any>;
 export interface PrepareConfigDeps {
     fetchJson: (path: string) => Promise<any>;
+    fetchConfigSchema: () => Promise<any>;
+    extraSelect?: string[];
     localize: (configuration: any, remoteOrigin: string, localOrigin: string) => any;
     remoteOrigin: string;
     localOrigin: string;

package/src/enrich.js

@@ -1,9 +1,10 @@
 // Rebuild the configuration dataset entries the same way data-fair does in production
 // (refreshConfigDatasetsRefs in api/src/applications/utils.ts): an application stores only
-// minimal dataset references, data-fair injects the full schema (with concepts), finalizedAt,
-// slug, isRest and the per-request userPermissions at serving time. We reproduce that so
-// applications following the skill contract read window.APPLICATION.configuration.datasets
-// identically in dev and in prod.
+// minimal dataset references, data-fair refreshes them at serving time with the keys already
+// stored in the entry, plus `finalizedAt` and `slug`, plus the `select` of the dataset selector
+// declared in the application's config schema (baseApp.datasetsFilters, deduced by initBaseApp in
+// api/src/base-applications/service.ts). Nothing more: an application that reads a property its
+// selector does not ask for works here only if it works in production too.
 //
 // Pure module, no imports: it takes its remote accessor and origin rewrite as dependencies,
 // wired in app.ts — so it can be unit tested without a build (test/enrich.test.ts).
@@ -12,52 +13,123 @@
 // key would otherwise be re-fetched, and re-warned about, on every single reload.
 const datasetsCache = new Map();
 const DATASETS_CACHE_TTL = 30_000;
-// The set of properties data-fair injects in a configuration dataset entry. Sticking to it
-// matters in both directions: forwarding the whole remote dataset would let an application rely,
-// in dev, on a property production never sends — and omitting one production does send makes an
-// application work in production and not here, which is worse still.
-//
-// data-fair derives the set per request (refreshConfigDatasetsRefs, api/src/applications/utils.ts):
-// the keys already stored in the entry, plus `finalizedAt` and `slug`, plus the `select` of the
-// dataset selector declared in the application's own config schema. We cannot derive it without
-// parsing that schema, so this list mirrors it and has to track it: it is the union of the
-// selects used by the published applications. `isRest` gates whether rest line editing
-// (POST/PATCH/DELETE /lines) is offered at all; `bbox` and `timePeriod` drive the initial map
-// bounds and period pickers; `attachmentsAsImage` switches list apps to thumbnail mode.
-export const INJECTED_DATASET_PROPS = ['id', 'href', 'page', 'title', 'slug', 'finalizedAt', 'schema', 'isRest', 'userPermissions', 'bbox', 'timePeriod', 'attachmentsAsImage', 'count'];
-export const enrichDataset = async (dataset, fetchJson) => {
-    if (!dataset?.id)
-        return dataset;
-    const cached = datasetsCache.get(dataset.id);
-    if (cached && Date.now() - cached.fetchedAt < DATASETS_CACHE_TTL) {
-        return cached.data ? { ...dataset, ...cached.data } : dataset;
+// data-fair's resolveLocalRefs (api/src/misc/utils/json-refs.ts): "#/..." pointers replaced by
+// their target, circular and unresolvable ones left as they are.
+const resolveLocalRefs = (root) => {
+    const atPointer = (pointer) => pointer.split('/').slice(1)
+        .map(p => p.replace(/~1/g, '/').replace(/~0/g, '~'))
+        .reduce((o, k) => (o !== null && typeof o === 'object') ? o[k] : undefined, root);
+    const walk = (node, stack) => {
+        if (Array.isArray(node))
+            return node.map(n => walk(n, stack));
+        if (node === null || typeof node !== 'object')
+            return node;
+        if (typeof node.$ref === 'string' && node.$ref.startsWith('#')) {
+            if (stack.includes(node.$ref))
+                return node;
+            const target = atPointer(node.$ref);
+            if (target === undefined)
+                return node;
+            return walk(target, [...stack, node.$ref]);
+        }
+        const res = {};
+        for (const [k, v] of Object.entries(node))
+            res[k] = walk(v, stack);
+        return res;
+    };
+    return walk(root, []);
+};
+// data-fair's getFragmentFetchUrl (api/src/base-applications/operations.ts)
+const fragmentFetchUrl = (fragment) => {
+    if (!fragment)
+        return null;
+    if (fragment['x-fromUrl'])
+        return fragment['x-fromUrl'];
+    const url = fragment.layout?.getItems?.url;
+    if (url)
+        return typeof url === 'string' ? url : url.expr;
+    return null;
+};
+// The datasetsFilters data-fair stores on the base application, one per dataset selector of the
+// config schema, matched by position with configuration.datasets. Like data-fair, a schema that
+// cannot be read gives no filter at all.
+export const datasetsFiltersFromSchema = (configSchema) => {
+    try {
+        const schema = resolveLocalRefs(configSchema);
+        const definition = schema?.properties?.datasets ?? schema?.allOf?.[0]?.properties?.datasets;
+        if (!definition)
+            return [];
+        let fetches = [];
+        if (definition.items && fragmentFetchUrl(definition))
+            fetches = [{ fromUrl: fragmentFetchUrl(definition), properties: definition.items.properties }];
+        if (fragmentFetchUrl(definition.items))
+            fetches = [{ fromUrl: fragmentFetchUrl(definition.items), properties: definition.items.properties }];
+        if (Array.isArray(definition.items))
+            fetches = definition.items.filter((item) => fragmentFetchUrl(item)).map((item) => ({ fromUrl: fragmentFetchUrl(item), properties: item.properties }));
+        return fetches.map(({ fromUrl, properties }) => {
+            const select = new URL(fromUrl, 'http://localhost').searchParams.get('select');
+            return { ...(select ? { select: select.split(',') } : {}), ...(properties ? { properties } : {}) };
+        });
+    }
+    catch {
+        return [];
     }
+};
+const fetchFreshDataset = async (id, fetchJson) => {
+    const cached = datasetsCache.get(id);
+    if (cached && Date.now() - cached.fetchedAt < DATASETS_CACHE_TTL)
+        return cached.data;
     try {
-        const fresh = await fetchJson('/datasets/' + encodeURIComponent(dataset.id));
-        const data = {};
-        for (const prop of INJECTED_DATASET_PROPS) {
-            if (fresh[prop] !== undefined)
-                data[prop] = fresh[prop];
-        }
-        data.userPermissions = fresh.userPermissions ?? [];
-        datasetsCache.set(dataset.id, { data, fetchedAt: Date.now() });
-        return { ...dataset, ...data };
+        const fresh = await fetchJson('/datasets/' + encodeURIComponent(id));
+        datasetsCache.set(id, { data: fresh, fetchedAt: Date.now() });
+        return fresh;
     }
     catch (err) {
         // a private dataset without an api key, or a network failure: keep the raw
         // configuration entry so the app still loads, and warn in the dev-server UI
-        console.warn('[dev-server] failed to enrich dataset ' + dataset.id + ', keeping raw configuration entry', err);
-        datasetsCache.set(dataset.id, { data: null, fetchedAt: Date.now() });
+        console.warn('[dev-server] failed to enrich dataset ' + id + ', keeping raw configuration entry', err);
+        datasetsCache.set(id, { data: null, fetchedAt: Date.now() });
+        return null;
+    }
+};
+export const enrichDataset = async (dataset, fetchJson, filter = {}) => {
+    if (!dataset?.id)
         return dataset;
+    const fresh = await fetchFreshDataset(dataset.id, fetchJson);
+    const result = { ...dataset };
+    if (fresh) {
+        for (const key of [...Object.keys(dataset), 'finalizedAt', 'slug', ...(filter.select ?? [])]) {
+            if (key === 'userPermissions')
+                result.userPermissions = fresh.userPermissions ?? [];
+            else if (key in fresh)
+                result[key] = fresh[key];
+            else if (key === 'partOf')
+                delete result.partOf;
+        }
     }
+    for (const [key, prop] of Object.entries(filter.properties ?? {})) {
+        if (prop?.default && !(key in result))
+            result[key] = prop.default;
+        if (prop?.const)
+            result[key] = prop.const;
+    }
+    return result;
 };
 // Enrich the datasets from the remote data-fair, then rewrite every remote origin to ours.
 // The rewrite is applied even when there is no dataset to enrich: a configuration can carry
 // remote urls anywhere (logos, links, tileserver styles), not only in datasets[].href.
 export const prepareConfig = async (configuration, deps) => {
     const datasets = configuration?.datasets?.filter((d) => !!d);
-    const enriched = datasets?.length
-        ? { ...configuration, datasets: await Promise.all(datasets.map(d => enrichDataset(d, deps.fetchJson))) }
-        : configuration;
+    let enriched = configuration;
+    if (datasets?.length) {
+        const filters = datasetsFiltersFromSchema(await deps.fetchConfigSchema().catch(() => null));
+        enriched = {
+            ...configuration,
+            datasets: await Promise.all(datasets.map((d, i) => {
+                const filter = filters[i] ?? {};
+                return enrichDataset(d, deps.fetchJson, deps.extraSelect ? { ...filter, select: [...(filter.select ?? []), ...deps.extraSelect] } : filter);
+            }))
+        };
+    }
     return deps.localize(enriched, deps.remoteOrigin, deps.localOrigin);
 };

package/src/env-file.d.ts

@@ -1,5 +1,6 @@
 export declare const MIN_BASE = 20000;
 export declare const MAX_BASE = 29997;
 export declare const renderEnv: (base: number, appPath: string) => string;
+export declare const mergeEnv: (existing: string, generated: string) => string;
 export declare const readEnvVar: (content: string, name: string) => string | undefined;
 export declare const findBase: (isFree: (port: number) => Promise<boolean>, draw: () => number, attempts?: number) => Promise<number>;

package/src/env-file.js

@@ -10,6 +10,15 @@ DEV_SERVER_PORT=${base + 1}
 E2E_PORT=${base + 2}
 APP_PATH=${appPath}
 `;
+// --force redraws our own lines and keeps every other one of an existing .env: the remote
+// settings a developer added there (DATAFAIR_URL, DATAFAIR_API_KEY...) are not ours to drop.
+const OWN_LINE = /^(# généré par df-dev-env|(APP_PORT|DEV_SERVER_PORT|E2E_PORT|APP_PATH)=)/;
+export const mergeEnv = (existing, generated) => {
+    const kept = existing.split('\n').filter(l => !OWN_LINE.test(l));
+    while (kept.length && kept[kept.length - 1].trim() === '')
+        kept.pop();
+    return kept.length ? generated + kept.join('\n') + '\n' : generated;
+};
 // Reads one variable out of an already written .env. Used to tell a .env that carries our ports
 // from one written for another purpose, and to keep the APP_PATH a developer chose when --force
 // redraws the ports. Deliberately naive: the file we read is the one we wrote.