@data-fair/dev-server 2.5.1 → 2.6.0

package/README.md

@@ -51,12 +51,23 @@ balise `<meta name="application-name">` de l'`index.html` local et de la version
 du `package.json`.
 
 Une application de base réservée à une organisation (`privateAccess`) est
-invisible à une requête anonyme : sans elle la liste est vide, quand bien même
-les applications qui l'utilisent seraient publiques. Le paramètre
-`privateAccess` la rend visible, mais data-fair répond 401 sans
-authentification — il n'est donc envoyé que si `DATAFAIR_API_KEY` est
-renseignée. Une clé d'API se crée depuis le compte ou l'organisation sur le
-data-fair distant, et se met dans le `.env` de l'application :
+invisible à une requête anonyme. Le paramètre `privateAccess` la rend visible,
+mais data-fair répond 401 sans authentification — il n'est donc envoyé que si
+`DATAFAIR_API_KEY` est renseignée.
+
+Ne pas la trouver n'arrête pas la recherche pour autant : les applications qui
+tournent dessus sont souvent publiques, et il suffit de connaître son URL pour
+les lister. Les URL des applications de base publiées suivent deux conventions
+(`<data-fair>/apps/<slug>/<mineure>/` et
+`https://cdn.jsdelivr.net/npm/@scope/<paquet>@<mineure>/dist/`), et le filtre
+de `/applications` est une égalité stricte : une URL devinée ne peut que ne
+rien renvoyer, jamais renvoyer autre chose. Les candidates sont donc essayées
+en une requête, ce qui suffit à lister les configurations publiques d'une
+application de base privée, sans aucune clé.
+
+Une clé reste nécessaire pour une application, elle, privée. Elle se crée
+depuis le compte ou l'organisation sur le data-fair distant, et se met dans le
+`.env` de l'application :
 
 ```
 DATAFAIR_API_KEY=xxx

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/dev-server",
-  "version": "2.5.1",
+  "version": "2.6.0",
   "description": "A development server for optimal development experience of data-fair applications.",
   "main": "src/index.js",
   "bin": {

package/src/app.js

@@ -2,7 +2,10 @@
 import config from './config.js';
 import uiConfig from './ui-config.js';
 import { localizeConfig } from './localize.js';
+import { remoteFetch, remoteFetchBuffer } from './remote.js';
+import { prepareConfig as prepareRemoteConfig } from './enrich.js';
 import { ATTACHMENTS_DIR, attachmentPath, copyAttachments, listAttachments } from './attachments.js';
+import { candidateBaseAppUrls } from './base-app-urls.js';
 import { WebSocket, WebSocketServer } from 'ws';
 import { createServer } from 'node:http';
 import express from 'express';
@@ -150,13 +153,16 @@ const localAppInfo = async () => {
         debug('failed to fetch local app index.html', err);
     }
     let version;
+    let packageName;
     try {
-        version = JSON.parse(readFileSync('package.json', 'utf8')).version;
+        const pkg = JSON.parse(readFileSync('package.json', 'utf8'));
+        version = pkg.version;
+        packageName = pkg.name;
     }
     catch (err) {
         debug('failed to read local package.json', err);
     }
-    return { name, version };
+    return { name, version, packageName };
 };
 // The owner the dev-server works on behalf of, in the "type:id[:department]" form the
 // data-fair api expects for its owner and privateAccess filters.
@@ -167,76 +173,14 @@ const ownerFilter = () => {
         filter += ':' + owner.department;
     return filter;
 };
-// Fetch a resource from the remote data-fair api
-const remoteFetch = async (path) => {
-    const res = await remoteFetchRaw(path);
-    return res.json();
-};
-// Same, for an attachment: a file, never json, and never decoded as text — see the app proxy
-// below, where decoding a binary body as utf8 destroys it.
-const remoteFetchBuffer = async (path) => {
-    const res = await remoteFetchRaw(path);
-    return Buffer.from(await res.arrayBuffer());
-};
-const remoteFetchRaw = async (path) => {
-    const res = await fetch(config.dataFair.url + '/api/v1' + path, {
-        headers: config.dataFair.apiKey ? { 'x-apiKey': config.dataFair.apiKey } : {}
-    });
-    if (!res.ok)
-        throw new Error(`error ${res.status} on remote data-fair: ${await res.text()}`);
-    return res;
-};
-// Short-lived cache for the dataset enrichment below, so that every preview reload does not
-// hammer the remote data-fair API. Failures are cached too: a private dataset without an api
-// key would otherwise be re-fetched, and re-warned about, on every single reload.
-const datasetsCache = new Map();
-const DATASETS_CACHE_TTL = 30_000;
-// The exact set of properties data-fair injects in a configuration dataset entry. Sticking to
-// it matters: forwarding the whole remote dataset would let an application rely, in dev, on a
-// property that production never sends.
-const INJECTED_DATASET_PROPS = ['id', 'href', 'page', 'title', 'slug', 'finalizedAt', 'schema', 'userPermissions'];
-// Rebuild a configuration dataset entry the same way data-fair does in production
-// (refreshConfigDatasetsRefs in api/src/applications/utils.ts): the app stores only
-// minimal references, data-fair injects the full schema (with concepts), finalizedAt,
-// slug and userPermissions at request time. We reproduce that so applications
-// following the skill contract read window.APPLICATION.configuration.datasets
-// identically in dev and in prod.
-const enrichDataset = async (dataset) => {
-    if (!dataset?.id)
-        return dataset;
-    const cached = datasetsCache.get(dataset.id);
-    if (cached && Date.now() - cached.fetchedAt < DATASETS_CACHE_TTL) {
-        return cached.data ? { ...dataset, ...cached.data } : dataset;
-    }
-    try {
-        const fresh = await remoteFetch('/datasets/' + encodeURIComponent(dataset.id));
-        const data = {};
-        for (const prop of INJECTED_DATASET_PROPS) {
-            if (fresh[prop] !== undefined)
-                data[prop] = fresh[prop];
-        }
-        data.userPermissions = fresh.userPermissions ?? [];
-        datasetsCache.set(dataset.id, { data, fetchedAt: Date.now() });
-        return { ...dataset, ...data };
-    }
-    catch (err) {
-        // a private dataset without an api key, or a network failure: keep the raw
-        // configuration entry so the app still loads, and warn in the dev-server UI
-        console.warn('[dev-server] failed to enrich dataset ' + dataset.id + ', keeping raw configuration entry', err);
-        datasetsCache.set(dataset.id, { data: null, fetchedAt: Date.now() });
-        return dataset;
-    }
-};
-// Enrich the datasets from the remote data-fair, then rewrite every remote origin to ours.
-// The rewrite is applied even when there is no dataset to enrich: a configuration can carry
-// remote urls anywhere (logos, links, tileserver styles), not only in datasets[].href.
-const prepareConfig = async (configuration) => {
-    const datasets = configuration?.datasets?.filter((d) => !!d);
-    const enriched = datasets?.length
-        ? { ...configuration, datasets: await Promise.all(datasets.map(enrichDataset)) }
-        : configuration;
-    return localizeConfig(enriched, new URL(config.dataFair.url).origin, `http://localhost:${config.port}`);
-};
+// Enrich the datasets from the remote data-fair and rewrite every remote origin to ours
+// (pure helpers in enrich.ts, wired here to our remote api and local origins).
+const prepareConfig = (configuration) => prepareRemoteConfig(configuration, {
+    fetchJson: remoteFetch,
+    localize: localizeConfig,
+    remoteOrigin: new URL(config.dataFair.url).origin,
+    localOrigin: `http://localhost:${config.port}`
+});
 // read the .dev-config.json file of the app under development
 const readDevConfig = () => existsSync('.dev-config.json') ? JSON.parse(readFileSync('.dev-config.json', 'utf8')) : {};
 // Reproduce the waiting strategy of the capture service (capture/api/utils/page.ts) so that a
@@ -368,7 +312,7 @@ const minorVersion = (version) => version.split('.').slice(0, 2).join('.');
 // list configurations of the app under development that exist on the remote data-fair
 // with the same minor version, useful to test quickly for regressions
 app.get('/configurations', async (req, res) => {
-    const { name, version } = await localAppInfo();
+    const { name, version, packageName } = await localAppInfo();
     const response = { appName: name, localVersion: version, remoteUrl: config.dataFair.url };
     if (!name) {
         res.status(400).send({ ...response, error: 'The "application-name" meta tag was not found in the local app index.html' });
@@ -379,10 +323,9 @@ app.get('/configurations', async (req, res) => {
         return;
     }
     response.minorVersion = minorVersion(version);
-    // a base application restricted to an organization is invisible to an anonymous request, and
-    // without it the listing below finds nothing at all — the applications are then never reached,
-    // however public they are. privateAccess is what makes it visible, and data-fair answers 401
-    // to it without credentials, so it is only sent when an api key is configured.
+    // a base application restricted to an organization is invisible to an anonymous request.
+    // privateAccess is what makes it visible, and data-fair answers 401 to it without
+    // credentials, so it is only sent when an api key is configured.
     response.authenticated = !!config.dataFair.apiKey;
     try {
         let baseAppsQuery = '/base-applications?applicationName=' + encodeURIComponent(name) + '&size=1000&count=false';
@@ -390,16 +333,21 @@ app.get('/configurations', async (req, res) => {
             baseAppsQuery += '&privateAccess=' + encodeURIComponent(ownerFilter());
         const baseApps = await remoteFetch(baseAppsQuery);
         const matchingBaseApps = (baseApps.results ?? []).filter((b) => typeof b.version === 'string' && minorVersion(b.version) === response.minorVersion);
-        // told apart from "the base application exists but carries no application": the UI has no
-        // other way to explain an empty list, and a missing api key is by far the likeliest cause
-        response.baseAppFound = matchingBaseApps.length > 0;
-        response.results = [];
-        for (const baseApp of matchingBaseApps) {
-            const applications = await remoteFetch('/applications?base-application=' + encodeURIComponent(baseApp.url) + '&size=1000&count=false&select=id,title,owner');
-            for (const application of applications.results ?? []) {
-                response.results.push({ id: application.id, title: application.title, owner: application.owner, baseAppVersion: baseApp.version });
-            }
-        }
+        // Not finding the base application is not the end of the search: it may simply be private
+        // while the applications running on it are public, and those are reachable as soon as its
+        // url is known. Guessing the url costs nothing — the filter below is an exact match.
+        const versionByUrl = new Map(matchingBaseApps.map((b) => [b.url, b.version]));
+        const baseAppUrls = matchingBaseApps.length
+            ? [...versionByUrl.keys()]
+            : candidateBaseAppUrls(config.dataFair.url, name, packageName, response.minorVersion);
+        // one request for the whole set: the data-fair list filters accept comma separated values
+        const applications = await remoteFetch('/applications?base-application=' + encodeURIComponent(baseAppUrls.join(',')) + '&size=1000&count=false&select=id,title,owner,url');
+        response.results = (applications.results ?? []).map((application) => ({
+            id: application.id,
+            title: application.title,
+            owner: application.owner,
+            baseAppVersion: versionByUrl.get(application.url) ?? response.minorVersion
+        }));
         response.results.sort((a, b) => (a.title || '').localeCompare(b.title || ''));
         res.send(response);
     }
@@ -412,13 +360,16 @@ app.get('/configurations', async (req, res) => {
 app.get('/configurations/:id', async (req, res) => {
     const id = encodeURIComponent(req.params.id);
     try {
-        // Sent as-is, with its remote origins: the UI stores this straight into .dev-config.json,
-        // which must stay portable. Origins are rewritten on the read path, in prepareConfig.
-        const configuration = await remoteFetch('/applications/' + id + '/configuration');
-        // The attachments come along: a configuration references them by name only, so without the
-        // files a copied production configuration renders with every image broken. Fetched from the
-        // application itself rather than from the configuration, which never lists them.
-        const application = await remoteFetch('/applications/' + id + '?select=attachments');
+        // The configuration is sent as-is, with its remote origins: the UI stores it straight into
+        // .dev-config.json, which must stay portable. Origins are rewritten on the read path, in
+        // prepareConfig. The attachments come along, since a configuration references them by name
+        // only and without the files a copied production configuration renders with every image
+        // broken; they are read from the application, which is where data-fair lists them.
+        // Neither read depends on the other, so they leave together instead of one after the other.
+        const [configuration, application] = await Promise.all([
+            remoteFetch('/applications/' + id + '/configuration'),
+            remoteFetch('/applications/' + id + '?select=attachments')
+        ]);
         const attachments = await copyAttachments(application.attachments ?? [], (name) => remoteFetchBuffer('/applications/' + id + '/attachments/' + encodeURIComponent(name)));
         if (attachments.failed.length) {
             console.warn('[dev-server] failed to copy attachments ' + attachments.failed.join(', ') + ', images referencing them will be broken');

package/src/attachments.js

@@ -82,6 +82,11 @@ export const attachmentPath = (dir, name) => {
         return undefined;
     return filePath;
 };
+// Attachments used to be downloaded one after the other, each waiting for a full round trip
+// to the remote data-fair before the next one started, so an application carrying a dozen
+// images cost a dozen round trips to copy. They go in parallel now, bounded so that a
+// configuration with many attachments does not open as many connections at once.
+const DOWNLOAD_CONCURRENCY = 5;
 // Replace the whole local attachments directory with the attachments of a remote application.
 // Replace, and not merge: copying a configuration replaces the current one entirely, so leaving
 // behind the images of the previously copied application would only produce a directory whose
@@ -93,32 +98,43 @@ export const attachmentPath = (dir, name) => {
 // where nothing downloads does empty the directory, which is the point: the configuration it
 // came with is being applied all the same, and its images are genuinely missing.
 export const copyAttachments = async (attachments, fetchAttachment, dir = ATTACHMENTS_DIR) => {
-    const result = { copied: [], failed: [] };
-    const downloaded = [];
-    for (const attachment of attachments ?? []) {
-        // a name with a path separator would write outside the directory, and data-fair has no
-        // reason to send one: skip it rather than sanitize a name the configuration still refers to
-        if (!attachment?.name || attachmentPath(dir, attachment.name) === undefined) {
-            if (attachment?.name)
-                result.failed.push(attachment.name);
-            continue;
+    const entries = attachments ?? [];
+    // indexed rather than appended: downloads no longer finish in the order they started, and
+    // both lists below are read by a developer against the configuration they came from
+    const downloaded = new Array(entries.length);
+    const failures = new Array(entries.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < entries.length) {
+            const index = next++;
+            const attachment = entries[index];
+            // a name with a path separator would write outside the directory, and data-fair has no
+            // reason to send one: skip it rather than sanitize a name the configuration still refers to
+            if (!attachment?.name || attachmentPath(dir, attachment.name) === undefined) {
+                if (attachment?.name)
+                    failures[index] = attachment.name;
+                continue;
+            }
+            try {
+                downloaded[index] = { attachment: { ...attachment, name: attachment.name }, body: await fetchAttachment(attachment.name) };
+            }
+            catch (err) {
+                failures[index] = attachment.name;
+            }
         }
-        try {
-            downloaded.push({ attachment: { ...attachment, name: attachment.name }, body: await fetchAttachment(attachment.name) });
-        }
-        catch (err) {
-            result.failed.push(attachment.name);
-        }
-    }
+    };
+    await Promise.all(Array.from({ length: Math.min(DOWNLOAD_CONCURRENCY, entries.length) }, worker));
+    const kept = downloaded.filter(entry => entry !== undefined);
+    const result = { copied: [], failed: failures.filter(name => name !== undefined) };
     // written aside then swapped in, so an interrupted copy leaves the previous directory intact
     const tmpDir = dir + '.tmp';
     rmSync(tmpDir, { recursive: true, force: true });
     mkdirSync(tmpDir, { recursive: true });
-    for (const { attachment, body } of downloaded) {
+    for (const { attachment, body } of kept) {
         writeFileSync(join(tmpDir, attachment.name), body);
         result.copied.push(attachment.name);
     }
-    writeFileSync(join(tmpDir, METADATA_FILE), JSON.stringify(downloaded.map(d => d.attachment), null, 2));
+    writeFileSync(join(tmpDir, METADATA_FILE), JSON.stringify(kept.map(d => d.attachment), null, 2));
     rmSync(dir, { recursive: true, force: true });
     renameSync(tmpDir, dir);
     return result;

package/src/base-app-urls.d.ts

@@ -0,0 +1 @@
+export declare const candidateBaseAppUrls: (dataFairUrl: string, appName: string, packageName: string | undefined, minorVersion: string) => string[];

package/src/base-app-urls.js

@@ -0,0 +1,37 @@
+// Candidate urls of the base application under development.
+//
+// Discovering it by name is the authoritative path, but it only works for a base application
+// the request is allowed to see: one restricted to an organization is invisible to an anonymous
+// request, while the applications running on it are often public — which is precisely the case
+// of an application developed for a single customer. The /applications listing filters on an
+// exact url, so a wrong candidate returns nothing and a right one returns the applications: a
+// guess can never produce a false match, and they all fit in a single request.
+//
+// The two conventions published base applications follow, as of today:
+//   https://koumoul.com/apps/<slug>/<minor>/
+//   https://cdn.jsdelivr.net/npm/@scope/<package>@<minor>/dist/
+// The slug is the repository name, which is not always the application name — "Liste et fiches"
+// is served under /apps/list-details/ — so both are tried, along with the package name stripped
+// of the prefixes repositories use.
+const slugs = (appName, packageName) => {
+    const candidates = [appName];
+    if (packageName) {
+        const unscoped = packageName.startsWith('@') ? packageName.slice(packageName.indexOf('/') + 1) : packageName;
+        candidates.push(unscoped, unscoped.replace(/^(app|data-fair)-/, ''));
+    }
+    return [...new Set(candidates.filter(Boolean))];
+};
+export const candidateBaseAppUrls = (dataFairUrl, appName, packageName, minorVersion) => {
+    const urls = [];
+    // the same origin as the data-fair the dev-server talks to: a base application is served
+    // beside it, never on some unrelated host
+    const origin = new URL(dataFairUrl).origin;
+    for (const slug of slugs(appName, packageName)) {
+        urls.push(`${origin}/apps/${slug}/${minorVersion}/`);
+    }
+    if (packageName) {
+        const scoped = packageName.startsWith('@') ? packageName : '@data-fair/' + packageName;
+        urls.push(`https://cdn.jsdelivr.net/npm/${scoped}@${minorVersion}/dist/`);
+    }
+    return urls;
+};

package/src/enrich.d.ts

@@ -0,0 +1,9 @@
+export declare const INJECTED_DATASET_PROPS: readonly ["id", "href", "page", "title", "slug", "finalizedAt", "schema", "isRest", "userPermissions"];
+export declare const enrichDataset: (dataset: any, fetchJson: (path: string) => Promise<any>) => Promise<any>;
+export interface PrepareConfigDeps {
+    fetchJson: (path: string) => Promise<any>;
+    localize: (configuration: any, remoteOrigin: string, localOrigin: string) => any;
+    remoteOrigin: string;
+    localOrigin: string;
+}
+export declare const prepareConfig: (configuration: any, deps: PrepareConfigDeps) => Promise<any>;

package/src/enrich.js

@@ -0,0 +1,57 @@
+// Rebuild the configuration dataset entries the same way data-fair does in production
+// (refreshConfigDatasetsRefs in api/src/applications/utils.ts): an application stores only
+// minimal dataset references, data-fair injects the full schema (with concepts), finalizedAt,
+// slug, isRest and the per-request userPermissions at serving time. We reproduce that so
+// applications following the skill contract read window.APPLICATION.configuration.datasets
+// identically in dev and in prod.
+//
+// Pure module, no imports: it takes its remote accessor and origin rewrite as dependencies,
+// wired in app.ts — so it can be unit tested without a build (test/enrich.test.ts).
+// Short-lived cache for the dataset enrichment below, so that every preview reload does not
+// hammer the remote data-fair API. Failures are cached too: a private dataset without an api
+// key would otherwise be re-fetched, and re-warned about, on every single reload.
+const datasetsCache = new Map();
+const DATASETS_CACHE_TTL = 30_000;
+// The set of properties data-fair injects in a configuration dataset entry. Sticking to it
+// matters: forwarding the whole remote dataset would let an application rely, in dev, on a
+// property that production never sends. data-fair refreshes the keys stored in the entry plus
+// the `select` of the app's config schema dataset selector — this list covers that contract,
+// including `isRest`, carried by the select of editing apps and used as the gate deciding
+// whether rest line editing (POST/PATCH/DELETE /lines) is offered at all.
+export const INJECTED_DATASET_PROPS = ['id', 'href', 'page', 'title', 'slug', 'finalizedAt', 'schema', 'isRest', 'userPermissions'];
+export const enrichDataset = async (dataset, fetchJson) => {
+    if (!dataset?.id)
+        return dataset;
+    const cached = datasetsCache.get(dataset.id);
+    if (cached && Date.now() - cached.fetchedAt < DATASETS_CACHE_TTL) {
+        return cached.data ? { ...dataset, ...cached.data } : dataset;
+    }
+    try {
+        const fresh = await fetchJson('/datasets/' + encodeURIComponent(dataset.id));
+        const data = {};
+        for (const prop of INJECTED_DATASET_PROPS) {
+            if (fresh[prop] !== undefined)
+                data[prop] = fresh[prop];
+        }
+        data.userPermissions = fresh.userPermissions ?? [];
+        datasetsCache.set(dataset.id, { data, fetchedAt: Date.now() });
+        return { ...dataset, ...data };
+    }
+    catch (err) {
+        // a private dataset without an api key, or a network failure: keep the raw
+        // configuration entry so the app still loads, and warn in the dev-server UI
+        console.warn('[dev-server] failed to enrich dataset ' + dataset.id + ', keeping raw configuration entry', err);
+        datasetsCache.set(dataset.id, { data: null, fetchedAt: Date.now() });
+        return dataset;
+    }
+};
+// Enrich the datasets from the remote data-fair, then rewrite every remote origin to ours.
+// The rewrite is applied even when there is no dataset to enrich: a configuration can carry
+// remote urls anywhere (logos, links, tileserver styles), not only in datasets[].href.
+export const prepareConfig = async (configuration, deps) => {
+    const datasets = configuration?.datasets?.filter((d) => !!d);
+    const enriched = datasets?.length
+        ? { ...configuration, datasets: await Promise.all(datasets.map(d => enrichDataset(d, deps.fetchJson))) }
+        : configuration;
+    return deps.localize(enriched, deps.remoteOrigin, deps.localOrigin);
+};

package/src/remote.d.ts

@@ -0,0 +1,3 @@
+export declare const remoteFetchRaw: (path: string) => Promise<Response>;
+export declare const remoteFetch: (path: string) => Promise<any>;
+export declare const remoteFetchBuffer: (path: string) => Promise<Buffer<ArrayBuffer>>;

package/src/remote.js

@@ -0,0 +1,22 @@
+// Access to the remote data-fair api. Only JSON and buffer reads: every mutation goes
+// through data-fair itself, the dev-server never writes on behalf of the developer.
+import config from './config.js';
+// Fetch a resource from the remote data-fair api
+export const remoteFetchRaw = async (path) => {
+    const res = await fetch(config.dataFair.url + '/api/v1' + path, {
+        headers: config.dataFair.apiKey ? { 'x-apiKey': config.dataFair.apiKey } : {}
+    });
+    if (!res.ok)
+        throw new Error(`error ${res.status} on remote data-fair: ${await res.text()}`);
+    return res;
+};
+export const remoteFetch = async (path) => {
+    const res = await remoteFetchRaw(path);
+    return res.json();
+};
+// Same, for an attachment: a file, never json, and never decoded as text — see the app proxy
+// below, where decoding a binary body as utf8 destroys it.
+export const remoteFetchBuffer = async (path) => {
+    const res = await remoteFetchRaw(path);
+    return Buffer.from(await res.arrayBuffer());
+};