@data-fair/dev-server 2.5.2 → 2.6.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/dev-server",
-  "version": "2.5.2",
+  "version": "2.6.1",
   "description": "A development server for optimal development experience of data-fair applications.",
   "main": "src/index.js",
   "bin": {

package/src/app.js

@@ -2,6 +2,8 @@
 import config from './config.js';
 import uiConfig from './ui-config.js';
 import { localizeConfig } from './localize.js';
+import { remoteFetch, remoteFetchBuffer } from './remote.js';
+import { prepareConfig as prepareRemoteConfig } from './enrich.js';
 import { ATTACHMENTS_DIR, attachmentPath, copyAttachments, listAttachments } from './attachments.js';
 import { candidateBaseAppUrls } from './base-app-urls.js';
 import { WebSocket, WebSocketServer } from 'ws';
@@ -171,76 +173,14 @@ const ownerFilter = () => {
         filter += ':' + owner.department;
     return filter;
 };
-// Fetch a resource from the remote data-fair api
-const remoteFetch = async (path) => {
-    const res = await remoteFetchRaw(path);
-    return res.json();
-};
-// Same, for an attachment: a file, never json, and never decoded as text — see the app proxy
-// below, where decoding a binary body as utf8 destroys it.
-const remoteFetchBuffer = async (path) => {
-    const res = await remoteFetchRaw(path);
-    return Buffer.from(await res.arrayBuffer());
-};
-const remoteFetchRaw = async (path) => {
-    const res = await fetch(config.dataFair.url + '/api/v1' + path, {
-        headers: config.dataFair.apiKey ? { 'x-apiKey': config.dataFair.apiKey } : {}
-    });
-    if (!res.ok)
-        throw new Error(`error ${res.status} on remote data-fair: ${await res.text()}`);
-    return res;
-};
-// Short-lived cache for the dataset enrichment below, so that every preview reload does not
-// hammer the remote data-fair API. Failures are cached too: a private dataset without an api
-// key would otherwise be re-fetched, and re-warned about, on every single reload.
-const datasetsCache = new Map();
-const DATASETS_CACHE_TTL = 30_000;
-// The exact set of properties data-fair injects in a configuration dataset entry. Sticking to
-// it matters: forwarding the whole remote dataset would let an application rely, in dev, on a
-// property that production never sends.
-const INJECTED_DATASET_PROPS = ['id', 'href', 'page', 'title', 'slug', 'finalizedAt', 'schema', 'userPermissions'];
-// Rebuild a configuration dataset entry the same way data-fair does in production
-// (refreshConfigDatasetsRefs in api/src/applications/utils.ts): the app stores only
-// minimal references, data-fair injects the full schema (with concepts), finalizedAt,
-// slug and userPermissions at request time. We reproduce that so applications
-// following the skill contract read window.APPLICATION.configuration.datasets
-// identically in dev and in prod.
-const enrichDataset = async (dataset) => {
-    if (!dataset?.id)
-        return dataset;
-    const cached = datasetsCache.get(dataset.id);
-    if (cached && Date.now() - cached.fetchedAt < DATASETS_CACHE_TTL) {
-        return cached.data ? { ...dataset, ...cached.data } : dataset;
-    }
-    try {
-        const fresh = await remoteFetch('/datasets/' + encodeURIComponent(dataset.id));
-        const data = {};
-        for (const prop of INJECTED_DATASET_PROPS) {
-            if (fresh[prop] !== undefined)
-                data[prop] = fresh[prop];
-        }
-        data.userPermissions = fresh.userPermissions ?? [];
-        datasetsCache.set(dataset.id, { data, fetchedAt: Date.now() });
-        return { ...dataset, ...data };
-    }
-    catch (err) {
-        // a private dataset without an api key, or a network failure: keep the raw
-        // configuration entry so the app still loads, and warn in the dev-server UI
-        console.warn('[dev-server] failed to enrich dataset ' + dataset.id + ', keeping raw configuration entry', err);
-        datasetsCache.set(dataset.id, { data: null, fetchedAt: Date.now() });
-        return dataset;
-    }
-};
-// Enrich the datasets from the remote data-fair, then rewrite every remote origin to ours.
-// The rewrite is applied even when there is no dataset to enrich: a configuration can carry
-// remote urls anywhere (logos, links, tileserver styles), not only in datasets[].href.
-const prepareConfig = async (configuration) => {
-    const datasets = configuration?.datasets?.filter((d) => !!d);
-    const enriched = datasets?.length
-        ? { ...configuration, datasets: await Promise.all(datasets.map(enrichDataset)) }
-        : configuration;
-    return localizeConfig(enriched, new URL(config.dataFair.url).origin, `http://localhost:${config.port}`);
-};
+// Enrich the datasets from the remote data-fair and rewrite every remote origin to ours
+// (pure helpers in enrich.ts, wired here to our remote api and local origins).
+const prepareConfig = (configuration) => prepareRemoteConfig(configuration, {
+    fetchJson: remoteFetch,
+    localize: localizeConfig,
+    remoteOrigin: new URL(config.dataFair.url).origin,
+    localOrigin: `http://localhost:${config.port}`
+});
 // read the .dev-config.json file of the app under development
 const readDevConfig = () => existsSync('.dev-config.json') ? JSON.parse(readFileSync('.dev-config.json', 'utf8')) : {};
 // Reproduce the waiting strategy of the capture service (capture/api/utils/page.ts) so that a
@@ -420,13 +360,16 @@ app.get('/configurations', async (req, res) => {
 app.get('/configurations/:id', async (req, res) => {
     const id = encodeURIComponent(req.params.id);
     try {
-        // Sent as-is, with its remote origins: the UI stores this straight into .dev-config.json,
-        // which must stay portable. Origins are rewritten on the read path, in prepareConfig.
-        const configuration = await remoteFetch('/applications/' + id + '/configuration');
-        // The attachments come along: a configuration references them by name only, so without the
-        // files a copied production configuration renders with every image broken. Fetched from the
-        // application itself rather than from the configuration, which never lists them.
-        const application = await remoteFetch('/applications/' + id + '?select=attachments');
+        // The configuration is sent as-is, with its remote origins: the UI stores it straight into
+        // .dev-config.json, which must stay portable. Origins are rewritten on the read path, in
+        // prepareConfig. The attachments come along, since a configuration references them by name
+        // only and without the files a copied production configuration renders with every image
+        // broken; they are read from the application, which is where data-fair lists them.
+        // Neither read depends on the other, so they leave together instead of one after the other.
+        const [configuration, application] = await Promise.all([
+            remoteFetch('/applications/' + id + '/configuration'),
+            remoteFetch('/applications/' + id + '?select=attachments')
+        ]);
         const attachments = await copyAttachments(application.attachments ?? [], (name) => remoteFetchBuffer('/applications/' + id + '/attachments/' + encodeURIComponent(name)));
         if (attachments.failed.length) {
             console.warn('[dev-server] failed to copy attachments ' + attachments.failed.join(', ') + ', images referencing them will be broken');

package/src/attachments.js

@@ -82,6 +82,11 @@ export const attachmentPath = (dir, name) => {
         return undefined;
     return filePath;
 };
+// Attachments used to be downloaded one after the other, each waiting for a full round trip
+// to the remote data-fair before the next one started, so an application carrying a dozen
+// images cost a dozen round trips to copy. They go in parallel now, bounded so that a
+// configuration with many attachments does not open as many connections at once.
+const DOWNLOAD_CONCURRENCY = 5;
 // Replace the whole local attachments directory with the attachments of a remote application.
 // Replace, and not merge: copying a configuration replaces the current one entirely, so leaving
 // behind the images of the previously copied application would only produce a directory whose
@@ -93,32 +98,43 @@ export const attachmentPath = (dir, name) => {
 // where nothing downloads does empty the directory, which is the point: the configuration it
 // came with is being applied all the same, and its images are genuinely missing.
 export const copyAttachments = async (attachments, fetchAttachment, dir = ATTACHMENTS_DIR) => {
-    const result = { copied: [], failed: [] };
-    const downloaded = [];
-    for (const attachment of attachments ?? []) {
-        // a name with a path separator would write outside the directory, and data-fair has no
-        // reason to send one: skip it rather than sanitize a name the configuration still refers to
-        if (!attachment?.name || attachmentPath(dir, attachment.name) === undefined) {
-            if (attachment?.name)
-                result.failed.push(attachment.name);
-            continue;
+    const entries = attachments ?? [];
+    // indexed rather than appended: downloads no longer finish in the order they started, and
+    // both lists below are read by a developer against the configuration they came from
+    const downloaded = new Array(entries.length);
+    const failures = new Array(entries.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < entries.length) {
+            const index = next++;
+            const attachment = entries[index];
+            // a name with a path separator would write outside the directory, and data-fair has no
+            // reason to send one: skip it rather than sanitize a name the configuration still refers to
+            if (!attachment?.name || attachmentPath(dir, attachment.name) === undefined) {
+                if (attachment?.name)
+                    failures[index] = attachment.name;
+                continue;
+            }
+            try {
+                downloaded[index] = { attachment: { ...attachment, name: attachment.name }, body: await fetchAttachment(attachment.name) };
+            }
+            catch (err) {
+                failures[index] = attachment.name;
+            }
         }
-        try {
-            downloaded.push({ attachment: { ...attachment, name: attachment.name }, body: await fetchAttachment(attachment.name) });
-        }
-        catch (err) {
-            result.failed.push(attachment.name);
-        }
-    }
+    };
+    await Promise.all(Array.from({ length: Math.min(DOWNLOAD_CONCURRENCY, entries.length) }, worker));
+    const kept = downloaded.filter(entry => entry !== undefined);
+    const result = { copied: [], failed: failures.filter(name => name !== undefined) };
     // written aside then swapped in, so an interrupted copy leaves the previous directory intact
     const tmpDir = dir + '.tmp';
     rmSync(tmpDir, { recursive: true, force: true });
     mkdirSync(tmpDir, { recursive: true });
-    for (const { attachment, body } of downloaded) {
+    for (const { attachment, body } of kept) {
         writeFileSync(join(tmpDir, attachment.name), body);
         result.copied.push(attachment.name);
     }
-    writeFileSync(join(tmpDir, METADATA_FILE), JSON.stringify(downloaded.map(d => d.attachment), null, 2));
+    writeFileSync(join(tmpDir, METADATA_FILE), JSON.stringify(kept.map(d => d.attachment), null, 2));
     rmSync(dir, { recursive: true, force: true });
     renameSync(tmpDir, dir);
     return result;

package/src/enrich.d.ts

@@ -0,0 +1,9 @@
+export declare const INJECTED_DATASET_PROPS: readonly ["id", "href", "page", "title", "slug", "finalizedAt", "schema", "isRest", "userPermissions"];
+export declare const enrichDataset: (dataset: any, fetchJson: (path: string) => Promise<any>) => Promise<any>;
+export interface PrepareConfigDeps {
+    fetchJson: (path: string) => Promise<any>;
+    localize: (configuration: any, remoteOrigin: string, localOrigin: string) => any;
+    remoteOrigin: string;
+    localOrigin: string;
+}
+export declare const prepareConfig: (configuration: any, deps: PrepareConfigDeps) => Promise<any>;

package/src/enrich.js

@@ -0,0 +1,57 @@
+// Rebuild the configuration dataset entries the same way data-fair does in production
+// (refreshConfigDatasetsRefs in api/src/applications/utils.ts): an application stores only
+// minimal dataset references, data-fair injects the full schema (with concepts), finalizedAt,
+// slug, isRest and the per-request userPermissions at serving time. We reproduce that so
+// applications following the skill contract read window.APPLICATION.configuration.datasets
+// identically in dev and in prod.
+//
+// Pure module, no imports: it takes its remote accessor and origin rewrite as dependencies,
+// wired in app.ts — so it can be unit tested without a build (test/enrich.test.ts).
+// Short-lived cache for the dataset enrichment below, so that every preview reload does not
+// hammer the remote data-fair API. Failures are cached too: a private dataset without an api
+// key would otherwise be re-fetched, and re-warned about, on every single reload.
+const datasetsCache = new Map();
+const DATASETS_CACHE_TTL = 30_000;
+// The set of properties data-fair injects in a configuration dataset entry. Sticking to it
+// matters: forwarding the whole remote dataset would let an application rely, in dev, on a
+// property that production never sends. data-fair refreshes the keys stored in the entry plus
+// the `select` of the app's config schema dataset selector — this list covers that contract,
+// including `isRest`, carried by the select of editing apps and used as the gate deciding
+// whether rest line editing (POST/PATCH/DELETE /lines) is offered at all.
+export const INJECTED_DATASET_PROPS = ['id', 'href', 'page', 'title', 'slug', 'finalizedAt', 'schema', 'isRest', 'userPermissions'];
+export const enrichDataset = async (dataset, fetchJson) => {
+    if (!dataset?.id)
+        return dataset;
+    const cached = datasetsCache.get(dataset.id);
+    if (cached && Date.now() - cached.fetchedAt < DATASETS_CACHE_TTL) {
+        return cached.data ? { ...dataset, ...cached.data } : dataset;
+    }
+    try {
+        const fresh = await fetchJson('/datasets/' + encodeURIComponent(dataset.id));
+        const data = {};
+        for (const prop of INJECTED_DATASET_PROPS) {
+            if (fresh[prop] !== undefined)
+                data[prop] = fresh[prop];
+        }
+        data.userPermissions = fresh.userPermissions ?? [];
+        datasetsCache.set(dataset.id, { data, fetchedAt: Date.now() });
+        return { ...dataset, ...data };
+    }
+    catch (err) {
+        // a private dataset without an api key, or a network failure: keep the raw
+        // configuration entry so the app still loads, and warn in the dev-server UI
+        console.warn('[dev-server] failed to enrich dataset ' + dataset.id + ', keeping raw configuration entry', err);
+        datasetsCache.set(dataset.id, { data: null, fetchedAt: Date.now() });
+        return dataset;
+    }
+};
+// Enrich the datasets from the remote data-fair, then rewrite every remote origin to ours.
+// The rewrite is applied even when there is no dataset to enrich: a configuration can carry
+// remote urls anywhere (logos, links, tileserver styles), not only in datasets[].href.
+export const prepareConfig = async (configuration, deps) => {
+    const datasets = configuration?.datasets?.filter((d) => !!d);
+    const enriched = datasets?.length
+        ? { ...configuration, datasets: await Promise.all(datasets.map(d => enrichDataset(d, deps.fetchJson))) }
+        : configuration;
+    return deps.localize(enriched, deps.remoteOrigin, deps.localOrigin);
+};

package/src/remote.d.ts

@@ -0,0 +1,3 @@
+export declare const remoteFetchRaw: (path: string) => Promise<Response>;
+export declare const remoteFetch: (path: string) => Promise<any>;
+export declare const remoteFetchBuffer: (path: string) => Promise<Buffer<ArrayBuffer>>;

package/src/remote.js

@@ -0,0 +1,22 @@
+// Access to the remote data-fair api. Only JSON and buffer reads: every mutation goes
+// through data-fair itself, the dev-server never writes on behalf of the developer.
+import config from './config.js';
+// Fetch a resource from the remote data-fair api
+export const remoteFetchRaw = async (path) => {
+    const res = await fetch(config.dataFair.url + '/api/v1' + path, {
+        headers: config.dataFair.apiKey ? { 'x-apiKey': config.dataFair.apiKey } : {}
+    });
+    if (!res.ok)
+        throw new Error(`error ${res.status} on remote data-fair: ${await res.text()}`);
+    return res;
+};
+export const remoteFetch = async (path) => {
+    const res = await remoteFetchRaw(path);
+    return res.json();
+};
+// Same, for an attachment: a file, never json, and never decoded as text — see the app proxy
+// below, where decoding a binary body as utf8 destroys it.
+export const remoteFetchBuffer = async (path) => {
+    const res = await remoteFetchRaw(path);
+    return Buffer.from(await res.arrayBuffer());
+};