@data-fair/dev-server 2.5.0 → 2.5.1

package/README.md

@@ -42,6 +42,39 @@ partagent rien d'autre que le fichier `.env`, écrit une seule fois par
 à la racine) il faut régénérer `.env` avant de lancer le dev-server :
 `node src/dev-env.js --force --app-path=`.
 
+## Configurations et pièces jointes distantes
+
+Le bouton « Copier une configuration distante » liste les applications qui, sur
+le data-fair distant, tournent sur l'application de base en cours de
+développement, dans la même version mineure. La recherche part du nom lu dans la
+balise `<meta name="application-name">` de l'`index.html` local et de la version
+du `package.json`.
+
+Une application de base réservée à une organisation (`privateAccess`) est
+invisible à une requête anonyme : sans elle la liste est vide, quand bien même
+les applications qui l'utilisent seraient publiques. Le paramètre
+`privateAccess` la rend visible, mais data-fair répond 401 sans
+authentification — il n'est donc envoyé que si `DATAFAIR_API_KEY` est
+renseignée. Une clé d'API se crée depuis le compte ou l'organisation sur le
+data-fair distant, et se met dans le `.env` de l'application :
+
+```
+DATAFAIR_API_KEY=xxx
+```
+
+Ce `.env` est celui que `df-dev-env` génère, et qu'il laisse intact tant qu'on
+ne passe pas `--force` — qui, lui, le réécrit entièrement et emporte la clé.
+
+La copie ramène aussi les pièces jointes de l'application distante dans
+`.dev-attachments/` (git-ignoré). Une configuration ne référence une pièce
+jointe que par son nom et reconstruit son URL à l'affichage
+(`application.href + '/attachments/' + name`) : sans les fichiers, une
+configuration de production copiée s'affiche avec toutes ses images cassées.
+Elles sont servies sous `/config/attachments/`, listées dans
+`window.APPLICATION.attachments`, et proposées par le formulaire de
+configuration comme data-fair le fait avec `context.attachments`. Un fichier
+déposé à la main dans `.dev-attachments/` est listé et servi de la même façon.
+
 ## Development
 
 Run development server :

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/dev-server",
-  "version": "2.5.0",
+  "version": "2.5.1",
   "description": "A development server for optimal development experience of data-fair applications.",
   "main": "src/index.js",
   "bin": {

package/src/app.js

@@ -2,6 +2,7 @@
 import config from './config.js';
 import uiConfig from './ui-config.js';
 import { localizeConfig } from './localize.js';
+import { ATTACHMENTS_DIR, attachmentPath, copyAttachments, listAttachments } from './attachments.js';
 import { WebSocket, WebSocketServer } from 'ws';
 import { createServer } from 'node:http';
 import express from 'express';
@@ -15,7 +16,7 @@ import { isElementNode, createTextNode, createElement, appendChild } from '@pars
 import escapeStringRegexp from 'escape-string-regexp';
 import eventPromise from '@data-fair/lib-utils/event-promise.js';
 import { createSpaMiddleware } from '@data-fair/lib-express/serve-spa.js';
-import { resolve } from 'node:path';
+import { join, resolve } from 'node:path';
 const debug = debugModule('df-dev-server');
 const app = express();
 const server = createServer(app);
@@ -110,6 +111,27 @@ app.post('/config/error', (req, res) => {
     }
     res.send();
 });
+// The attachments of the application under development, read by the dev-server UI to feed the
+// `context.attachments` of the configuration form — data-fair feeds it with
+// application.attachments, and an application configuration references an attachment by name.
+app.get('/config/attachments', (req, res) => {
+    res.send(listAttachments());
+});
+// window.APPLICATION.href points at /config, so an application building an attachment url the
+// production way — application.href + '/attachments/' + name — lands here. Same path shape as
+// data-fair (applications/:id/attachments/*), same no-cache: a re-copied file shows up at once.
+app.get('/config/attachments/*attachmentPath', (req, res) => {
+    const relFilePath = join(...req.params.attachmentPath);
+    const filePath = attachmentPath(ATTACHMENTS_DIR, relFilePath);
+    if (!filePath || !existsSync(filePath)) {
+        res.status(404).send({ error: `attachment "${relFilePath}" not found in ${ATTACHMENTS_DIR}` });
+        return;
+    }
+    res.setHeader('Cache-Control', 'no-cache');
+    // served from a root, and not from the absolute path: send() refuses any path holding a
+    // hidden segment, and the attachments directory is itself a dotted one
+    res.sendFile(relFilePath, { root: resolve(ATTACHMENTS_DIR) });
+});
 // Identification of the app under development: its name is read from the
 // "application-name" meta tag of the local index.html and its version from the
 // package.json of the current directory.
@@ -136,14 +158,33 @@ const localAppInfo = async () => {
     }
     return { name, version };
 };
+// The owner the dev-server works on behalf of, in the "type:id[:department]" form the
+// data-fair api expects for its owner and privateAccess filters.
+const ownerFilter = () => {
+    const owner = config.dataFair.owner;
+    let filter = `${owner.type}:${owner.id}`;
+    if (owner.department)
+        filter += ':' + owner.department;
+    return filter;
+};
 // Fetch a resource from the remote data-fair api
 const remoteFetch = async (path) => {
+    const res = await remoteFetchRaw(path);
+    return res.json();
+};
+// Same, for an attachment: a file, never json, and never decoded as text — see the app proxy
+// below, where decoding a binary body as utf8 destroys it.
+const remoteFetchBuffer = async (path) => {
+    const res = await remoteFetchRaw(path);
+    return Buffer.from(await res.arrayBuffer());
+};
+const remoteFetchRaw = async (path) => {
     const res = await fetch(config.dataFair.url + '/api/v1' + path, {
         headers: config.dataFair.apiKey ? { 'x-apiKey': config.dataFair.apiKey } : {}
     });
     if (!res.ok)
         throw new Error(`error ${res.status} on remote data-fair: ${await res.text()}`);
-    return res.json();
+    return res;
 };
 // Short-lived cache for the dataset enrichment below, so that every preview reload does not
 // hammer the remote data-fair API. Failures are cached too: a private dataset without an api
@@ -338,9 +379,20 @@ app.get('/configurations', async (req, res) => {
         return;
     }
     response.minorVersion = minorVersion(version);
+    // a base application restricted to an organization is invisible to an anonymous request, and
+    // without it the listing below finds nothing at all — the applications are then never reached,
+    // however public they are. privateAccess is what makes it visible, and data-fair answers 401
+    // to it without credentials, so it is only sent when an api key is configured.
+    response.authenticated = !!config.dataFair.apiKey;
     try {
-        const baseApps = await remoteFetch('/base-applications?applicationName=' + encodeURIComponent(name) + '&size=1000&count=false');
+        let baseAppsQuery = '/base-applications?applicationName=' + encodeURIComponent(name) + '&size=1000&count=false';
+        if (response.authenticated)
+            baseAppsQuery += '&privateAccess=' + encodeURIComponent(ownerFilter());
+        const baseApps = await remoteFetch(baseAppsQuery);
         const matchingBaseApps = (baseApps.results ?? []).filter((b) => typeof b.version === 'string' && minorVersion(b.version) === response.minorVersion);
+        // told apart from "the base application exists but carries no application": the UI has no
+        // other way to explain an empty list, and a missing api key is by far the likeliest cause
+        response.baseAppFound = matchingBaseApps.length > 0;
         response.results = [];
         for (const baseApp of matchingBaseApps) {
             const applications = await remoteFetch('/applications?base-application=' + encodeURIComponent(baseApp.url) + '&size=1000&count=false&select=id,title,owner');
@@ -358,10 +410,20 @@ app.get('/configurations', async (req, res) => {
 // copy a configuration from the remote data-fair, keeping its remote origins so that
 // .dev-config.json stays portable — see localize.ts
 app.get('/configurations/:id', async (req, res) => {
+    const id = encodeURIComponent(req.params.id);
     try {
         // Sent as-is, with its remote origins: the UI stores this straight into .dev-config.json,
         // which must stay portable. Origins are rewritten on the read path, in prepareConfig.
-        res.send(await remoteFetch('/applications/' + encodeURIComponent(req.params.id) + '/configuration'));
+        const configuration = await remoteFetch('/applications/' + id + '/configuration');
+        // The attachments come along: a configuration references them by name only, so without the
+        // files a copied production configuration renders with every image broken. Fetched from the
+        // application itself rather than from the configuration, which never lists them.
+        const application = await remoteFetch('/applications/' + id + '?select=attachments');
+        const attachments = await copyAttachments(application.attachments ?? [], (name) => remoteFetchBuffer('/applications/' + id + '/attachments/' + encodeURIComponent(name)));
+        if (attachments.failed.length) {
+            console.warn('[dev-server] failed to copy attachments ' + attachments.failed.join(', ') + ', images referencing them will be broken');
+        }
+        res.send({ configuration, attachments });
     }
     catch (err) {
         res.status(500).send({ error: err.message });
@@ -419,6 +481,8 @@ app.use('/app', createProxyMiddleware({
                             configuration,
                             exposedUrl: `http://localhost:${config.port}/app`,
                             href: `http://localhost:${config.port}/config`,
+                            // data-fair serializes the whole application document, attachments included
+                            attachments: listAttachments(),
                             apiUrl: `http://localhost:${config.port}/data-fair/api/v1`,
                             wsUrl: `ws://localhost:${config.port}/data-fair`,
                             owner: config.dataFair && config.dataFair.owner

package/src/attachments.d.ts

@@ -0,0 +1,19 @@
+export declare const ATTACHMENTS_DIR = ".dev-attachments";
+export type Attachment = {
+    name: string;
+    title: string;
+    size: number;
+    mimetype: string;
+    updatedAt: string;
+};
+export declare const listAttachments: (dir?: string) => Attachment[];
+export declare const attachmentPath: (dir: string, name: string) => string | undefined;
+export type CopyAttachmentsResult = {
+    copied: string[];
+    failed: string[];
+};
+export declare const copyAttachments: (attachments: {
+    name?: string;
+    title?: string;
+    mimetype?: string;
+}[], fetchAttachment: (name: string) => Promise<Buffer>, dir?: string) => Promise<CopyAttachmentsResult>;

package/src/attachments.js

@@ -0,0 +1,125 @@
+// Attachments of the application under development.
+//
+// data-fair lets an application carry attached files: they are listed in the application
+// object (window.APPLICATION.attachments), offered by the configuration form through the
+// `context.attachments` of vjsf, and served under <application.href>/attachments/<name>.
+// An application configuration therefore stores an attachment by name only — see the
+// attachmentImage pattern used by app-eco-watt and friends — and rebuilds its url at
+// render time. We mirror that whole contract locally, with the files sitting in a
+// git-ignored directory, so an image referenced by a copied production configuration
+// displays in dev exactly as it does in production.
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { extname, join, resolve, sep } from 'node:path';
+export const ATTACHMENTS_DIR = '.dev-attachments';
+// Titles are metadata of the remote application, nothing on disk holds them. They are kept
+// beside the files, in a dotfile: hidden entries are excluded from the listing below, so the
+// sidecar can never be mistaken for an attachment.
+const METADATA_FILE = '.metadata.json';
+// Enough to cover what an application attaches (images, and the occasional document). An
+// unknown extension falls back to the mimetype the remote data-fair reported, then to the
+// generic binary type — never to a guess that a browser would act upon.
+const MIME_TYPES = {
+    '.avif': 'image/avif',
+    '.csv': 'text/csv',
+    '.gif': 'image/gif',
+    '.ico': 'image/vnd.microsoft.icon',
+    '.jpeg': 'image/jpeg',
+    '.jpg': 'image/jpeg',
+    '.json': 'application/json',
+    '.pdf': 'application/pdf',
+    '.png': 'image/png',
+    '.svg': 'image/svg+xml',
+    '.txt': 'text/plain',
+    '.webp': 'image/webp'
+};
+const readMetadata = (dir) => {
+    const path = join(dir, METADATA_FILE);
+    if (!existsSync(path))
+        return {};
+    try {
+        const parsed = JSON.parse(readFileSync(path, 'utf8'));
+        if (!Array.isArray(parsed))
+            return {};
+        return Object.fromEntries(parsed.filter(a => a?.name).map(a => [a.name, a]));
+    }
+    catch (err) {
+        // a hand-edited or truncated sidecar must not take the whole listing down with it:
+        // the files themselves are the source of truth, the metadata only decorates them
+        return {};
+    }
+};
+// The directory is the source of truth, not the sidecar: an image a developer drops in by
+// hand is listed just like one copied from a remote application, and a file deleted by hand
+// disappears. Size and date come from the file, since the local file is what is served.
+export const listAttachments = (dir = ATTACHMENTS_DIR) => {
+    if (!existsSync(dir))
+        return [];
+    const metadata = readMetadata(dir);
+    return readdirSync(dir, { withFileTypes: true })
+        .filter(entry => entry.isFile() && !entry.name.startsWith('.'))
+        .map(entry => {
+        const stats = statSync(join(dir, entry.name));
+        const known = metadata[entry.name];
+        return {
+            name: entry.name,
+            title: known?.title ?? entry.name,
+            size: stats.size,
+            mimetype: MIME_TYPES[extname(entry.name).toLowerCase()] ?? known?.mimetype ?? 'application/octet-stream',
+            updatedAt: stats.mtime.toISOString()
+        };
+    })
+        .sort((a, b) => a.name.localeCompare(b.name));
+};
+// The requested name reaches us from an application configuration copied from a remote
+// data-fair, so it is never to be trusted: resolve it and refuse anything that lands outside
+// the attachments directory. undefined means "no such attachment", never "here is /etc/passwd".
+export const attachmentPath = (dir, name) => {
+    if (!name)
+        return undefined;
+    const dirPath = resolve(dir);
+    const filePath = resolve(dirPath, name);
+    if (!filePath.startsWith(dirPath + sep))
+        return undefined;
+    return filePath;
+};
+// Replace the whole local attachments directory with the attachments of a remote application.
+// Replace, and not merge: copying a configuration replaces the current one entirely, so leaving
+// behind the images of the previously copied application would only produce a directory whose
+// content matches no configuration at all.
+//
+// Everything is downloaded before anything is written, then the directory is swapped in one
+// rename: an interrupted or partly failing copy — a private file with no api key, a network
+// hiccup — never leaves a half-written directory behind, and what did download is kept. A copy
+// where nothing downloads does empty the directory, which is the point: the configuration it
+// came with is being applied all the same, and its images are genuinely missing.
+export const copyAttachments = async (attachments, fetchAttachment, dir = ATTACHMENTS_DIR) => {
+    const result = { copied: [], failed: [] };
+    const downloaded = [];
+    for (const attachment of attachments ?? []) {
+        // a name with a path separator would write outside the directory, and data-fair has no
+        // reason to send one: skip it rather than sanitize a name the configuration still refers to
+        if (!attachment?.name || attachmentPath(dir, attachment.name) === undefined) {
+            if (attachment?.name)
+                result.failed.push(attachment.name);
+            continue;
+        }
+        try {
+            downloaded.push({ attachment: { ...attachment, name: attachment.name }, body: await fetchAttachment(attachment.name) });
+        }
+        catch (err) {
+            result.failed.push(attachment.name);
+        }
+    }
+    // written aside then swapped in, so an interrupted copy leaves the previous directory intact
+    const tmpDir = dir + '.tmp';
+    rmSync(tmpDir, { recursive: true, force: true });
+    mkdirSync(tmpDir, { recursive: true });
+    for (const { attachment, body } of downloaded) {
+        writeFileSync(join(tmpDir, attachment.name), body);
+        result.copied.push(attachment.name);
+    }
+    writeFileSync(join(tmpDir, METADATA_FILE), JSON.stringify(downloaded.map(d => d.attachment), null, 2));
+    rmSync(dir, { recursive: true, force: true });
+    renameSync(tmpDir, dir);
+    return result;
+};