@data-fair/dev-server 2.5.0 → 2.5.2

package/README.md

@@ -42,6 +42,50 @@ partagent rien d'autre que le fichier `.env`, écrit une seule fois par
 à la racine) il faut régénérer `.env` avant de lancer le dev-server :
 `node src/dev-env.js --force --app-path=`.
 
+## Configurations et pièces jointes distantes
+
+Le bouton « Copier une configuration distante » liste les applications qui, sur
+le data-fair distant, tournent sur l'application de base en cours de
+développement, dans la même version mineure. La recherche part du nom lu dans la
+balise `<meta name="application-name">` de l'`index.html` local et de la version
+du `package.json`.
+
+Une application de base réservée à une organisation (`privateAccess`) est
+invisible à une requête anonyme. Le paramètre `privateAccess` la rend visible,
+mais data-fair répond 401 sans authentification — il n'est donc envoyé que si
+`DATAFAIR_API_KEY` est renseignée.
+
+Ne pas la trouver n'arrête pas la recherche pour autant : les applications qui
+tournent dessus sont souvent publiques, et il suffit de connaître son URL pour
+les lister. Les URL des applications de base publiées suivent deux conventions
+(`<data-fair>/apps/<slug>/<mineure>/` et
+`https://cdn.jsdelivr.net/npm/@scope/<paquet>@<mineure>/dist/`), et le filtre
+de `/applications` est une égalité stricte : une URL devinée ne peut que ne
+rien renvoyer, jamais renvoyer autre chose. Les candidates sont donc essayées
+en une requête, ce qui suffit à lister les configurations publiques d'une
+application de base privée, sans aucune clé.
+
+Une clé reste nécessaire pour une application, elle, privée. Elle se crée
+depuis le compte ou l'organisation sur le data-fair distant, et se met dans le
+`.env` de l'application :
+
+```
+DATAFAIR_API_KEY=xxx
+```
+
+Ce `.env` est celui que `df-dev-env` génère, et qu'il laisse intact tant qu'on
+ne passe pas `--force` — qui, lui, le réécrit entièrement et emporte la clé.
+
+La copie ramène aussi les pièces jointes de l'application distante dans
+`.dev-attachments/` (git-ignoré). Une configuration ne référence une pièce
+jointe que par son nom et reconstruit son URL à l'affichage
+(`application.href + '/attachments/' + name`) : sans les fichiers, une
+configuration de production copiée s'affiche avec toutes ses images cassées.
+Elles sont servies sous `/config/attachments/`, listées dans
+`window.APPLICATION.attachments`, et proposées par le formulaire de
+configuration comme data-fair le fait avec `context.attachments`. Un fichier
+déposé à la main dans `.dev-attachments/` est listé et servi de la même façon.
+
 ## Development
 
 Run development server :

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/dev-server",
-  "version": "2.5.0",
+  "version": "2.5.2",
   "description": "A development server for optimal development experience of data-fair applications.",
   "main": "src/index.js",
   "bin": {

package/src/app.js

@@ -2,6 +2,8 @@
 import config from './config.js';
 import uiConfig from './ui-config.js';
 import { localizeConfig } from './localize.js';
+import { ATTACHMENTS_DIR, attachmentPath, copyAttachments, listAttachments } from './attachments.js';
+import { candidateBaseAppUrls } from './base-app-urls.js';
 import { WebSocket, WebSocketServer } from 'ws';
 import { createServer } from 'node:http';
 import express from 'express';
@@ -15,7 +17,7 @@ import { isElementNode, createTextNode, createElement, appendChild } from '@pars
 import escapeStringRegexp from 'escape-string-regexp';
 import eventPromise from '@data-fair/lib-utils/event-promise.js';
 import { createSpaMiddleware } from '@data-fair/lib-express/serve-spa.js';
-import { resolve } from 'node:path';
+import { join, resolve } from 'node:path';
 const debug = debugModule('df-dev-server');
 const app = express();
 const server = createServer(app);
@@ -110,6 +112,27 @@ app.post('/config/error', (req, res) => {
     }
     res.send();
 });
+// The attachments of the application under development, read by the dev-server UI to feed the
+// `context.attachments` of the configuration form — data-fair feeds it with
+// application.attachments, and an application configuration references an attachment by name.
+app.get('/config/attachments', (req, res) => {
+    res.send(listAttachments());
+});
+// window.APPLICATION.href points at /config, so an application building an attachment url the
+// production way — application.href + '/attachments/' + name — lands here. Same path shape as
+// data-fair (applications/:id/attachments/*), same no-cache: a re-copied file shows up at once.
+app.get('/config/attachments/*attachmentPath', (req, res) => {
+    const relFilePath = join(...req.params.attachmentPath);
+    const filePath = attachmentPath(ATTACHMENTS_DIR, relFilePath);
+    if (!filePath || !existsSync(filePath)) {
+        res.status(404).send({ error: `attachment "${relFilePath}" not found in ${ATTACHMENTS_DIR}` });
+        return;
+    }
+    res.setHeader('Cache-Control', 'no-cache');
+    // served from a root, and not from the absolute path: send() refuses any path holding a
+    // hidden segment, and the attachments directory is itself a dotted one
+    res.sendFile(relFilePath, { root: resolve(ATTACHMENTS_DIR) });
+});
 // Identification of the app under development: its name is read from the
 // "application-name" meta tag of the local index.html and its version from the
 // package.json of the current directory.
@@ -128,22 +151,44 @@ const localAppInfo = async () => {
         debug('failed to fetch local app index.html', err);
     }
     let version;
+    let packageName;
     try {
-        version = JSON.parse(readFileSync('package.json', 'utf8')).version;
+        const pkg = JSON.parse(readFileSync('package.json', 'utf8'));
+        version = pkg.version;
+        packageName = pkg.name;
     }
     catch (err) {
         debug('failed to read local package.json', err);
     }
-    return { name, version };
+    return { name, version, packageName };
+};
+// The owner the dev-server works on behalf of, in the "type:id[:department]" form the
+// data-fair api expects for its owner and privateAccess filters.
+const ownerFilter = () => {
+    const owner = config.dataFair.owner;
+    let filter = `${owner.type}:${owner.id}`;
+    if (owner.department)
+        filter += ':' + owner.department;
+    return filter;
 };
 // Fetch a resource from the remote data-fair api
 const remoteFetch = async (path) => {
+    const res = await remoteFetchRaw(path);
+    return res.json();
+};
+// Same, for an attachment: a file, never json, and never decoded as text — see the app proxy
+// below, where decoding a binary body as utf8 destroys it.
+const remoteFetchBuffer = async (path) => {
+    const res = await remoteFetchRaw(path);
+    return Buffer.from(await res.arrayBuffer());
+};
+const remoteFetchRaw = async (path) => {
     const res = await fetch(config.dataFair.url + '/api/v1' + path, {
         headers: config.dataFair.apiKey ? { 'x-apiKey': config.dataFair.apiKey } : {}
     });
     if (!res.ok)
         throw new Error(`error ${res.status} on remote data-fair: ${await res.text()}`);
-    return res.json();
+    return res;
 };
 // Short-lived cache for the dataset enrichment below, so that every preview reload does not
 // hammer the remote data-fair API. Failures are cached too: a private dataset without an api
@@ -327,7 +372,7 @@ const minorVersion = (version) => version.split('.').slice(0, 2).join('.');
 // list configurations of the app under development that exist on the remote data-fair
 // with the same minor version, useful to test quickly for regressions
 app.get('/configurations', async (req, res) => {
-    const { name, version } = await localAppInfo();
+    const { name, version, packageName } = await localAppInfo();
     const response = { appName: name, localVersion: version, remoteUrl: config.dataFair.url };
     if (!name) {
         res.status(400).send({ ...response, error: 'The "application-name" meta tag was not found in the local app index.html' });
@@ -338,16 +383,31 @@ app.get('/configurations', async (req, res) => {
         return;
     }
     response.minorVersion = minorVersion(version);
+    // a base application restricted to an organization is invisible to an anonymous request.
+    // privateAccess is what makes it visible, and data-fair answers 401 to it without
+    // credentials, so it is only sent when an api key is configured.
+    response.authenticated = !!config.dataFair.apiKey;
     try {
-        const baseApps = await remoteFetch('/base-applications?applicationName=' + encodeURIComponent(name) + '&size=1000&count=false');
+        let baseAppsQuery = '/base-applications?applicationName=' + encodeURIComponent(name) + '&size=1000&count=false';
+        if (response.authenticated)
+            baseAppsQuery += '&privateAccess=' + encodeURIComponent(ownerFilter());
+        const baseApps = await remoteFetch(baseAppsQuery);
         const matchingBaseApps = (baseApps.results ?? []).filter((b) => typeof b.version === 'string' && minorVersion(b.version) === response.minorVersion);
-        response.results = [];
-        for (const baseApp of matchingBaseApps) {
-            const applications = await remoteFetch('/applications?base-application=' + encodeURIComponent(baseApp.url) + '&size=1000&count=false&select=id,title,owner');
-            for (const application of applications.results ?? []) {
-                response.results.push({ id: application.id, title: application.title, owner: application.owner, baseAppVersion: baseApp.version });
-            }
-        }
+        // Not finding the base application is not the end of the search: it may simply be private
+        // while the applications running on it are public, and those are reachable as soon as its
+        // url is known. Guessing the url costs nothing — the filter below is an exact match.
+        const versionByUrl = new Map(matchingBaseApps.map((b) => [b.url, b.version]));
+        const baseAppUrls = matchingBaseApps.length
+            ? [...versionByUrl.keys()]
+            : candidateBaseAppUrls(config.dataFair.url, name, packageName, response.minorVersion);
+        // one request for the whole set: the data-fair list filters accept comma separated values
+        const applications = await remoteFetch('/applications?base-application=' + encodeURIComponent(baseAppUrls.join(',')) + '&size=1000&count=false&select=id,title,owner,url');
+        response.results = (applications.results ?? []).map((application) => ({
+            id: application.id,
+            title: application.title,
+            owner: application.owner,
+            baseAppVersion: versionByUrl.get(application.url) ?? response.minorVersion
+        }));
         response.results.sort((a, b) => (a.title || '').localeCompare(b.title || ''));
         res.send(response);
     }
@@ -358,10 +418,20 @@ app.get('/configurations', async (req, res) => {
 // copy a configuration from the remote data-fair, keeping its remote origins so that
 // .dev-config.json stays portable — see localize.ts
 app.get('/configurations/:id', async (req, res) => {
+    const id = encodeURIComponent(req.params.id);
     try {
         // Sent as-is, with its remote origins: the UI stores this straight into .dev-config.json,
         // which must stay portable. Origins are rewritten on the read path, in prepareConfig.
-        res.send(await remoteFetch('/applications/' + encodeURIComponent(req.params.id) + '/configuration'));
+        const configuration = await remoteFetch('/applications/' + id + '/configuration');
+        // The attachments come along: a configuration references them by name only, so without the
+        // files a copied production configuration renders with every image broken. Fetched from the
+        // application itself rather than from the configuration, which never lists them.
+        const application = await remoteFetch('/applications/' + id + '?select=attachments');
+        const attachments = await copyAttachments(application.attachments ?? [], (name) => remoteFetchBuffer('/applications/' + id + '/attachments/' + encodeURIComponent(name)));
+        if (attachments.failed.length) {
+            console.warn('[dev-server] failed to copy attachments ' + attachments.failed.join(', ') + ', images referencing them will be broken');
+        }
+        res.send({ configuration, attachments });
     }
     catch (err) {
         res.status(500).send({ error: err.message });
@@ -419,6 +489,8 @@ app.use('/app', createProxyMiddleware({
                             configuration,
                             exposedUrl: `http://localhost:${config.port}/app`,
                             href: `http://localhost:${config.port}/config`,
+                            // data-fair serializes the whole application document, attachments included
+                            attachments: listAttachments(),
                             apiUrl: `http://localhost:${config.port}/data-fair/api/v1`,
                             wsUrl: `ws://localhost:${config.port}/data-fair`,
                             owner: config.dataFair && config.dataFair.owner

package/src/attachments.d.ts

@@ -0,0 +1,19 @@
+export declare const ATTACHMENTS_DIR = ".dev-attachments";
+export type Attachment = {
+    name: string;
+    title: string;
+    size: number;
+    mimetype: string;
+    updatedAt: string;
+};
+export declare const listAttachments: (dir?: string) => Attachment[];
+export declare const attachmentPath: (dir: string, name: string) => string | undefined;
+export type CopyAttachmentsResult = {
+    copied: string[];
+    failed: string[];
+};
+export declare const copyAttachments: (attachments: {
+    name?: string;
+    title?: string;
+    mimetype?: string;
+}[], fetchAttachment: (name: string) => Promise<Buffer>, dir?: string) => Promise<CopyAttachmentsResult>;

package/src/attachments.js

@@ -0,0 +1,125 @@
+// Attachments of the application under development.
+//
+// data-fair lets an application carry attached files: they are listed in the application
+// object (window.APPLICATION.attachments), offered by the configuration form through the
+// `context.attachments` of vjsf, and served under <application.href>/attachments/<name>.
+// An application configuration therefore stores an attachment by name only — see the
+// attachmentImage pattern used by app-eco-watt and friends — and rebuilds its url at
+// render time. We mirror that whole contract locally, with the files sitting in a
+// git-ignored directory, so an image referenced by a copied production configuration
+// displays in dev exactly as it does in production.
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { extname, join, resolve, sep } from 'node:path';
+export const ATTACHMENTS_DIR = '.dev-attachments';
+// Titles are metadata of the remote application, nothing on disk holds them. They are kept
+// beside the files, in a dotfile: hidden entries are excluded from the listing below, so the
+// sidecar can never be mistaken for an attachment.
+const METADATA_FILE = '.metadata.json';
+// Enough to cover what an application attaches (images, and the occasional document). An
+// unknown extension falls back to the mimetype the remote data-fair reported, then to the
+// generic binary type — never to a guess that a browser would act upon.
+const MIME_TYPES = {
+    '.avif': 'image/avif',
+    '.csv': 'text/csv',
+    '.gif': 'image/gif',
+    '.ico': 'image/vnd.microsoft.icon',
+    '.jpeg': 'image/jpeg',
+    '.jpg': 'image/jpeg',
+    '.json': 'application/json',
+    '.pdf': 'application/pdf',
+    '.png': 'image/png',
+    '.svg': 'image/svg+xml',
+    '.txt': 'text/plain',
+    '.webp': 'image/webp'
+};
+const readMetadata = (dir) => {
+    const path = join(dir, METADATA_FILE);
+    if (!existsSync(path))
+        return {};
+    try {
+        const parsed = JSON.parse(readFileSync(path, 'utf8'));
+        if (!Array.isArray(parsed))
+            return {};
+        return Object.fromEntries(parsed.filter(a => a?.name).map(a => [a.name, a]));
+    }
+    catch (err) {
+        // a hand-edited or truncated sidecar must not take the whole listing down with it:
+        // the files themselves are the source of truth, the metadata only decorates them
+        return {};
+    }
+};
+// The directory is the source of truth, not the sidecar: an image a developer drops in by
+// hand is listed just like one copied from a remote application, and a file deleted by hand
+// disappears. Size and date come from the file, since the local file is what is served.
+export const listAttachments = (dir = ATTACHMENTS_DIR) => {
+    if (!existsSync(dir))
+        return [];
+    const metadata = readMetadata(dir);
+    return readdirSync(dir, { withFileTypes: true })
+        .filter(entry => entry.isFile() && !entry.name.startsWith('.'))
+        .map(entry => {
+        const stats = statSync(join(dir, entry.name));
+        const known = metadata[entry.name];
+        return {
+            name: entry.name,
+            title: known?.title ?? entry.name,
+            size: stats.size,
+            mimetype: MIME_TYPES[extname(entry.name).toLowerCase()] ?? known?.mimetype ?? 'application/octet-stream',
+            updatedAt: stats.mtime.toISOString()
+        };
+    })
+        .sort((a, b) => a.name.localeCompare(b.name));
+};
+// The requested name reaches us from an application configuration copied from a remote
+// data-fair, so it is never to be trusted: resolve it and refuse anything that lands outside
+// the attachments directory. undefined means "no such attachment", never "here is /etc/passwd".
+export const attachmentPath = (dir, name) => {
+    if (!name)
+        return undefined;
+    const dirPath = resolve(dir);
+    const filePath = resolve(dirPath, name);
+    if (!filePath.startsWith(dirPath + sep))
+        return undefined;
+    return filePath;
+};
+// Replace the whole local attachments directory with the attachments of a remote application.
+// Replace, and not merge: copying a configuration replaces the current one entirely, so leaving
+// behind the images of the previously copied application would only produce a directory whose
+// content matches no configuration at all.
+//
+// Everything is downloaded before anything is written, then the directory is swapped in one
+// rename: an interrupted or partly failing copy — a private file with no api key, a network
+// hiccup — never leaves a half-written directory behind, and what did download is kept. A copy
+// where nothing downloads does empty the directory, which is the point: the configuration it
+// came with is being applied all the same, and its images are genuinely missing.
+export const copyAttachments = async (attachments, fetchAttachment, dir = ATTACHMENTS_DIR) => {
+    const result = { copied: [], failed: [] };
+    const downloaded = [];
+    for (const attachment of attachments ?? []) {
+        // a name with a path separator would write outside the directory, and data-fair has no
+        // reason to send one: skip it rather than sanitize a name the configuration still refers to
+        if (!attachment?.name || attachmentPath(dir, attachment.name) === undefined) {
+            if (attachment?.name)
+                result.failed.push(attachment.name);
+            continue;
+        }
+        try {
+            downloaded.push({ attachment: { ...attachment, name: attachment.name }, body: await fetchAttachment(attachment.name) });
+        }
+        catch (err) {
+            result.failed.push(attachment.name);
+        }
+    }
+    // written aside then swapped in, so an interrupted copy leaves the previous directory intact
+    const tmpDir = dir + '.tmp';
+    rmSync(tmpDir, { recursive: true, force: true });
+    mkdirSync(tmpDir, { recursive: true });
+    for (const { attachment, body } of downloaded) {
+        writeFileSync(join(tmpDir, attachment.name), body);
+        result.copied.push(attachment.name);
+    }
+    writeFileSync(join(tmpDir, METADATA_FILE), JSON.stringify(downloaded.map(d => d.attachment), null, 2));
+    rmSync(dir, { recursive: true, force: true });
+    renameSync(tmpDir, dir);
+    return result;
+};

package/src/base-app-urls.d.ts

@@ -0,0 +1 @@
+export declare const candidateBaseAppUrls: (dataFairUrl: string, appName: string, packageName: string | undefined, minorVersion: string) => string[];

package/src/base-app-urls.js

@@ -0,0 +1,37 @@
+// Candidate urls of the base application under development.
+//
+// Discovering it by name is the authoritative path, but it only works for a base application
+// the request is allowed to see: one restricted to an organization is invisible to an anonymous
+// request, while the applications running on it are often public — which is precisely the case
+// of an application developed for a single customer. The /applications listing filters on an
+// exact url, so a wrong candidate returns nothing and a right one returns the applications: a
+// guess can never produce a false match, and they all fit in a single request.
+//
+// The two conventions published base applications follow, as of today:
+//   https://koumoul.com/apps/<slug>/<minor>/
+//   https://cdn.jsdelivr.net/npm/@scope/<package>@<minor>/dist/
+// The slug is the repository name, which is not always the application name — "Liste et fiches"
+// is served under /apps/list-details/ — so both are tried, along with the package name stripped
+// of the prefixes repositories use.
+const slugs = (appName, packageName) => {
+    const candidates = [appName];
+    if (packageName) {
+        const unscoped = packageName.startsWith('@') ? packageName.slice(packageName.indexOf('/') + 1) : packageName;
+        candidates.push(unscoped, unscoped.replace(/^(app|data-fair)-/, ''));
+    }
+    return [...new Set(candidates.filter(Boolean))];
+};
+export const candidateBaseAppUrls = (dataFairUrl, appName, packageName, minorVersion) => {
+    const urls = [];
+    // the same origin as the data-fair the dev-server talks to: a base application is served
+    // beside it, never on some unrelated host
+    const origin = new URL(dataFairUrl).origin;
+    for (const slug of slugs(appName, packageName)) {
+        urls.push(`${origin}/apps/${slug}/${minorVersion}/`);
+    }
+    if (packageName) {
+        const scoped = packageName.startsWith('@') ? packageName : '@data-fair/' + packageName;
+        urls.push(`https://cdn.jsdelivr.net/npm/${scoped}@${minorVersion}/dist/`);
+    }
+    return urls;
+};