@data-fair/dev-server 1.13.1 → 2.0.0

package/package.json

@@ -1,19 +1,26 @@
 {
   "name": "@data-fair/dev-server",
-  "version": "1.13.1",
+  "version": "2.0.0",
   "description": "A development server for optimal development experience of data-fair applications.",
-  "main": "server/index.js",
+  "main": "src/index.ts",
   "bin": {
-    "df-dev-server": "server/index.js"
+    "df-dev-server": "src/index.ts"
   },
+  "type": "module",
   "scripts": {
-    "dev": "NODE_ENV=development nodemon server",
-    "dev-src": "http-server test-apps/minimal --port=3000 --cors --silent",
-    "dev-vue3": "APP_URL=http://localhost:3000/app/ APP_SCRIPT=dev-src-vue3 NODE_ENV=development nodemon server",
-    "dev-src-vue3": "cd test-apps/vue3 && npm run dev",
-    "prepare": "npm run lint && nuxt generate",
-    "lint": "eslint --ext .js,.vue ."
+    "dev": "NODE_ENV=development node --watch src/index.ts",
+    "dev-test-app-minimal": "http-server test-apps/minimal --port=3000 --cors --silent",
+    "dev-test-app-vue3": "cd test-apps/vue3 && npm run dev",
+    "dev-zellij": "zellij --layout .zellij.kdl",
+    "prepublishOnly": "npm run lint && npm run check-types && npm -w ui run build",
+    "lint": "eslint . && npm -w ui run lint",
+    "lint-fix": "eslint --fix . && npm -w ui run lint-fix",
+    "check-types": "tsc && npm -w ui run check-types",
+    "build-types": "df-build-types ."
   },
+  "workspaces": [
+    "ui"
+  ],
   "repository": {
     "type": "git",
     "url": "git+https://github.com/koumoul-dev/df-dev-server.git"
@@ -22,66 +29,44 @@
     "data-fair",
     "dev"
   ],
-  "author": "Alban Mouton <alban.mouton@gmail.com>",
+  "author": "Alban Mouton <alban.mouton@koumoul.com>",
   "license": "MIT",
   "bugs": {
     "url": "https://github.com/koumoul-dev/df-dev-server/issues"
   },
   "homepage": "https://github.com/koumoul-dev/df-dev-server#readme",
-  "eslintConfig": {
-    "root": true,
-    "env": {
-      "node": true
-    },
-    "extends": [
-      "plugin:vue/essential",
-      "eslint:recommended",
-      "standard",
-      "plugin:vue/recommended"
-    ],
-    "rules": {
-      "vue/no-v-html": "off",
-      "vue/multi-word-component-names": "off"
-    }
+  "devDependencies": {
+    "@data-fair/lib-types-builder": "^1.8.2",
+    "@types/config": "^3.3.5",
+    "@types/cors": "^2.8.19",
+    "@types/debug": "^4.1.12",
+    "@types/express": "^5.0.3",
+    "@types/node": "^24.0.7",
+    "@types/ws": "^8.18.1",
+    "eslint": "^9.30.0",
+    "eslint-plugin-vue": "^10.2.0",
+    "eslint-plugin-vuetify": "^2.5.2",
+    "http-server": "^14.1.1",
+    "neostandard": "^0.12.1",
+    "typescript": "^5.8.3"
   },
   "dependencies": {
-    "@koumoul/nuxt-config-inject": "^0.4.4",
-    "@koumoul/vjsf": "^2.18.0",
-    "@koumoul/v-iframe": "^1.3.4",
-    "ajv": "^8.8.2",
-    "ajv-i18n": "^4.2.0",
-    "body-parser": "^1.19.1",
-    "chalk": "^3.0.0",
-    "config": "^3.3.6",
+    "@data-fair/lib-express": "^1.20.4",
+    "@parse5/tools": "^0.6.0",
+    "chalk": "^5.4.1",
+    "config": "^4.0.0",
     "cors": "^2.8.5",
-    "dot-prop": "^6.0.1",
-    "dotenv": "^10.0.0",
-    "escape-string-regexp": "^2.0.0",
-    "event-to-promise": "^0.8.0",
-    "express": "^4.17.2",
-    "fs-extra": "^10.0.0",
-    "html-extractor": "^0.2.2",
-    "http-proxy-middleware": "^2.0.1",
-    "iframe-resizer": "^4.3.2",
-    "parse5": "^6.0.1",
-    "tree-kill": "^1.2.2",
-    "ws": "^8.3.0"
+    "debug": "^4.4.1",
+    "dotenv": "^17.0.0",
+    "express": "^5.1.0",
+    "http-proxy-middleware": "^3.0.5",
+    "parse5": "^7.3.0",
+    "prom-client": "^15.1.3",
+    "ws": "^8.18.3"
   },
-  "devDependencies": {
-    "@babel/eslint-parser": "^7.16.0",
-    "@nuxtjs/axios": "^5.13.6",
-    "@nuxtjs/i18n": "^7.2.0",
-    "@nuxtjs/vuetify": "^1.12.3",
-    "eslint": "^7.12.1",
-    "eslint-config-standard": "^16.0.3",
-    "eslint-plugin-import": "^2.25.2",
-    "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^5.1.1",
-    "eslint-plugin-vue": "^8.0.3",
-    "http-server": "^0.12.3",
-    "nodemon": "^2.0.15",
-    "nuxt": "^2.15.8",
-    "reconnecting-websocket": "^4.4.0",
-    "vuetify": "^2.6.1"
-  }
+  "files": [
+    "./config/dist/**/*",
+    "./ui/dist/**/*",
+    "./src/**/*"
+  ]
 }

package/src/app.ts

@@ -0,0 +1,278 @@
+// Express app for TaxMan own API and UI
+
+import config from './config.ts'
+import uiConfig from './ui-config.ts'
+import { WebSocket, WebSocketServer } from 'ws'
+import { createServer } from 'node:http'
+import express from 'express'
+import cors from 'cors'
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+import debugModule from 'debug'
+import { createProxyMiddleware, fixRequestBody } from 'http-proxy-middleware'
+import * as parse5 from 'parse5'
+import chalk from 'chalk'
+import { isElementNode, createTextNode, createElement } from '@parse5/tools'
+import escapeStringRegexp from 'escape-string-regexp'
+import eventPromise from '@data-fair/lib-utils/event-promise.js'
+import { createSpaMiddleware } from '@data-fair/lib-express/serve-spa.js'
+import { resolve } from 'node:path'
+
+const debug = debugModule('df-dev-server')
+
+const app = express()
+const server = createServer(app)
+
+// web socket server, used both by dev-server UI (opened on / path)
+// and by application (opened on /data-fair), in this case we transmit all messages to and from data-fair
+const wss = new WebSocketServer({ noServer: true })
+let devServerWS: WebSocket
+let dataFairWS: WebSocket
+server.on('upgrade', function upgrade (req, socket, head) {
+  wss.handleUpgrade(req, socket, head, (ws) => {
+    if (req.url === '/data-fair') {
+      if (dataFairWS) dataFairWS.terminate()
+      dataFairWS = ws
+      ws.on('message', (data: string) => {
+        if (dataFairOutputWS) {
+          const message = JSON.parse(data)
+          if (message.type === 'subscribe') message.apiKey = config.dataFair.apiKey
+          dataFairOutputWS.send(JSON.stringify(message))
+        }
+      })
+    } else {
+      if (devServerWS) devServerWS.terminate()
+      devServerWS = ws
+    }
+    wss.emit('connection', ws, req)
+  })
+})
+
+const dfWsUrl = config.dataFair.url.replace('http://', 'ws://').replace('https://', 'wss://') + '/'
+const dataFairOutputWS = new WebSocket(dfWsUrl)
+dataFairOutputWS.on('open', () => dataFairOutputWS.send('connect to external websocket server ' + dfWsUrl))
+dataFairOutputWS.on('message', (data) => { if (dataFairWS) dataFairWS.send(data.toString()) })
+dataFairOutputWS.on('error', (err) => console.error('failed to connect to external websocket server', dfWsUrl, err))
+
+app.use(cors())
+app.use(express.json())
+
+// very basic CRUD of config
+app.get('/config', (req, res, next) => {
+  const devConfig = existsSync('.dev-config.json') ? JSON.parse(readFileSync('.dev-config.json', 'utf8')) : {}
+  debug('read dev config', devConfig)
+  res.send(devConfig)
+})
+app.put('/config', (req, res, next) => {
+  debug('save dev config', req.body)
+  writeFileSync('.dev-config.json', JSON.stringify(req.body, null, 2))
+  res.send(req.body)
+})
+app.post('/config/error', (req, res) => {
+  console.log('Application sent an error', req.body)
+  if (devServerWS) devServerWS.send(JSON.stringify({ type: 'app-error', data: req.body }))
+  else console.error('no dev-server websocket open to send error to')
+  res.send()
+})
+
+// re-expose the application performing similar modifications to the body as data-fair
+const appUrl = new URL(config.app.url)
+const appPrefix = appUrl.pathname === '/' ? '' : appUrl.pathname
+app.use('/app', createProxyMiddleware({
+  target: appUrl.origin,
+  secure: false,
+  changeOrigin: true,
+  selfHandleResponse: true, // so that the onProxyRes takes care of sending the response
+  on: {
+    proxyReq (proxyReq, req, res) {
+      proxyReq.path = appPrefix + proxyReq.path
+      proxyReq.setHeader('Accept-Encoding', 'identity') // disable compression
+    },
+    proxyRes (proxyRes, req, res) {
+      const configuration = existsSync('.dev-config.json') ? JSON.parse(readFileSync('.dev-config.json', 'utf8')) : {}
+      // console.log('inject config', configuration)
+      const dataBuffers: Buffer[] = []
+      proxyRes.on('data', (data) => { dataBuffers.push(data) })
+      proxyRes.on('end', async () => {
+        try {
+          let output = Buffer.concat(dataBuffers).toString()
+          if (output.includes('%APPLICATION%')) {
+            const filledBody = output.replace(/%APPLICATION%/g, JSON.stringify({
+              id: 'dev-application',
+              title: 'Dev application',
+              configuration,
+              exposedUrl: `http://localhost:${config.port}/app`,
+              href: `http://localhost:${config.port}/config`,
+              apiUrl: `http://localhost:${config.port}/data-fair/api/v1`,
+              wsUrl: `ws://localhost:${config.port}/data-fair`,
+              owner: config.dataFair && config.dataFair.owner
+            }))
+            const document = parse5.parse(filledBody)
+            const html = document.childNodes.filter(isElementNode).find(c => c.tagName === 'html')
+            if (!html) throw new Error('HTML structure is broken, expect html, head and body elements')
+            const headNode = html.childNodes.filter(isElementNode).find(c => c.tagName === 'head')
+            const bodyNode = html.childNodes.filter(isElementNode).find(c => c.tagName === 'body')
+            if (!headNode || !bodyNode) throw new Error('HTML structure is broken, expect html, head and body elements')
+
+            // add a script to simulate instrumentation by capture service
+            // @ts-ignore
+            const query: Record<string, string> = req.query
+            if (query.thumbnail === 'true') {
+              const script = createElement('script', { type: 'text/javascript' })
+              script.childNodes.push(createTextNode(`
+              console.log('[capture] Simulate a screenshot capture context')
+              var triggerCalled = false
+              window.triggerCapture = function (animationSupported) {
+                triggerCalled = true
+                console.log('[capture] triggerCapture called')
+                if (animationSupported) {
+                  console.log('[capture] this application supports animated screenshots')
+                  var i = 0
+                  const interval = setInterval(function () {
+                    i++
+                    if (i === 1800) {
+                      console.error('[capture] stop after the maximum number of frames was attained')
+                      clearInterval(interval)
+                    }
+                    var stopped = window.animateCaptureFrame()
+                    if (stopped) {
+                      console.log('[capture] animation was stopped after ' + i + ' frames')
+                      clearInterval(interval)
+                    }
+                  }, 67)
+                  return true
+                } else {
+                  console.log('[capture] this application does not support animated screenshots')
+                }
+              }
+              setTimeout(function() {
+                if (!triggerCalled) {
+                  console.error('[capture] triggerCapture was not called after a 5s wait')
+                }
+              }, 5000)
+                              `))
+              headNode.childNodes.push(script)
+            }
+
+            const meta: Record<string, string> = {}
+            for (const node of headNode.childNodes.filter(isElementNode)) {
+              if (node.tagName === 'meta') {
+                const name = node.attrs.find(a => a.name === 'name')?.value
+                const content = node.attrs.find(a => a.name === 'content')?.value
+                if (name !== undefined && content !== undefined) meta[name] = content
+              }
+            }
+            if (meta['df:sync-state'] === 'true' || meta['df:overflow'] === 'true') {
+              bodyNode.childNodes.push(createElement('script', {
+                type: 'text/javascript',
+                src: 'https://cdn.jsdelivr.net/npm/@data-fair/frame@0.12/dist/v-iframe-compat/d-frame-content.min.js'
+              }))
+            }
+
+            output = parse5.serialize(document)
+
+            // proxyRes.headers['content-length'] = output.length
+            delete proxyRes.headers['content-length']
+            delete proxyRes.headers['last-modified']
+            delete proxyRes.headers['max-age']
+            delete proxyRes.headers.etag
+            proxyRes.headers['cache-control'] = 'no-cache'
+          }
+          res.writeHead(proxyRes.statusCode ?? 200, proxyRes.headers)
+          res.end(output)
+        } catch (err: any) {
+          console.error(err)
+          res.writeHead(500)
+          res.end(err.message)
+        }
+      })
+    }
+  }
+}))
+
+for (const proxyPath of config.app.proxyPaths) {
+  app.use(proxyPath, createProxyMiddleware({
+    target: appUrl.origin,
+    secure: false,
+    changeOrigin: true,
+    ws: true
+  }))
+}
+
+// re-expose a data-fair instance to access datasets, etc.
+const dfUrl = new URL(config.dataFair.url)
+app.use('/data-fair', createProxyMiddleware({
+  target: dfUrl.origin,
+  secure: false,
+  changeOrigin: true,
+  selfHandleResponse: true, // so that the onProxyRes takes care of sending the response
+  on: {
+    proxyReq (proxyReq, req, res) {
+      proxyReq.path = '/data-fair' + proxyReq.path
+
+      // no gzip so that we can process the content
+      proxyReq.setHeader('accept-encoding', 'identity')
+
+      // authentication through api key only
+      proxyReq.setHeader('cookie', '')
+      if (config.dataFair.apiKey) proxyReq.setHeader('x-apiKey', config.dataFair.apiKey)
+
+      // body was already parsed by body-parser and no longer available as a stream
+      fixRequestBody(proxyReq, req)
+    },
+    proxyRes (proxyRes, req, res) {
+      // console.log('DF RES', proxyRes)
+      if (proxyRes.headers['content-type'] && proxyRes.headers['content-type'].startsWith('application/json')) {
+        let body = ''
+        proxyRes.on('data', (data) => { body += data.toString() })
+        proxyRes.on('end', () => {
+          const output = body.replace(new RegExp(escapeStringRegexp(config.dataFair.url), 'g'), 'http://localhost:5888/data-fair')
+          // proxyRes.headers['content-length'] = output.length
+          delete proxyRes.headers['content-length']
+          res.writeHead(proxyRes.statusCode ?? 200, proxyRes.headers)
+          // make all references to data-fair url point to local proxy
+          res.end(output)
+        })
+      } else {
+        res.writeHead(proxyRes.statusCode ?? 200, proxyRes.headers)
+        proxyRes.pipe(res)
+      }
+    }
+  }
+}))
+
+// also re-expose the simple-directory instance matching data-fair
+app.use('/simple-directory', createProxyMiddleware({
+  target: dfUrl.origin,
+  secure: false,
+  changeOrigin: true,
+  on: {
+    proxyReq (proxyReq, req, res) {
+      proxyReq.path = '/simple-directory' + proxyReq.path
+    }
+  }
+}))
+
+if (config.serveUi) {
+  app.use(await createSpaMiddleware(resolve(import.meta.dirname, '../ui/dist'), uiConfig))
+} else {
+  app.use('/', createProxyMiddleware({
+    target: 'http://localhost:6220',
+    secure: false
+  }))
+}
+
+// Run app and return it in a promise
+export const run = async () => {
+  server.listen(config.port)
+  await eventPromise(server, 'listening')
+  console.log(chalk.bold.blue('\nDataFair dev server available on ') + chalk.underline.bold.blue(`http://localhost:${config.port}`))
+  return server
+}
+
+export const stop = async () => {
+  if (devServerWS) devServerWS.terminate()
+  if (dataFairWS) dataFairWS.terminate()
+  server.close()
+  await eventPromise(server, 'close')
+  await app.get('client').close()
+}

package/src/config.ts

@@ -0,0 +1,12 @@
+import { assertValid, type DevServerConfig } from '../config/type/index.js'
+import config from 'config'
+
+export type { DevServerConfig } from '../config/type/index.ts'
+
+// we reload the config instead of using the singleton from the config module for testing purposes
+// @ts-ignore
+const devServerConfig = process.env.NODE_ENV === 'test' ? config.util.loadFileConfigs(process.env.NODE_CONFIG_DIR, { skipConfigSources: true }) : config
+
+assertValid(devServerConfig, { lang: 'en', name: 'config', internal: true })
+
+export default devServerConfig as DevServerConfig

package/{server/index.js → src/index.ts}

@@ -1,14 +1,15 @@
 #!/usr/bin/env node
 
-const path = require('path')
-require('dotenv').config()
-process.env.NODE_CONFIG_DIR = path.join(__dirname, '../config')
-const app = require('./app')
+import 'dotenv/config'
+import path from 'node:path'
+
+process.env.NODE_CONFIG_DIR = path.join(import.meta.dirname, '../config')
+const app = await import('./app.ts')
 
 app.run().then(app => {
   // nothing to do
 }, error => {
-  console.error('Failure in customers process', error)
+  console.error('Failure in dev server process', error)
   process.exit(-1)
 })
 

package/src/ui-config.ts

@@ -0,0 +1,6 @@
+import config from './config.ts'
+
+export const uiConfig = { dataFair: config.dataFair }
+
+export type UiConfig = typeof uiConfig
+export default uiConfig