@ebay/muse-runner 1.0.21

package/lib/server.js

@@ -0,0 +1,463 @@
+import express from 'express';
+import expressWs from 'express-ws';
+import _ from 'lodash';
+import cors from 'cors';
+import fs from 'fs-extra';
+import path from 'path';
+import Conf from 'conf';
+import museCore from '@ebay/muse-core';
+import * as url from 'url';
+import crypto from 'crypto';
+import { exec } from 'node:child_process';
+import fallback from 'express-history-api-fallback';
+import MuseRunner from './MuseRunner.js';
+import openBrowser from 'react-dev-utils/openBrowser.js';
+import terminals from './apis/terminals.js';
+import gitStatus from './apis/gitStatus.js';
+import settings from './apis/settings.js';
+import startPlugin from './apis/startPlugin.js';
+import stopPlugin from './apis/stopPlugin.js';
+import utils, { handleAsyncError } from './utils.js';
+
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
+
+console.log('Starting Muse Runner...');
+
+const config = new Conf({
+  projectName: process.env.MUSE_RUNNER_CONFIG_NAME || 'muse-runner',
+});
+console.log('Config path: ', config.path);
+const app = express();
+app.use(express.json({ limit: '5MB' }));
+app.use(express.urlencoded({ extended: true }));
+app.use(cors());
+expressWs(app);
+
+const port = parseInt(process.env.MUSE_RUNNER_PORT || '6066', 10);
+
+function setupWebSocket(app) {
+  const sockets = [];
+  app.ws('/api/muse-runner-socket', (ws) => {
+    ws.on('close', () => _.pull(sockets, ws));
+    sockets.push(ws);
+  });
+
+  return {
+    emit(msg) {
+      sockets.forEach((socket) => {
+        try {
+          socket.send(JSON.stringify(msg));
+        } catch (e) {
+          // ignore socket send error
+        }
+      });
+    },
+  };
+}
+
+const io = setupWebSocket(app);
+
+const bound = [];
+const msgCache = {};
+function bindOutputToSocket(id, output) {
+  if (bound.includes(output)) return;
+  bound.push(output);
+  const arr = [];
+  const flush = _.throttle(() => {
+    if (!msgCache[id]) msgCache[id] = [];
+    const msg = {
+      type: 'output',
+      data: {
+        id: id || `muse-runner`,
+        output: arr,
+      },
+    };
+    msgCache[id].push(_.cloneDeep(msg));
+    if (msgCache[id].length > 100) msgCache[id].shift();
+    io.emit(msg);
+    arr.length = 0;
+  }, 100);
+
+  const onData = (data) => {
+    arr.push(data.toString().replace(/\n/g, '\n\r'));
+    flush();
+  };
+  if (output.onData) {
+    output.onData(onData);
+  } else {
+    output.on('data', onData);
+  }
+}
+
+config.onDidAnyChange(() => {
+  io.emit({
+    type: 'config-data-changed',
+    data: config.store,
+  });
+});
+
+const getRunningData = () => {
+  return {
+    apps: runner.runningApps.map((r) => ({
+      app: r.app,
+      env: r.env,
+      id: r.id,
+      port: r.port,
+    })),
+    plugins: runner.runningPlugins.map((r) => ({
+      name: r.pluginInfo.name,
+      dir: r.dir,
+      port: r.port,
+    })),
+  };
+};
+const handleRunningDataChange = () => {
+  io.emit({
+    type: 'running-data-changed',
+    data: getRunningData(),
+  });
+};
+
+const runner = new MuseRunner();
+
+runner.on('start-plugin', ({ pluginRunner }) => {
+  pluginRunner.on('exit', (code) => {
+    handleRunningDataChange();
+    io.emit({
+      type: 'plugin-exited',
+      data: {
+        pluginName: pluginRunner.pluginInfo.name,
+        dir: pluginRunner.dir,
+        code,
+      },
+    });
+  });
+  bindOutputToSocket(`plugin:${pluginRunner.dir}`, pluginRunner.cmd.ptyProcess);
+  msgCache[`plugin:${pluginRunner.dir}}`] = [];
+  handleRunningDataChange();
+});
+
+app.post('/api/clear-msg-cache', (req, res) => {
+  const { id } = req.body;
+  if (id) {
+    delete msgCache[id];
+  }
+  res.end('ok');
+});
+app.post(
+  '/api/start-app',
+  handleAsyncError(async (req, res) => {
+    const { id } = req.body;
+    const appList = config.get('appList', []);
+    const app = appList.find((a) => a.id === id);
+    if (!app) throw new Error(`App not found: ${id}`);
+
+    const appRunner = await runner.startApp({
+      id,
+      app: app.app,
+      env: app.env,
+      port: app.port,
+    });
+
+    appRunner.on('exit', (code) => {
+      handleRunningDataChange();
+      io.emit({
+        type: 'app-exited',
+        data: {
+          appId: id,
+          code,
+        },
+      });
+    });
+    handleRunningDataChange();
+    appRunner.worker.on('message', (msg) => {
+      if (msg.type === 'call-parent-api') {
+        switch (msg.payload.key) {
+          case 'get-app-config':
+            appRunner.worker.postMessage({
+              type: 'resolve-promise',
+              payload: {
+                promiseId: msg.payload.promiseId,
+                result: {
+                  ...utils.getAppConfig({ id, runner, config }),
+                  port: appRunner.port,
+                },
+              },
+            });
+            break;
+          default:
+            break;
+        }
+      }
+    });
+    bindOutputToSocket(`app:${id}`, appRunner.worker.stdout);
+    msgCache[`app:${id}`] = [];
+    res.setHeader('Content-Type', 'application/json');
+    res.end(
+      JSON.stringify({
+        id,
+        app: app.app,
+        env: app.env,
+        port: appRunner.port,
+      }),
+    );
+  }),
+);
+
+app.post(
+  '/api/new-app',
+  handleAsyncError(async (req, res) => {
+    const id = crypto.randomBytes(6).toString('hex');
+    const appList = config.get('appList', []);
+    const { app, env, ...rest } = req.body;
+    appList.push({ id, app, env, ...rest });
+    config.set('appList', appList);
+    res.setHeader('Content-Type', 'application/json');
+    res.end(JSON.stringify({ id, app, env, ...rest }));
+  }),
+);
+
+app.post(
+  '/api/update-app',
+  handleAsyncError(async (req, res) => {
+    const appList = config.get('appList', []);
+    const { id, app, env, ...rest } = req.body;
+    const foundApp = appList.find((a) => a.id === id);
+    if (!foundApp) throw new Error(`App not found: ${id}`);
+    Object.assign(foundApp, { app, env, ...rest });
+    config.set('appList', appList);
+    res.end('ok');
+  }),
+);
+
+app.post(
+  '/api/remove-app',
+  handleAsyncError(async (req, res) => {
+    const appList = config.get('appList', []);
+    const { id } = req.body;
+    _.remove(appList, (a) => a.id === id);
+    config.set('appList', appList);
+    res.end('ok');
+  }),
+);
+
+app.post(
+  '/api/stop-app',
+  handleAsyncError(async (req, res) => {
+    const { id } = req.body;
+    await runner.stopApp({ id });
+    res.end('ok');
+  }),
+);
+
+app.post('/api/start-plugin', startPlugin({ config, runner }));
+app.post('/api/stop-plugin', stopPlugin({ runner }));
+
+app.post(
+  '/api/update-plugin',
+  handleAsyncError(async (req, res) => {
+    const { dir, pluginName, ...rest } = req.body;
+    if (dir && !fs.existsSync(dir)) throw new Error(`Folder not exist: ${dir}`);
+    const plugins = config.get('plugins', {});
+    if (!plugins[pluginName]) plugins[pluginName] = {};
+    Object.assign(plugins[pluginName], { dir, ...rest });
+    config.set('plugins', plugins);
+    res.end('ok');
+  }),
+);
+
+app.post(
+  '/api/attach-plugin',
+  handleAsyncError(async (req, res) => {
+    const { appId, pluginName, mode = 'local', ...rest } = req.body;
+    const appList = config.get('appList', []);
+    const app = appList.find((a) => a.id === appId);
+    if (!app) throw new Error(`App not found: ${appId}`);
+    app.plugins = app.plugins || [];
+
+    const toSave = {
+      name: pluginName,
+      mode,
+      ...rest,
+    };
+    const foundIndex = app.plugins.findIndex((p) => p.name === pluginName);
+
+    if (foundIndex === -1) {
+      app.plugins.push(toSave);
+    } else {
+      app.plugins[foundIndex] = toSave;
+    }
+
+    config.set('appList', appList);
+    res.end('ok');
+  }),
+);
+
+app.post(
+  '/api/detach-plugin',
+  handleAsyncError(async (req, res) => {
+    const { appId, pluginName } = req.body;
+    const appList = config.get('appList', []);
+    const app = appList.find((a) => a.id === appId);
+    if (!app) throw new Error(`App not found: ${appId}`);
+    _.remove(app.plugins, (p) => p.name === pluginName);
+    config.set('appList', appList);
+    res.end('ok');
+  }),
+);
+
+// app.post(
+//   '/api/link-plugin',
+//   handleAsyncError(async (req, res) => {
+//     const { mainPlugin, linkedPlugin } = req.body;
+//     if (mainPlugin === linkedPlugin) throw new Error('Cannot link with self');
+//     const plugins = config.get('plugins', {});
+//     if (!plugins[mainPlugin]) {
+//       plugins[mainPlugin] = {};
+//     }
+//     if (!plugins[mainPlugin].linkedPlugins) {
+//       plugins[mainPlugin].linkedPlugins = [];
+//     }
+//     if (!_.find(plugins[mainPlugin].linkedPlugins, { name: linkedPlugin })) {
+//       plugins[mainPlugin].linkedPlugins.push({ name: linkedPlugin });
+//     }
+//     config.set('plugins', plugins);
+//     res.end('ok');
+//   }),
+// );
+
+// app.post(
+//   '/api/unlink-plugin',
+//   handleAsyncError(async (req, res) => {
+//     const { mainPlugin, linkedPlugin } = req.body;
+//     const plugins = config.get('plugins', {});
+
+//     if (plugins[mainPlugin]?.linkedPlugins) {
+//       _.remove(plugins[mainPlugin]?.linkedPlugins, (p) => p.name === linkedPlugin);
+//     }
+//     config.set('plugins', plugins);
+//     res.end('ok');
+//   }),
+// );
+
+app.get(
+  '/api/muse-data',
+  handleAsyncError(async (req, res) => {
+    const key = req.query.key;
+    const data = await museCore.data.get(key);
+    res.setHeader('Content-Type', 'application/json');
+    res.end(JSON.stringify(data));
+  }),
+);
+
+app.get(
+  '/api/init-data',
+  handleAsyncError(async (req, res) => {
+    const [apps, plugins] = await Promise.all([
+      museCore.data.get('muse.apps'),
+      museCore.data.get('muse.plugins'),
+    ]);
+
+    if (!apps || !plugins) throw new Error('Failed to get Muse apps or plugins.');
+
+    res.setHeader('Content-Type', 'application/json');
+    res.end(
+      JSON.stringify({
+        msgCache,
+        apps,
+        plugins,
+        https: process.env.HTTPS === 'true',
+        museLocalHost: process.env.MUSE_LOCAL_HOST_NAME || 'localhost',
+      }),
+    );
+  }),
+);
+
+app.get(
+  '/api/config-data',
+  handleAsyncError(async (req, res) => {
+    res.setHeader('Content-Type', 'application/json');
+    res.end(JSON.stringify(config.store));
+  }),
+);
+
+app.get(
+  '/api/running-data',
+  handleAsyncError((req, res) => {
+    res.setHeader('Content-Type', 'application/json');
+    res.end(JSON.stringify(getRunningData()));
+  }),
+);
+
+app.post(
+  '/api/clear-output',
+  handleAsyncError((req, res) => {
+    const { id } = req.body;
+    if (id) {
+      msgCache[id] = [];
+    }
+    res.end('ok');
+  }),
+);
+
+app.post(
+  '/api/open-code',
+  handleAsyncError(async (req, res) => {
+    const { dir } = req.body;
+    if (!fs.existsSync(dir)) throw new Error(`Folder not exist: ${dir}`);
+    exec(`code "${dir}"`);
+    res.end('ok');
+  }),
+);
+
+app.post(
+  '/api/sort-apps',
+  handleAsyncError(async (req, res) => {
+    const { appIds } = req.body;
+    const appList = config.get('appList', []);
+    appList.sort((a, b) => {
+      const aIndex = appIds.indexOf(a.id);
+      const bIndex = appIds.indexOf(b.id);
+      return aIndex - bIndex;
+    });
+    config.set('appList', appList);
+    res.end('ok');
+  }),
+);
+
+app.post(
+  '/api/sort-plugins',
+  handleAsyncError(async (req, res) => {
+    const { pluginNames, appId } = req.body;
+    const appList = config.get('appList', []);
+    const app = appList.find((a) => a.id === appId);
+    if (!app) throw new Error(`App not found: ${appId}`);
+    app.plugins.sort((a, b) => {
+      const aIndex = pluginNames.indexOf(a.name);
+      const bIndex = pluginNames.indexOf(b.name);
+      return aIndex - bIndex;
+    });
+    config.set('appList', appList);
+    res.end('ok');
+  }),
+);
+
+app.post('/api/open-browser', (req, res) => {
+  const { url } = req.body;
+  openBrowser(url);
+  res.end('ok');
+});
+
+terminals({ app, io, config });
+gitStatus({ app, io, config });
+settings({ app, io, config });
+
+// Serving runner Muse app
+// public folder is generated by "muse export muserunner staging ./public"
+app.use(express.static('./public'));
+app.use(fallback('index.html', { root: path.join(__dirname, '../public') }));
+
+app.listen(port, () => {
+  console.log(`Muse Runner started at: http://localhost:${port}`);
+  openBrowser(`http://localhost:${port}`);
+});

package/lib/test.js

@@ -0,0 +1,20 @@
+import * as os from 'node:os';
+import * as pty from 'node-pty';
+
+const shell = os.platform() === 'win32' ? 'powershell.exe' : 'bash';
+
+const ptyProcess = pty.spawn(shell, [], {
+  name: 'xterm-color',
+  cols: 80,
+  rows: 30,
+  cwd: process.env.HOME,
+  env: process.env
+});
+
+ptyProcess.onData((data) => {
+  process.stdout.write(data);
+});
+
+ptyProcess.write('ls\r');
+ptyProcess.resize(100, 40);
+ptyProcess.write('ls\r');

package/lib/upgrades/up_240102.js

@@ -0,0 +1,25 @@
+// Upgrade in 20240102
+import _ from 'lodash';
+import Conf from 'conf';
+const config = new Conf({
+  projectName: process.env.MUSE_RUNNER_CONFIG_NAME || 'muse-runner-ebay',
+});
+
+export default function upgrade240102() {
+  console.log('Checking config data...');
+  // upgrade plugin config data structure
+  if (config.get('pluginDir') && !config.get('plugins')) {
+    console.log('Updating config data...');
+    const pluginDir = config.get('pluginDir');
+    const linkedPlugins = config.get('linkedPlugins') || {};
+    const plugins = _.mapValues(pluginDir, (value, key) => ({
+      dir: value,
+      linkedPlugins: linkedPlugins[key] || undefined,
+    }));
+
+    config.set('plugins', plugins);
+    config.delete('pluginDir');
+    config.delete('linkedPlugins');
+    console.log('Config data updated.');
+  }
+}

package/lib/utils.js

@@ -0,0 +1,79 @@
+import fs from 'fs-extra';
+import path from 'path';
+import _ from 'lodash';
+
+// Detect package manager, we only support npm, yarn or pnpm.
+const pkgManagers = {
+  npm: 'package-lock.json',
+  pnpm: 'pnpm-lock.yaml',
+  yarn: 'yarn.lock',
+};
+
+const utils = {
+  getPluginInfo: (dir) => {
+    const pkgJson = fs.readJsonSync(path.join(dir, 'package.json'));
+    return {
+      name: pkgJson.name,
+      pkgJson,
+      type: pkgJson.muse?.type || 'normal',
+      esModule: pkgJson.type === 'module',
+      devConfig: pkgJson.muse?.devConfig,
+    };
+  },
+  getPkgManager: (dir) => {
+    const pmStatus = Object.entries(pkgManagers)
+      .map(([name, lockFile]) => {
+        return fs.existsSync(path.join(dir, lockFile)) ? name : null;
+      })
+      .filter(Boolean);
+
+    if (pmStatus.length > 1) {
+      throw new Error(
+        `Multiple lock files found: ${pmStatus.join(
+          ', ',
+        )}. There should be one package manager for one project.`,
+      );
+    }
+
+    return pmStatus[0] || 'pnpm';
+  },
+  getAppConfig: ({ config, runner, id }) => {
+    const appList = config.get('appList', []);
+    const plugins = config.get('plugins', {});
+    const runningPlugins = runner.runningPlugins;
+    const app = _.find(appList, { id });
+    app.plugins?.forEach((p) => {
+      const pluginConfig = plugins[p.name] || {};
+      p.dir = pluginConfig.dir;
+      const found = runningPlugins.find((p2) => p2.pluginInfo.name === p.name);
+      if (found) {
+        p.running = true;
+        p.port = found.port;
+        p.type = found.pluginInfo.type;
+        p.protocol = pluginConfig.protocol || (process.env.HTTPS === 'true' ? 'https' : 'http');
+        p.esModule = found.pluginInfo.esModule;
+      }
+
+      if (pluginConfig.linkedPlugins) {
+        p.linkedPlugins = pluginConfig.linkedPlugins.map((lp) => ({
+          name: lp.name,
+          dir: plugins[lp.name]?.dir,
+        }));
+      }
+    });
+    app.protocol = app.protocol || (process.env.HTTPS === 'true' ? 'https' : 'http');
+
+    return app;
+  },
+};
+
+export const handleAsyncError = (fn) => async (req, res) => {
+  try {
+    return await fn(req, res);
+  } catch (e) {
+    console.error(e);
+    res.status(500).send(e.message);
+  }
+};
+
+export default utils;

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@ebay/muse-runner",
+  "version": "1.0.21",
+  "description": "",
+  "main": "index.js",
+  "type": "module",
+  "bin": {
+    "muse-runner": "./bin/muse-runner.js"
+  },
+  "files": [
+    "bin",
+    "lib",
+    "public",
+    ".muserc"
+  ],
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "chalk": "^5.3.0",
+    "chokidar": "^3.5.3",
+    "conf": "^11.0.2",
+    "cors": "^2.8.5",
+    "express": "^4.18.2",
+    "express-history-api-fallback": "^2.2.1",
+    "express-ws": "^5.0.2",
+    "fs-extra": "^11.1.1",
+    "get-port": "^7.0.0",
+    "js-plugin": "^1.1.0",
+    "lodash": "^4.17.21",
+    "node-pty": "^1.0.0",
+    "package-json": "^8.1.1",
+    "react-dev-utils": "^12.0.1",
+    "semver": "^7.5.4",
+    "simple-git": "^3.20.0",
+    "@ebay/muse-dev-utils": "^1.0.72",
+    "@ebay/muse-core": "^1.0.44",
+    "@ebay/muse-client": "^1.0.27",
+    "@ebay/muse-express-middleware": "^1.0.58"
+  },
+  "devDependencies": {
+    "eslint": "^8.47.0"
+  },
+  "scripts": {
+    "start": "node lib/server.js",
+    "build": "muse export muserunner staging ./public",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  }
+}

package/public/index.html

@@ -0,0 +1,14 @@
+
+<!doctype html>
+<html lang="en">
+<head>
+  <title>Muse Runner</title>
+  <link rel="shortcut icon" href="/muse-assets/p/app-icon.muserunner/v0.0.1/dist/icon.png" />
+  <script>
+    window.MUSE_GLOBAL = {"app":{"name":"muserunner","createdBy":"pwang7","createdAt":"2023-08-21T07:14:47.583Z","owners":["pwang7","leizhang"],"pluginConfig":{"@ebay/muse-layout-antd":{"core":true}},"title":"Muse Runner","config":{"entry":"","theme":"dark","noSSO":true},"dedicatedTeam":"35728|DevX Platform","description":"NOTE: this is a Muse app placeholder, only used for export by \"muse export-app\". It's not expected to be accessed by url.","variables":{"aaa":"aaa"},"iconId":1},"env":{"name":"staging","createdBy":"pwang7","createdAt":"2023-08-21T07:15:30.612Z","url":"muserunner.muse.qa.ebay.com","e2eTestMode":"self-only","variables":{"aaa":"aaa"}},"appName":"muserunner","envName":"staging","plugins":[{"name":"@ebay/muse-boot-default","version":"1.0.25","type":"boot","variables":{"var":"test"}},{"name":"@ebay/muse-layout-antd","version":"1.1.26","type":"normal","esModule":true},{"name":"@ebay/muse-lib-antd","version":"1.2.22","type":"lib"},{"name":"@ebay/muse-lib-react","version":"1.2.20","type":"lib"},{"name":"muse-runner-ui","version":"1.0.26","type":"normal","esModule":false}],"isDev":false,"cdn":"/muse-assets","bootPlugin":"@ebay/muse-boot-default","serviceWorker":"/muse-sw.js"};
+  </script> 
+</head>
+<body></body>
+<script src="/muse-assets/p/@ebay.muse-boot-default/v1.0.25/dist/boot.js"></script>
+</html>
+  

package/public/muse-assets/p/@ebay.muse-boot-default/v1.0.25/dist/asset-manifest.json

@@ -0,0 +1,10 @@
+{
+  "files": {
+    "main.js": "/boot.js",
+    "static/media/logo.png": "/static/media/logo.0629cb217459ef0a31a2.png",
+    "boot.js.map": "/boot.js.map"
+  },
+  "entrypoints": [
+    "boot.js"
+  ]
+}