@marsx-dev/launcher 0.0.2 → 0.0.5

package/dist/utils/v3.js

@@ -42,12 +42,11 @@ const SFC_FIELD_MAP = {
 };
 function convertV3ToSfc(block) {
     const sfc = {
-        identity: {
+        path: {
             folder: block.Folder,
             name: block.Name,
             blockTypeName: block.Type,
             ext: 'vue',
-            fullName: '',
             filePath: '',
         },
         metadata: {},
@@ -83,9 +82,9 @@ function convertV3ToSfc(block) {
 exports.convertV3ToSfc = convertV3ToSfc;
 function convertSfcToV3(sfc) {
     const block = {
-        Name: sfc.identity.name,
-        Type: sfc.identity.blockTypeName,
-        Folder: sfc.identity.folder,
+        Name: sfc.path.name,
+        Type: sfc.path.blockTypeName,
+        Folder: sfc.path.folder,
         ...sfc.metadata,
         ...sfc.jsons,
     };
@@ -95,3 +94,4 @@ function convertSfcToV3(sfc) {
     return block;
 }
 exports.convertSfcToV3 = convertSfcToV3;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidjMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvdXRpbHMvdjMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBU0EsTUFBTSxhQUFhLEdBQTJCO0lBQzVDLFFBQVE7SUFDUixRQUFRLEVBQUUsTUFBTTtJQUNoQixJQUFJLEVBQUUsTUFBTTtJQUNaLEtBQUssRUFBRSxNQUFNO0lBQ2IsTUFBTSxFQUFFLE1BQU07SUFDZCxNQUFNLEVBQUUsTUFBTTtJQUNkLEtBQUssRUFBRSxNQUFNO0lBQ2IsVUFBVTtJQUNWLGFBQWEsRUFBRSxJQUFJO0lBQ25CLElBQUksRUFBRSxNQUFNO0lBQ1osSUFBSSxFQUFFLE1BQU07SUFDWixHQUFHLEVBQUUsS0FBSztJQUNWLGFBQWEsRUFBRSxJQUFJO0lBQ25CLHVCQUF1QixFQUFFLElBQUk7SUFDN0IsR0FBRyxFQUFFLEtBQUs7SUFDVixhQUFhLEVBQUUsSUFBSTtJQUNuQixPQUFPLEVBQUUsS0FBSztJQUNkLGlCQUFpQixFQUFFLElBQUk7SUFDdkIsTUFBTSxFQUFFLElBQUk7SUFDWixHQUFHLEVBQUUsS0FBSztJQUNWLFFBQVEsRUFBRSxJQUFJO0lBQ2QsY0FBYyxFQUFFLElBQUk7SUFDcEIscUJBQXFCLEVBQUUsSUFBSTtJQUMzQiw2QkFBNkI7SUFDN0IsaUJBQWlCO0lBQ2pCLElBQUksRUFBRSxRQUFRO0lBQ2QsSUFBSSxFQUFFLFFBQVE7SUFDZCxNQUFNLEVBQUUsUUFBUTtJQUNoQixrQ0FBa0M7SUFDbEMsR0FBRyxFQUFFLFFBQVE7SUFDYixPQUFPLEVBQUUsUUFBUTtJQUNqQixXQUFXLEVBQUUsUUFBUTtJQUNyQixTQUFTLEVBQUUsUUFBUTtJQUNuQixTQUFTLEVBQUUsUUFBUTtJQUNuQixTQUFTLEVBQUUsUUFBUTtJQUNuQixTQUFTLEVBQUUsUUFBUTtJQUNuQixtREFBbUQ7Q0FDcEQsQ0FBQztBQUVGLFNBQWdCLGNBQWMsQ0FBQyxLQUFtQjtJQUNoRCxNQUFNLEdBQUcsR0FBYTtRQUNwQixJQUFJLEVBQUU7WUFDSixNQUFNLEVBQUUsS0FBSyxDQUFDLE1BQU07WUFDcEIsSUFBSSxFQUFFLEtBQUssQ0FBQyxJQUFJO1lBQ2hCLGFBQWEsRUFBRSxLQUFLLENBQUMsSUFBSTtZQUN6QixHQUFHLEVBQUUsS0FBSztZQUNWLFFBQVEsRUFBRSxFQUFFO1NBQ2I7UUFDRCxRQUFRLEVBQUUsRUFBRTtRQUNaLEtBQUssRUFBRSxFQUFFO1FBQ1QsT0FBTyxFQUFFLEVBQUU7UUFDWCxVQUFVLEVBQUUsSUFBSTtLQUNqQixDQUFDO0lBQ0YsS0FBSyxNQUFNLENBQUMsSUFBSSxFQUFFLEtBQUssQ0FBQyxJQUFJLE1BQU0sQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUU7UUFDakQsTUFBTSxPQUFPLEdBQUcsYUFBYSxDQUFDLElBQUksQ0FBQyxJQUFJLFVBQVUsQ0FBQztRQUVsRCxRQUFRLE9BQU8sRUFBRTtZQUNmLEtBQUssTUFBTTtnQkFDVCxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVEsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDO29CQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsbUNBQW1DLElBQUksV0FBVyxDQUFDLENBQUM7Z0JBQzVILEdBQUcsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLEdBQUcsS0FBSyxDQUFDO2dCQUN4QixNQUFNO1lBQ1IsS0FBSyxRQUFRO2dCQUNYLE1BQU07WUFDUixLQUFLLFVBQVU7Z0JBQ2IsR0FBRyxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsR0FBRyxLQUFLLENBQUM7Z0JBQzNCLE1BQU07WUFDUjtnQkFDRSxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVE7b0JBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQywwQkFBMEIsSUFBSSxXQUFXLENBQUMsQ0FBQztnQkFDMUYsR0FBRyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLE1BQU0sRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFFLE9BQU8sRUFBRSxDQUFDO2dCQUNyRCxNQUFNO1NBQ1Q7S0FDRjtJQUVELElBQUksS0FBSyxDQUFDLEdBQUcsRUFBRTtRQUNiLEdBQUcsQ0FBQyxRQUFRLENBQUMsS0FBSyxDQUFDLEdBQUcsRUFBRSxJQUFJLEVBQUUsS0FBSyxDQUFDLEdBQUcsQ0FBQyxJQUFJLEVBQUUsQ0FBQztLQUNoRDtJQUNELE9BQU8sR0FBRyxDQUFDO0FBQ2IsQ0FBQztBQXRDRCx3Q0FzQ0M7QUFFRCxTQUFnQixjQUFjLENBQUMsR0FBYTtJQUMxQyxNQUFNLEtBQUssR0FBaUI7UUFDMUIsSUFBSSxFQUFFLEdBQUcsQ0FBQyxJQUFJLENBQUMsSUFBSTtRQUNuQixJQUFJLEVBQUUsR0FBRyxDQUFDLElBQUksQ0FBQyxhQUFhO1FBQzVCLE1BQU0sRUFBRSxHQUFHLENBQUMsSUFBSSxDQUFDLE1BQU07UUFDdkIsR0FBRyxHQUFHLENBQUMsUUFBUTtRQUNmLEdBQUcsR0FBRyxDQUFDLEtBQUs7S0FDYixDQUFDO0lBRUYsS0FBSyxNQUFNLENBQUMsR0FBRyxFQUFFLEtBQUssQ0FBQyxJQUFJLE1BQU0sQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLE9BQU8sQ0FBQyxFQUFFO1FBQ3RELEtBQUssQ0FBQyxHQUFHLENBQUMsR0FBRyxLQUFLLENBQUMsTUFBTSxDQUFDO0tBQzNCO0lBRUQsT0FBTyxLQUFLLENBQUM7QUFDZixDQUFDO0FBZEQsd0NBY0MifQ==

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@marsx-dev/launcher",
-  "version": "0.0.2",
+  "version": "0.0.5",
   "description": "MarsX launcher",
   "author": "MarsX team <team@marsx.dev>",
   "homepage": "https://marsx.dev/",
@@ -19,6 +19,7 @@
   },
   "files": [
     "dist",
+    "src",
     "LICENSE",
     "README.md"
   ],
@@ -39,67 +40,27 @@
   },
   "dependencies": {
     "@vue/compiler-sfc": "^3.2.29",
-    "axios": "^0.26.1",
     "chalk": "^4.1.2",
     "commander": "^9.1.0",
-    "config": "^3.3.7",
     "json-stable-stringify": "^1.0.1",
     "json5": "^2.2.0",
+    "yup": "^0.32.11"
+  },
+  "peerDependencies": {
+    "@types/config": "^0.0.41",
+    "@types/lodash": "^4.14.178",
+    "@types/node": "^17.0.21",
+    "@types/prettier": "^2.4.4",
+    "axios": "^0.26.1",
+    "config": "^3.3.7",
     "lodash": "^4.17.21",
     "mongodb": "^4.3.1",
     "prettier": "^2.5.1",
     "typescript": "^4.5.5"
   },
-  "peerDependencies": {
-    "@fastify/session": "^6.4.0",
-    "@stitches/core": "^1.2.6",
-    "@types/accepts": "^1.3.5",
-    "@types/bcrypt": "^5.0.0",
-    "@types/mkdirp": "^1.0.2",
-    "@types/nodegit": "^0.27.9",
-    "@types/sharp": "^0.29.5",
-    "@types/uuid": "^8.3.4",
-    "@types/ws": "^8.5.2",
-    "aws-sdk": "^2.1073.0",
-    "azure-storage": "^2.10.7",
-    "babel-core": "^6.26.3",
-    "babel-plugin-transform-react-jsx": "^6.24.1",
-    "base-x": "^4.0.0",
-    "bcrypt": "^5.0.1",
-    "chokidar": "^3.5.3",
-    "dotenv": "^16.0.0",
-    "fastify": "^3.27.2",
-    "fastify-accepts": "^2.1.0",
-    "fastify-compress": "^4.0.1",
-    "fastify-cookie": "^5.6.0",
-    "fastify-cors": "^6.0.3",
-    "fastify-csrf": "^3.1.0",
-    "fastify-flash": "^2.0.2",
-    "fastify-formbody": "^5.2.0",
-    "fastify-helmet": "^7.0.1",
-    "fastify-multipart": "^5.3.1",
-    "fastify-request-context": "^2.2.0",
-    "fastify-static": "^4.5.0",
-    "fastify-websocket": "^4.2.0",
-    "fontverter": "^2.0.0",
-    "jsonwebtoken": "^8.5.1",
-    "mobile-detect": "^1.4.5",
-    "moment-timezone": "^0.5.34",
-    "nodegit": "^0.28.0-alpha.10",
-    "route-pattern": "^0.0.6",
-    "route-sort": "^1.0.0",
-    "sharp": "^0.30.1",
-    "source-map": "^0.7.3",
-    "subset-font": "^1.4.0",
-    "uuid": "^8.3.2",
-    "xxhash": "^0.3.0"
-  },
   "devDependencies": {
-    "@types/config": "^0.0.41",
     "@types/json-stable-stringify": "^1.0.34",
-    "@types/lodash": "^4.14.178",
-    "@types/node": "^17.0.21",
-    "@types/prettier": "^2.4.4",
+    "@types/yup": "^0.29.13",
     "@typescript-eslint/eslint-plugin": "^5.10.2",
     "@typescript-eslint/parser": "^5.10.2",
     "eslint": "^8.8.0",

package/src/cli/index.ts

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+
+import chalk from 'chalk';
+import { Command } from 'commander';
+import { initProject } from './init';
+import { migrateV3ToV4 } from './migrate';
+import { start } from './start';
+
+export class CliError extends Error {}
+
+async function main() {
+  try {
+    const program = new Command();
+
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const version = require('../../package.json').version;
+    program.name('marsx').description('CLI for MarsX launcher').version(version);
+
+    program.command('start', { isDefault: true }).description('Start MarsX server').action(start);
+    program
+      .command('init <project_name>')
+      .description('Initialize a new MarsX project named <project_name> in a directory of the same name')
+      .action(initProject);
+    program.command('migrate').description('Migrate MarsX V3 project to V4').action(migrateV3ToV4);
+
+    await program.parseAsync();
+  } catch (e) {
+    if (e instanceof CliError) {
+      console.error(chalk.red(e.message));
+    } else {
+      console.error(chalk.red(e));
+    }
+    process.exit(1);
+  }
+}
+
+main();

package/src/cli/init.ts

@@ -0,0 +1,142 @@
+import chalk from 'chalk';
+import { spawnSync } from 'child_process';
+import { randomBytes } from 'crypto';
+import path from 'path';
+import { Config, CustomEnvironmentVariables } from '../configuration';
+import { isDirectory, writeFileMakeDir } from '../utils/fileUtils';
+import { assert } from '../utils/textUtils';
+import { CliError } from './index';
+
+const FASTIFY_DEPS = [
+  '@fastify/session',
+  'fastify',
+  'fastify-accepts',
+  'fastify-compress',
+  'fastify-cookie',
+  'fastify-cors',
+  'fastify-csrf',
+  'fastify-flash',
+  'fastify-formbody',
+  'fastify-helmet',
+  'fastify-multipart',
+  'fastify-request-context',
+  'fastify-static',
+  'fastify-websocket',
+];
+
+const COMMON_DEPS = [
+  '@marsx-dev/launcher',
+  'aws-sdk',
+  'axios',
+  'azure-storage',
+  'base-x',
+  'bcrypt',
+  'chokidar',
+  'lodash',
+  'mongodb4@npm:mongodb@4',
+  'mongodb3@npm:mongodb@3',
+  'typescript',
+  'uuid',
+  'xxhash',
+];
+
+const V3_DEPS = [
+  'babel-core',
+  'babel-plugin-transform-react-jsx',
+  'config',
+  'crypto-js',
+  'jsonwebtoken',
+  'mobile-detect',
+  'moment-timezone',
+  'route-pattern',
+  'route-sort',
+];
+
+const DEFAULT_DEPS = [...FASTIFY_DEPS, ...COMMON_DEPS, ...V3_DEPS];
+
+export async function initProject(projectName: string) {
+  if (!projectName.match(/^\w+$/) || projectName !== projectName.toLowerCase()) {
+    throw new CliError(
+      `Project name "${projectName}" may contain only lower case alphanumeric characters and underscores (eg. my_project_name)`,
+    );
+  }
+  const name = {
+    snakeCase: projectName,
+    withDashes: projectName.replace('_', '-'),
+    noSep: projectName.replace('_', ''),
+  };
+
+  const projectDir = path.resolve(projectName);
+
+  if (await isDirectory(projectDir)) {
+    throw new CliError(`Project dir "${projectName}" already exist. Delete it or choose a different name.`);
+  }
+
+  console.log(`Creating a new MarsX project in ${chalk.green(projectDir)}`);
+
+  const packageJson = {
+    name: name.withDashes,
+    version: '0.0.0',
+    private: true,
+    scripts: {
+      start: 'marsx start',
+    },
+    dependencies: {},
+  };
+  const packageJsonStr = JSON.stringify(packageJson, null, 2);
+  await writeFileMakeDir(path.join(projectDir, 'package.json'), packageJsonStr);
+
+  const config: Config = {
+    production: false,
+    port: 3000,
+    blocksDir: 'blocks',
+    cacheDir: '.cache',
+    mongoConn: '<CONN_STR>',
+    mongoDbName: name.withDashes,
+    azureStorageConnection: '<CONN_STR>',
+    azureStorageAccountName: name.noSep,
+    azureStorageUrl: `https://${name.noSep}.blob.core.windows.net`,
+    webFilesTable: 'webFiles',
+    webRecentFilesTable: 'webRecentFiles',
+    webFilesBlob: 'web-files',
+    secret: (await randomBytes(32)).toString('hex'),
+    importProjects: [
+      {
+        name: 'marsx-core',
+        url: 'https://core.marsx.dev',
+        api_key: '<API_KEY>',
+        git_commit_ish: 'main',
+      },
+    ],
+  };
+
+  await writeFileMakeDir(path.join(projectDir, 'config', 'default.json'), JSON.stringify(config, null, 2));
+  await writeFileMakeDir(
+    path.join(projectDir, 'config', 'custom-environment-variables.json'),
+    JSON.stringify(CustomEnvironmentVariables, null, 2),
+  );
+  await writeFileMakeDir(path.join(projectDir, '.gitignore'), 'node_modules\ndist\n.cache\n');
+
+  function run(...args: string[]) {
+    const cmd = args[0];
+    assert(cmd);
+    spawnSync(cmd, args.slice(1), { cwd: projectDir, stdio: 'inherit' });
+  }
+
+  console.log('\nInstalling dependencies. This might take a couple of minutes.');
+  run('npm', 'i', ...DEFAULT_DEPS);
+  console.log('');
+
+  run('git', 'init', '--initial-branch=main');
+  run('git', 'add', '.gitignore');
+  run('git', 'add', '-A');
+  run('git', 'commit', '-m', 'Initial MarsX commit');
+
+  console.log(`\n${chalk.green('Success!')} Created ${projectName} at ${projectDir}`);
+
+  console.log(`\nNext steps:`);
+  console.log(`  - Open new project directory: ${chalk.underline(`cd ${projectName}`)}`);
+  console.log(`  - Contact MarsX to get config params and update them: ${chalk.underline('code config/default.json')}`);
+  console.log(`  - (optionally) Migrate V3 blocks using: ${chalk.underline('npx marsx migrate')}`);
+  console.log(`  - Start local server: ${chalk.underline('npm run start')}`);
+}

package/src/cli/migrate.ts

@@ -0,0 +1,23 @@
+import chalk from 'chalk';
+import { MongoClient } from 'mongodb';
+import path from 'path';
+import { config } from '../configuration';
+import { serializeSfc } from '../utils/sfc';
+import { writeFileMakeDir } from '../utils/fileUtils';
+import { convertV3ToSfc, V3MongoBlock } from '../utils/v3';
+
+export async function migrateV3ToV4() {
+  console.log('Connecting to MongoDB...');
+  const mongoConn = await new MongoClient(config.mongoConn).connect();
+  const db = mongoConn.db(config.mongoDbName);
+  const allBlocks = await db.collection<V3MongoBlock>('blocks').find().toArray();
+  console.log(`Downloaded ${allBlocks.length} block(s)`);
+
+  for (const block of allBlocks) {
+    const serialized = serializeSfc(convertV3ToSfc(block));
+    await writeFileMakeDir(path.join(config.blocksDir, serialized.filePath), serialized.content);
+    console.log(`Saved ${serialized.filePath}`);
+  }
+
+  console.log(chalk.green('\nMigration complete successfully!'));
+}

package/src/cli/start.ts

@@ -0,0 +1,34 @@
+import child_process from 'child_process';
+import { config } from '../configuration';
+import { launchBooter } from '../launcher';
+import { assert } from '../utils/textUtils';
+
+export const RESTART_EXIT_CODe = 9;
+
+export const start = async () => {
+  if (config.production || process.env['MARSX_NO_SPAWN']) {
+    console.log(`MarsX process ${process.pid} starting...`);
+    await launchBooter();
+    console.log(`MarsX process ${process.pid} started`);
+  } else {
+    const spawnChildProc = () => {
+      assert(process.argv[0]);
+      const child = child_process.spawn(process.argv[0], ['--inspect', '--enable-source-maps', ...process.argv.slice(1)], {
+        cwd: process.cwd(),
+        env: { ...process.env, MARSX_NO_SPAWN: 'true' },
+        stdio: 'inherit',
+      });
+
+      child.on('close', (code: number) => {
+        console.log(`MarsX process ${child.pid} terminated with ${code}`);
+        if (code === RESTART_EXIT_CODe) {
+          setTimeout(spawnChildProc, 100);
+        } else {
+          process.exit(code);
+        }
+      });
+    };
+
+    spawnChildProc();
+  }
+};

package/src/configuration.ts

@@ -0,0 +1,104 @@
+process.env['SUPPRESS_NO_CONFIG_WARNING'] = 'true';
+import configModule from 'config';
+import _ from 'lodash';
+import path from 'path';
+import * as yup from 'yup';
+import { ValidationError } from 'yup';
+
+export interface ImportProjectConfig {
+  name: string;
+  url: string;
+  api_key: string;
+  git_commit_ish?: string | undefined;
+}
+
+export interface Config {
+  production: boolean;
+  port: number;
+  blocksDir: string;
+  cacheDir: string;
+  mongoConn: string;
+  mongoDbName: string;
+  azureStorageConnection: string;
+  azureStorageAccountName: string;
+  azureStorageUrl: string;
+  secret: string;
+  webFilesTable: string;
+  webRecentFilesTable: string;
+  webFilesBlob: string;
+  importProjects: ImportProjectConfig[];
+}
+
+export class ConfigError extends Error {}
+
+const ImportProjectSchema = yup.object({
+  name: yup.string().required(),
+  url: yup.string().url().required(),
+  api_key: yup.string().required(),
+  git_commit_ish: yup.string().optional(),
+});
+
+const ConfigSchema = yup.object().shape({
+  production: yup.boolean().required(),
+  port: yup.number().required().positive().integer(),
+  blocksDir: yup
+    .string()
+    .required()
+    .transform(v => path.resolve(v)),
+  cacheDir: yup
+    .string()
+    .required()
+    .transform(v => path.resolve(v)),
+  mongoConn: yup.string().required(),
+  mongoDbName: yup.string().required(),
+  azureStorageConnection: yup.string().required(),
+  azureStorageAccountName: yup.string().required(),
+  azureStorageUrl: yup.string().required(),
+  secret: yup.string().required(),
+  webFilesTable: yup.string().required(),
+  webRecentFilesTable: yup.string().required(),
+  webFilesBlob: yup.string().required(),
+  importProjects: yup.array().of(ImportProjectSchema).required(),
+});
+
+function ErrorThrowingConfig(errorMessage: string) {
+  return new Proxy({} as never, {
+    get() {
+      throw new ConfigError(errorMessage);
+    },
+  });
+}
+
+function validateConfig(): Config {
+  if (configModule.util.getConfigSources().length === 0) {
+    return ErrorThrowingConfig('Config file not found, ensure you have "config/default.json" file.');
+  }
+
+  const configObject = configModule.util.toObject();
+  try {
+    return ConfigSchema.validateSync(configObject, { abortEarly: false, stripUnknown: false });
+  } catch (e) {
+    if (e instanceof ValidationError) {
+      return ErrorThrowingConfig(e.errors.join('\n'));
+    } else {
+      throw e;
+    }
+  }
+}
+
+export const config = validateConfig();
+
+function getEnvVarMapping() {
+  const result: Record<string, string | { __name: string; __format: string }> = {};
+  for (const [name, field] of Object.entries(ConfigSchema.fields)) {
+    const envVar = `MARSX_${_.snakeCase(name).toUpperCase()}`;
+    if ('type' in field && (field.type === 'object' || field.type === 'array')) {
+      result[name] = { __name: envVar, __format: 'json' };
+    } else {
+      result[name] = envVar;
+    }
+  }
+  return result;
+}
+
+export const CustomEnvironmentVariables = getEnvVarMapping();

package/src/index.ts

@@ -0,0 +1,6 @@
+export { config, ImportProjectConfig, Config } from './configuration';
+export { compileSfcSource, transpileTypescript } from './utils/compile';
+export { parseSFC, serializeSfc, parseSfcPath, serializeSfcPath, SfcBlock, SfcPath, SfcSource } from './utils/sfc';
+export { convertSfcToV3, convertV3ToSfc, V3MongoBlock } from './utils/v3';
+export * as textUtils from './utils/textUtils';
+export * as fileUtils from './utils/fileUtils';

package/src/launcher.ts

@@ -0,0 +1,39 @@
+import { config } from './configuration';
+import { loadAllBlocks } from './loader';
+import { compileSfcSource, transpileTypescript } from './utils/compile';
+import { parseSFC, serializeSfc } from './utils/sfc';
+import { writeFileMakeDir } from './utils/fileUtils';
+import { assert } from './utils/textUtils';
+import { convertSfcToV3, convertV3ToSfc } from './utils/v3';
+
+const LAUNCHER_UTILS = {
+  config,
+  parseSFC,
+  serializeSfc,
+  convertV3ToSfc,
+  convertSfcToV3,
+  transpileTypescript,
+  compileSfcSource,
+  writeFileMakeDir,
+};
+
+export async function launchBooter(booterBlockName = 'Booter') {
+  const allBlocks = await loadAllBlocks();
+
+  const booterBlocks = allBlocks.filter(b => b.path.name === booterBlockName);
+  if (booterBlocks.length === 0)
+    throw new Error(`Booter block ${booterBlockName} not found. Ensure you have it locally or it is imported.`);
+
+  // Take last because order of imports matter and local blocks always override imported.
+  const booterBlock = booterBlocks[booterBlocks.length - 1];
+  assert(booterBlock);
+
+  const compiledBooterPath = await compileSfcSource(booterBlock, 'BlockFunction');
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    await require(compiledBooterPath).default(allBlocks, LAUNCHER_UTILS);
+  } catch (e) {
+    console.error('Booter failed with:', e);
+    process.exit(1);
+  }
+}

package/src/loader.ts

@@ -0,0 +1,65 @@
+import axios from 'axios';
+import crypto from 'crypto';
+import { promises as fs } from 'fs';
+import stringify from 'json-stable-stringify';
+import _ from 'lodash';
+import path from 'path';
+import { config, ImportProjectConfig } from './configuration';
+import { parseSFC, SfcBlock } from './utils/sfc';
+import { isFile, listFilesRecursive, writeFileMakeDir } from './utils/fileUtils';
+import { convertV3ToSfc, V3MongoBlock } from './utils/v3';
+
+async function downloadFromExternal(externalImport: ImportProjectConfig): Promise<SfcBlock[]> {
+  const params = { api_key: externalImport.api_key, git_commit_ish: externalImport.git_commit_ish || '' };
+
+  console.log(`Downloading blocks from ${externalImport.url}`);
+  try {
+    const v4Resp = await axios.get<SfcBlock[]>(`${externalImport.url}/api/GetExportedAppBlocksV4`, { params });
+    return v4Resp.data;
+  } catch (e) {
+    console.log(`${externalImport.url} does not support V4, fallback to V3`);
+  }
+
+  const v3Resp = await axios.get<V3MongoBlock[]>(`${externalImport.url}/api/GetExportedAppBlocks`, { params });
+  return v3Resp.data.map(b => convertV3ToSfc(b));
+}
+
+async function loadCachedOrDownload(externalImport: ImportProjectConfig): Promise<SfcBlock[]> {
+  const hash = crypto.createHash('md5').update(stringify(externalImport)).digest('hex');
+  const cacheFileName = `${externalImport.name}_${externalImport.git_commit_ish}_${hash}.json`.replace(/[^\w.]+/g, '_');
+  const cacheFilePath = path.join(config.cacheDir, 'imports', cacheFileName);
+
+  if (await isFile(cacheFilePath)) {
+    const content = await fs.readFile(cacheFilePath);
+    console.log(`Loading cached blocks from ${externalImport.url}`);
+    return JSON.parse(content.toString('utf-8'));
+  }
+
+  const data = await downloadFromExternal(externalImport);
+  console.log(`Downloaded ${data.length} blocks from ${externalImport.url}`);
+
+  await writeFileMakeDir(cacheFilePath, JSON.stringify(data, null, 2), 'utf-8');
+  return data;
+}
+
+async function downloadAll(externalImports: ImportProjectConfig[]) {
+  const responses = await Promise.all(externalImports.map(loadCachedOrDownload));
+  return _.flatten(responses);
+}
+
+async function readBlockFile(blocksDir: string, filePath: string): Promise<SfcBlock> {
+  const content = await fs.readFile(filePath);
+  const relPath = path.relative(blocksDir, filePath);
+  return parseSFC(relPath, content);
+}
+
+async function readBlockFiles(blocksDir: string) {
+  const files = await listFilesRecursive(blocksDir);
+  return await Promise.all(files.filter(f => !path.basename(f).startsWith('.')).map(f => readBlockFile(blocksDir, f)));
+}
+
+export async function loadAllBlocks(): Promise<SfcBlock[]> {
+  const externalBlocks = await downloadAll(config.importProjects);
+  const localBlocks = await readBlockFiles(config.blocksDir);
+  return [...externalBlocks, ...localBlocks];
+}

package/src/utils/compile.ts

@@ -0,0 +1,59 @@
+import { SourceMapPayload } from 'module';
+import path from 'path';
+import ts from 'typescript';
+import { config } from '../configuration';
+import { SfcBlock } from './sfc';
+import { writeFileMakeDir } from './fileUtils';
+import { assert } from './textUtils';
+
+function genSourceMapComment(sourceMap: SourceMapPayload): string {
+  return `\n//# sourceMappingURL=data:application/json;base64,${Buffer.from(JSON.stringify(sourceMap)).toString('base64')}`;
+}
+
+export function transpileTypescript(
+  sourceCode: string,
+  options?: {
+    sourceRoot?: string;
+    originalFilePath?: string;
+    compiledFilePath?: string;
+    lineOffset?: number | undefined;
+  },
+): string {
+  const compiledFilePath = options?.compiledFilePath || '<UNKNOWN>.js';
+  const originalFilePath = options?.originalFilePath || '<UNKNOWN>.ts';
+
+  if (options?.lineOffset) {
+    sourceCode = '\n'.repeat(options.lineOffset) + sourceCode;
+  }
+
+  const transpiled = ts.transpileModule(sourceCode, {
+    compilerOptions: {
+      module: ts.ModuleKind.CommonJS,
+      target: ts.ScriptTarget.Latest,
+      esModuleInterop: true,
+      sourceMap: true,
+      // inlineSources: true,
+      // inlineSourceMap: true,
+    },
+  });
+  assert(transpiled.sourceMapText);
+
+  const sourceMap: SourceMapPayload = JSON.parse(transpiled.sourceMapText);
+  sourceMap.sourceRoot = options?.sourceRoot || '';
+  sourceMap.file = compiledFilePath;
+  sourceMap.sources = [originalFilePath];
+  const sourceMapComment = genSourceMapComment(sourceMap);
+  return transpiled.outputText.replace(/\n\/\/# sourceMappingURL=.+/, '') + sourceMapComment;
+}
+
+export async function compileSfcSource(sfcBlock: SfcBlock, sourceId: string): Promise<string> {
+  const sourceCode = sfcBlock.sources[sourceId];
+  if (!sourceCode) throw new Error(`Source code block ${sourceId} not found in ${sfcBlock.path.filePath}`);
+
+  const compiledFilePath = path.join(config.cacheDir, 'compiled', `${sfcBlock.path.filePath}.${sourceId}.js`);
+  const originalFilePath = path.join(config.blocksDir, sfcBlock.path.filePath);
+  const code = transpileTypescript(sourceCode.source, { compiledFilePath, originalFilePath, lineOffset: sourceCode.lineOffset });
+
+  await writeFileMakeDir(compiledFilePath, code, 'utf-8');
+  return compiledFilePath;
+}