@travetto/compiler 3.0.0-rc.8 → 3.0.0

package/support/transpile.ts

@@ -0,0 +1,210 @@
+import path from 'path';
+import fs from 'fs/promises';
+import os from 'os';
+import cp from 'child_process';
+import { createRequire } from 'module';
+
+import { DeltaEvent, ManifestContext, ManifestRoot, Package } from '@travetto/manifest';
+
+import { LogUtil } from './log';
+
+type ModFile = { input: string, output: string, stale: boolean };
+export type CompileResult = 'restart' | 'complete' | 'skipped';
+
+const OPT_CACHE: Record<string, import('typescript').CompilerOptions> = {};
+const SRC_REQ = createRequire(path.resolve('node_modules'));
+const RECENT_STAT = (stat: { ctimeMs: number, mtimeMs: number }): number => Math.max(stat.ctimeMs, stat.mtimeMs);
+
+/**
+ * Transpile utilities for launching
+ */
+export class TranspileUtil {
+  /**
+   * Write text file, and ensure folder exists
+   */
+  static writeTextFile = (file: string, content: string): Promise<void> =>
+    fs.mkdir(path.dirname(file), { recursive: true }).then(() => fs.writeFile(file, content, 'utf8'));
+
+  /**
+   * Returns the compiler options
+   */
+  static async getCompilerOptions(ctx: ManifestContext): Promise<{}> {
+    if (!(ctx.workspacePath in OPT_CACHE)) {
+      let tsconfig = path.resolve(ctx.workspacePath, 'tsconfig.json');
+
+      if (!await fs.stat(tsconfig).then(_ => true, _ => false)) {
+        tsconfig = SRC_REQ.resolve('@travetto/compiler/tsconfig.trv.json');
+      }
+
+      const ts = (await import('typescript')).default;
+
+      const { options } = ts.parseJsonSourceFileConfigFileContent(
+        ts.readJsonConfigFile(tsconfig, ts.sys.readFile), ts.sys, ctx.workspacePath
+      );
+
+      OPT_CACHE[ctx.workspacePath] = {
+        ...options,
+        allowJs: true,
+        sourceMap: false,
+        inlineSourceMap: true,
+        resolveJsonModule: true,
+        sourceRoot: ctx.workspacePath,
+        rootDir: ctx.workspacePath,
+        outDir: path.resolve(ctx.workspacePath),
+        module: ctx.moduleType === 'commonjs' ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext,
+      };
+    }
+    return OPT_CACHE[ctx.workspacePath];
+  }
+
+  /**
+   * Output a file, support for ts, js, and package.json
+   */
+  static async transpileFile(ctx: ManifestContext, inputFile: string, outputFile: string): Promise<void> {
+    if (inputFile.endsWith('.ts') || inputFile.endsWith('.js')) {
+      const compilerOut = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules');
+
+      const text = (await fs.readFile(inputFile, 'utf8'))
+        .replace(/from '([.][^']+)'/g, (_, i) => `from '${i.replace(/[.]js$/, '')}.js'`)
+        .replace(/from '(@travetto\/(.*?))'/g, (_, i, s) => `from '${path.resolve(compilerOut, `${i}${s.includes('/') ? '.js' : '/__index__.js'}`)}'`);
+
+      const ts = (await import('typescript')).default;
+      const content = ts.transpile(text, await this.getCompilerOptions(ctx), inputFile);
+      await this.writeTextFile(outputFile, content);
+    } else if (inputFile.endsWith('package.json')) {
+      const pkg: Package = JSON.parse(await fs.readFile(inputFile, 'utf8'));
+      const main = pkg.main?.replace(/[.]ts$/, '.js');
+      const files = pkg.files?.map(x => x.replace('.ts', '.js'));
+
+      const content = JSON.stringify({ ...pkg, main, type: ctx.moduleType, files }, null, 2);
+      await this.writeTextFile(outputFile, content);
+    }
+  }
+
+  /**
+   * Scan directory to find all project sources for comparison
+   */
+  static async getModuleSources(ctx: ManifestContext, module: string, seed: string[]): Promise<ModFile[]> {
+    const inputFolder = (ctx.mainModule === module) ?
+      process.cwd() :
+      path.dirname(SRC_REQ.resolve(`${module}/package.json`));
+
+    const folders = seed.filter(x => !/[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+    const files = seed.filter(x => /[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+
+    while (folders.length) {
+      const sub = folders.pop();
+      if (!sub) {
+        continue;
+      }
+
+      for (const file of await fs.readdir(sub).catch(() => [])) {
+        if (file.startsWith('.')) {
+          continue;
+        }
+        const resolvedInput = path.resolve(sub, file);
+        const stat = await fs.stat(resolvedInput);
+
+        if (stat.isDirectory()) {
+          folders.push(resolvedInput);
+        } else if (file.endsWith('.d.ts')) {
+          // Do nothing
+        } else if (file.endsWith('.ts') || file.endsWith('.js')) {
+          files.push(resolvedInput);
+        }
+      }
+    }
+
+    const outputFolder = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', module);
+    const out: ModFile[] = [];
+    for (const input of files) {
+      const output = input.replace(inputFolder, outputFolder).replace(/[.]ts$/, '.js');
+      const inputTs = await fs.stat(input).then(RECENT_STAT, () => 0);
+      if (inputTs) {
+        const outputTs = await fs.stat(output).then(RECENT_STAT, () => 0);
+        await fs.mkdir(path.dirname(output), { recursive: true, });
+        out.push({ input, output, stale: inputTs > outputTs });
+      }
+    }
+
+    return out;
+  }
+
+  /**
+   * Recompile folder if stale
+   */
+  static async compileIfStale(ctx: ManifestContext, scope: string, mod: string, seed: string[]): Promise<string[]> {
+    const files = await this.getModuleSources(ctx, mod, seed);
+    const changes = files.filter(x => x.stale).map(x => x.input);
+    const out: string[] = [];
+
+    try {
+      await LogUtil.withLogger(scope, async log => {
+        if (files.some(f => f.stale)) {
+          log('debug', 'Starting');
+          for (const file of files.filter(x => x.stale)) {
+            await this.transpileFile(ctx, file.input, file.output);
+          }
+          if (changes.length) {
+            out.push(...changes.map(x => `${mod}/${x}`));
+            log('debug', `Source changed: ${changes.join(', ')}`);
+          }
+          log('debug', 'Completed');
+        } else {
+          log('debug', 'Skipped');
+        }
+      }, false, [mod]);
+    } catch (err) {
+      console.error(err);
+    }
+    return out;
+  }
+
+  /**
+   * Run compiler
+   */
+  static async runCompiler(ctx: ManifestContext, manifest: ManifestRoot, changed: DeltaEvent[], watch: boolean, onMessage: (msg: unknown) => void): Promise<CompileResult> {
+    const compiler = path.resolve(ctx.workspacePath, ctx.compilerFolder);
+    const main = path.resolve(compiler, 'node_modules', '@travetto/compiler/support/compiler-entry.js');
+    const deltaFile = path.resolve(os.tmpdir(), `manifest-delta.${process.pid}.${process.ppid}.${Date.now()}.json`);
+
+    const changedFiles = changed[0]?.file === '*' ? ['*'] : changed.map(ev =>
+      path.resolve(manifest.workspacePath, manifest.modules[ev.module].sourceFolder, ev.file)
+    );
+
+    let proc: cp.ChildProcess | undefined;
+    let kill: (() => void) | undefined;
+
+    try {
+      await this.writeTextFile(deltaFile, changedFiles.join('\n'));
+
+      return await LogUtil.withLogger('compiler-exec', log => new Promise<CompileResult>((res, rej) => {
+        proc = cp.spawn(process.argv0, [main, deltaFile, `${watch}`], {
+          env: {
+            ...process.env,
+            TRV_MANIFEST: path.resolve(ctx.workspacePath, ctx.outputFolder, 'node_modules', ctx.mainModule),
+          },
+          stdio: [0, 1, 2, 'ipc'],
+        })
+          .on('message', msg => {
+            if (LogUtil.isLogEvent(msg)) {
+              log(...msg);
+            } else if (msg === 'restart') {
+              res(msg);
+            } else {
+              onMessage(msg);
+            }
+          })
+          .on('exit', code => (code !== null && code > 0) ? rej(new Error('Failed during compilation')) : res('complete'));
+        kill = (): void => { proc?.kill('SIGKILL'); };
+        process.on('exit', kill);
+      }));
+    } finally {
+      if (proc?.killed === false) { proc.kill('SIGKILL'); }
+      if (kill) {
+        process.removeListener('exit', kill);
+      }
+      await fs.rm(deltaFile, { force: true });
+    }
+  }
+}

package/tsconfig.trv.json

@@ -17,6 +17,7 @@
     "inlineSourceMap": false,
     "removeComments": true,
     "importHelpers": true,
+    "noEmitHelpers": true,
     "outDir": "build/"
   },
 }

package/bin/transpile.d.ts

@@ -1,32 +0,0 @@
-import type { ManifestContext, Package } from '@travetto/manifest';
-
-declare namespace Transpile {
-  type CompileCommand = 'build' | 'watch' | 'clean' | 'manifest';
-
-  /**
-   * Writes a package json file
-   */
-  function writePackageJson(ctx: ManifestContext, inputFile: string, outputFile: string, transform?: (pkg: Package) => Package): Promise<void>;
-
-  /**
-   * Transpiles a file
-   */
-  function transpileFile(ctx: ManifestContext, inputFile: string, outputFile: string): Promise<void>;
-
-  /**
-   * Write js file
-   */
-  function writeJsFile(ctx: ManifestContext, inputFile: string, outputFile: string): Promise<void>;
-
-  /**
-   * Build an entire package
-   */
-  function buildPackage(ctx: ManifestContext, name: string, sourcePath: string, mainSource: string, extraSource: string[]): Promise<string>;
-
-  /**
-   * Get Context for building
-   */
-  function getContext(folder?: string): Promise<ManifestContext>;
-}
-
-export = Transpile;

package/bin/transpile.js

@@ -1,227 +0,0 @@
-// @ts-check
-
-const _opts = {};
-
-/**
- * @typedef {import('@travetto/manifest').Package} Pkg
- * @typedef {import('@travetto/manifest').ManifestContext} ManifestContext
- */
-
-function $imp(mod) {
-  try { return require(mod); } catch { return import(mod).then(x => x.default); }
-}
-
-/** @type {() => import('typescript')} */
-const $getTs = $imp.bind(null, 'typescript');
-/** @type {() => import('fs/promises')} */
-const $getFs = $imp.bind(null, 'fs/promises');
-/** @type {() => import('path')} */
-const $getPath = $imp.bind(null, 'path');
-/** @type {() => ({createRequire:(folder:string) => ({ resolve: (file:string)=>string})})} */
-const $getModule = $imp.bind(null, 'module');
-/** @param {string} x */
-const toPosix = x => x.replace(/\\/g, '/');
-
-/**
- * Returns the package.json
- * @param {string} inputFolder
- * @returns {Promise<Pkg>}
- */
-async function $getPkg(inputFolder) {
-  const fs = await $getFs();
-  const path = await $getPath();
-  if (!inputFolder.endsWith('.json')) {
-    inputFolder = path.resolve(inputFolder, 'package.json');
-  }
-  return JSON.parse(await fs.readFile(inputFolder, 'utf8'));
-}
-
-/**
- * Get workspace root
- * @return {Promise<string>}
- */
-async function $getWorkspaceRoot() {
-  const path = await $getPath();
-  const fs = await $getFs();
-  let folder = process.cwd();
-  let prevFolder = '';
-  while (folder !== prevFolder) {
-    try {
-      const pkg = await $getPkg(folder);
-      if (!!pkg.workspaces || !!pkg.travetto?.isolated) {
-        return folder;
-      }
-    } catch { }
-    if (await fs.stat(path.resolve(folder, '.git')).catch(() => { })) {
-      break;
-    }
-    prevFolder = folder;
-    folder = path.dirname(folder);
-  }
-  return process.cwd();
-}
-
-/**
- * Returns the compiler options
- * @param {ManifestContext} ctx
- * @returns
- */
-async function $getOpts(ctx) {
-  if (!(ctx.workspacePath in _opts)) {
-    const path = await $getPath();
-    const fs = await $getFs();
-    const ts = await $getTs();
-    const mod = await $getModule();
-    const req = mod.createRequire(`${ctx.workspacePath}/node_modules`);
-
-    const framework = req.resolve('@travetto/compiler/tsconfig.trv.json');
-    const self = path.resolve(ctx.workspacePath, 'tsconfig.json');
-    const loc = (await fs.stat(self).catch(() => false)) ? self : framework;
-    const { options } = ts.parseJsonSourceFileConfigFileContent(
-      ts.readJsonConfigFile(loc, ts.sys.readFile), ts.sys, ctx.workspacePath
-    );
-    options.inlineSourceMap = true;
-    options.sourceMap = false;
-    try {
-      const { type } = await $getPkg(ctx.workspacePath);
-      if (type) {
-        options.module = type.toLowerCase() === 'commonjs' ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext;
-      }
-    } catch { }
-
-    _opts[ctx.workspacePath] = options;
-  }
-  return _opts[ctx.workspacePath];
-}
-
-/**
- * Writes a package json file
- * @param {ManifestContext} ctx
- * @param {string} inputFile
- * @param {string} outputFile
- * @param {(pkg:Pkg) => Pkg} transform
- */
-async function writePackageJson(ctx, inputFile, outputFile, transform) {
-  const opts = await $getOpts(ctx);
-  const ts = await $getTs();
-  const isEsm = opts.module !== ts.ModuleKind.CommonJS;
-  let pkg = await $getPkg(inputFile);
-  pkg = transform?.(pkg) ?? pkg;
-  pkg.main = pkg.main?.replace(/[.]ts$/, '.js');
-  pkg.type = isEsm ? 'module' : 'commonjs';
-  pkg.files = pkg.files?.map(x => x.replace('.ts', '.js'));
-
-  ts.sys.writeFile(outputFile, JSON.stringify(pkg, null, 2));
-}
-
-/**
- * Transpiles a file
- * @param {ManifestContext} ctx
- * @param {string} inputFile
- * @param {string} outputFile
- */
-async function transpileFile(ctx, inputFile, outputFile) {
-  const ts = await $getTs();
-  const fs = await $getFs();
-
-  const opts = await $getOpts(ctx);
-  const content = ts.transpile(await fs.readFile(inputFile, 'utf8'), opts, inputFile)
-    .replace(/^((?:im|ex)port .*from '[.][^']+)(')/mg, (_, a, b) => `${a}.js${b}`)
-    .replace(/^(import [^\n]*from '[^.][^\n/]+[/][^\n/]+[/][^\n']+)(')/mg, (_, a, b) => `${a}.js${b}`);
-
-  ts.sys.writeFile(outputFile, content);
-}
-
-/**
- * Writes a js file
- * @param {ManifestContext} ctx
- * @param {string} inputFile
- * @param {string} outputFile
- */
-async function writeJsFile(ctx, inputFile, outputFile) {
-  const ts = await $getTs();
-  const fs = await $getFs();
-
-  const opts = await $getOpts(ctx);
-  const isEsm = opts.module !== ts.ModuleKind.CommonJS;
-
-  let content = await fs.readFile(inputFile, 'utf8');
-  if (isEsm) {
-    content = content
-      .replace(/^(?:async )?function [^$]/mg, a => `export ${a}`)
-      .replace(/^module.exports.*/mg, '');
-  }
-
-  await fs.writeFile(outputFile, content, 'utf8');
-}
-
-/**
- * Write an entire package
- * @param {ManifestContext} ctx
- * @param {string} name
- * @param {string} sourcePath
- * @param {string} mainSource
- * @param {string[]} extraSources
- */
-async function buildPackage(ctx, name, sourcePath, mainSource, extraSources) {
-  const path = await $getPath();
-  const fs = await $getFs();
-
-  const files = [mainSource, ...extraSources].map(x => ({ src: x, out: x.replace(/[.]ts$/, '.js') }));
-  const main = files[0].out;
-  const outputPath = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', name);
-
-  for (const { src, out } of files) {
-    const inputFile = path.resolve(sourcePath, src);
-    const outputFile = path.resolve(outputPath, out);
-
-    const [outStat, inStat] = await Promise.all([
-      fs.stat(outputFile).catch(() => undefined),
-      fs.stat(inputFile)
-    ]);
-
-    if (!outStat || (outStat.mtimeMs < inStat.mtimeMs)) {
-      await fs.mkdir(path.dirname(outputFile), { recursive: true });
-
-      if (inputFile.endsWith('.ts')) {
-        await transpileFile(ctx, inputFile, outputFile);
-      } else if (inputFile.endsWith('.js')) {
-        await writeJsFile(ctx, inputFile, outputFile);
-      } else if (inputFile.endsWith('.json')) {
-        await writePackageJson(ctx, inputFile, outputFile,
-          (pkg) => ({ ...pkg, files: files.map(x => x.out), name, main }));
-      }
-    }
-  }
-
-  return path.resolve(outputPath, main);
-}
-
-/**
- * Gets build context
- * @return {Promise<ManifestContext>}
- */
-async function getContext(folder = process.cwd()) {
-  const path = await $getPath();
-
-  const workspacePath = path.resolve(await $getWorkspaceRoot());
-  const mainPath = toPosix(folder);
-
-  const { name: mainModule, workspaces, travetto } = (await $getPkg(mainPath));
-  const monoRepo = workspacePath !== mainPath || !!workspaces;
-
-  // All relative to workspacePath
-  const manifestFile = `node_modules/${mainModule}/manifest.json`;
-
-  return {
-    mainModule,
-    mainPath,
-    workspacePath,
-    monoRepo,
-    manifestFile,
-    outputFolder: travetto?.outputFolder ?? '.trv_output',
-    compilerFolder: '.trv_compiler'
-  };
-}
-
-module.exports = { transpileFile, writePackageJson, writeJsFile, buildPackage, getContext };

package/support/bin/compiler-bootstrap.ts

@@ -1,151 +0,0 @@
-import path from 'path';
-import fs from 'fs/promises';
-import cp from 'child_process';
-
-import type { ManifestState, ManifestContext, ManifestRoot } from '@travetto/manifest';
-
-import { log, compileIfStale, getProjectSources, addNodePath, importManifest } from './utils';
-
-const PRECOMPILE_MODS = [
-  '@travetto/terminal',
-  '@travetto/manifest',
-  '@travetto/transformer',
-  '@travetto/compiler'
-];
-
-let manifestTemp;
-
-/**
- * Step 0
- */
-export async function precompile(ctx: ManifestContext): Promise<void> {
-  for (const mod of PRECOMPILE_MODS) {
-    await compileIfStale(ctx, `[0] Compiling ${mod}`, await getProjectSources(ctx, mod),);
-  }
-}
-
-export async function writeManifest(ctx: ManifestContext, manifest: ManifestRoot): Promise<void> {
-  const { ManifestUtil } = await importManifest(ctx);
-  return ManifestUtil.writeManifest(ctx, manifest);
-}
-
-async function rewriteManifests(ctx: ManifestContext, state: ManifestState): Promise<void> {
-  const { ManifestUtil } = await importManifest(ctx);
-
-  // Write out all changed manifests
-  const { getContext } = await import('../../bin/transpile');
-
-  const dirtyModules = [...Object.entries(state.delta)].filter(x => x[1].length > 0).map(([mod]) => mod);
-  for (const module of dirtyModules) {
-    const subCtx = await getContext(state.manifest.modules[module].source);
-    await ManifestUtil.createAndWriteManifest(subCtx);
-  }
-}
-
-/**
- *  Step 1
- */
-export async function buildManifest(ctx: ManifestContext): Promise<ManifestState> {
-  log('[1] Manifest Generation');
-  const { ManifestUtil } = await importManifest(ctx);
-  return ManifestUtil.produceState(ctx);
-}
-
-function shouldRebuildCompiler({ delta }: ManifestState): { total: boolean, transformers: [string, string][] } {
-  // Did enough things change to re-stage and build the compiler
-  const transformersChanged = Object.entries(delta)
-    .flatMap(([mod, files]) => files.map(x => [mod, x.file]))
-    .filter((ev): ev is [string, string] => ev[1].startsWith('support/transform'));
-  const transformerChanged = (delta['@travetto/transformer'] ?? []);
-  const compilerChanged = delta['@travetto/compiler'] ?? [];
-
-  const changed = transformerChanged.length || transformersChanged.length || compilerChanged.length;
-  if (changed) {
-    if (compilerChanged.length) {
-      log('[2] Compiler source changed @travetto/compiler', compilerChanged);
-    }
-    if (transformerChanged.length) {
-      log('[2] Compiler source changed @travetto/transformer', transformerChanged);
-    }
-    if (transformersChanged.length) {
-      log('[2] Compiler source changed */support/transform', transformersChanged);
-    }
-  }
-  return { total: changed > 0, transformers: transformersChanged };
-}
-
-/**
- *  Step 2
- */
-async function buildCompiler(state: ManifestState, ctx: ManifestContext): Promise<void> {
-  const changed = shouldRebuildCompiler(state);
-
-  if (changed.transformers.length) {
-    state = await buildManifest(ctx);
-    let x = 0;
-    for (const [mod, file] of changed.transformers) {
-      await compileIfStale(
-        ctx,
-        `[2.${x += 1}] ${file} Bootstrapping`,
-        await getProjectSources(ctx, mod, ['package.json', file])
-      );
-    }
-  }
-
-  log('[2] Compiler Ready');
-}
-
-/**
- *  Step 4
- */
-async function compileOutput(state: ManifestState, ctx: ManifestContext, watch?: boolean): Promise<void> {
-  let changes = Object.values(state.delta).flat();
-
-  // Remove files that should go away
-  await Promise.all(changes.filter(x => x.type === 'removed')
-    .map(x => fs.unlink(path.resolve(ctx.workspacePath, ctx.outputFolder, x.file)).catch(() => { })));
-
-  changes = changes.filter(x => x.type !== 'removed');
-
-  const { ManifestUtil } = await importManifest(ctx);
-  const resolve = ManifestUtil.resolveFile.bind(null, ctx, state.manifest, '@travetto/compiler');
-
-  manifestTemp ??= await ManifestUtil.writeState(state);
-  const cwd = path.resolve(ctx.workspacePath, ctx.compilerFolder);
-
-  if (changes.length) {
-    log('[3] Changed Sources', changes);
-
-    // Blocking call, compile only
-    const res = cp.spawnSync(process.argv0,
-      [resolve('support/main.output'), manifestTemp],
-      { cwd, stdio: 'inherit', encoding: 'utf8' }
-    );
-
-    if (res.status) {
-      throw new Error(res.stderr);
-    }
-
-    await rewriteManifests(ctx, state);
-  }
-
-  if (watch) {
-    // Rewrite state with updated manifest
-    const newState = await ManifestUtil.produceState(ctx);
-    manifestTemp = await ManifestUtil.writeState(newState);
-
-    // Run with watching
-    cp.spawnSync(process.argv0,
-      [resolve('support/main.output'), manifestTemp, 'true'], { cwd, stdio: 'inherit' }
-    );
-  }
-}
-
-export async function compile(ctx: ManifestContext, watch?: boolean): Promise<ManifestState> {
-  await precompile(ctx); // Step 0
-  const state = await buildManifest(ctx); // Step 1
-  await buildCompiler(state, ctx); // Step 2
-  await compileOutput(state, ctx, watch); // Step 3
-  await addNodePath(path.resolve(ctx.workspacePath, ctx.outputFolder));
-  return state;
-}