@travetto/compiler 3.0.0-rc.11 → 3.0.0-rc.12

package/src/util.ts

@@ -1,10 +1,8 @@
 import ts from 'typescript';
-import fs from 'fs/promises';
-import { readdirSync } from 'fs';
 
-import { ManifestModule, ManifestRoot, Package, PackageUtil, path } from '@travetto/manifest';
+import { ManifestRoot, Package, path } from '@travetto/manifest';
 
-type InputToSource = (inputFile: string) => ({ source: string, module: ManifestModule } | undefined);
+type InputToSource = (inputFile: string) => ({ source: string } | undefined);
 export type FileWatchEvent = { type: 'create' | 'delete' | 'update', path: string };
 
 const nativeCwd = process.cwd();
@@ -14,6 +12,15 @@ const nativeCwd = process.cwd();
  */
 export class CompilerUtil {
 
+  /**
+   * Map input file to output format, generally converting ts extensions to js
+   * @param file
+   * @returns
+   */
+  static inputToOutput(file: string): string {
+    return file.replace(/[.][tj]s$/, '.js');
+  }
+
   /**
    * Determines if write callback data has sourcemap information
    * @param data
@@ -27,14 +34,14 @@ export class CompilerUtil {
    * Rewrite's sourcemap locations to real folders
    * @returns
    */
-  static rewriteSourceMap(text: string, inputToSource: InputToSource): string {
+  static rewriteSourceMap(root: string, text: string, inputToSource: InputToSource): string {
     const data: { sourceRoot: string, sources: string[] } = JSON.parse(text);
-    const [src] = data.sources;
+    const src = path.resolve(data.sourceRoot, data.sources[0]);
 
-    const { source: file, module } = inputToSource(src) ?? {};
-    if (file && module) {
-      data.sourceRoot = module.source;
-      data.sources = [file.replace(`${module.source}/`, '')];
+    const { source: file } = inputToSource(src) ?? {};
+    if (file) {
+      data.sourceRoot = root;
+      data.sources = [file];
       text = JSON.stringify(data);
     }
 
@@ -49,13 +56,14 @@ export class CompilerUtil {
    * @returns
    */
   static rewriteInlineSourceMap(
+    root: string,
     text: string,
     inputToSource: InputToSource,
     { sourceMapUrlPos }: ts.WriteFileCallbackData & { sourceMapUrlPos: number }
   ): string {
     const sourceMapUrl = text.substring(sourceMapUrlPos);
     const [prefix, sourceMapData] = sourceMapUrl.split('base64,');
-    const rewritten = this.rewriteSourceMap(Buffer.from(sourceMapData, 'base64url').toString('utf8'), inputToSource);
+    const rewritten = this.rewriteSourceMap(root, Buffer.from(sourceMapData, 'base64url').toString('utf8'), inputToSource);
     return [
       text.substring(0, sourceMapUrlPos),
       prefix,
@@ -74,10 +82,10 @@ export class CompilerUtil {
   static rewritePackageJSON(manifest: ManifestRoot, text: string, opts: ts.CompilerOptions): string {
     const pkg: Package = JSON.parse(text);
     if (pkg.files) {
-      pkg.files = pkg.files.map(x => x.replace(/[.]ts$/, '.js'));
+      pkg.files = pkg.files.map(x => this.inputToOutput(x));
     }
     if (pkg.main) {
-      pkg.main = pkg.main.replace(/[.]ts$/, '.js');
+      pkg.main = this.inputToOutput(pkg.main);
     }
     pkg.type = opts.module !== ts.ModuleKind.CommonJS ? 'module' : 'commonjs';
     for (const key of ['devDependencies', 'dependencies', 'peerDependencies'] as const) {
@@ -92,20 +100,6 @@ export class CompilerUtil {
     return JSON.stringify(pkg, null, 2);
   }
 
-  /**
-   * Read the given tsconfig.json values for the project
-   * @param path
-   * @returns
-   */
-  static async readTsConfigOptions(file: string): Promise<ts.CompilerOptions> {
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    const { options } = ts.parseJsonSourceFileConfigFileContent(
-      ts.readJsonConfigFile(file, ts.sys.readFile), ts.sys, nativeCwd
-    );
-    options.target = ts.ScriptTarget.ESNext;
-    return options;
-  }
-
   /**
    * Build transpilation error
    * @param filename The name of the file
@@ -133,59 +127,6 @@ export class CompilerUtil {
     return new Error(`Transpiling ${filename.replace(nativeCwd, '.')} failed: \n${errors.join('\n')}`);
   }
 
-  /**
-   * Allows for watching of explicit folders
-   *
-   * @param onEvent
-   * @returns
-   */
-  static async fileWatcher(
-    folders: string[],
-    onEvent: (ev: FileWatchEvent, folder: string) => void
-  ): Promise<() => Promise<void>> {
-    const watcher = await import('@parcel/watcher');
-    const subs: (() => Promise<void>)[] = [];
-    for (const folder of folders) {
-      const sub = await watcher.subscribe(folder, (err, events) => {
-        for (const ev of events) {
-          onEvent(ev, folder);
-        }
-      }, { ignore: ['node_modules', ...readdirSync(folder).filter(x => x.startsWith('.') && x.length > 2)] });
-      subs.push(() => sub.unsubscribe());
-    }
-    const readiedSubs = await Promise.all(subs);
-    return () => Promise.all(readiedSubs.map(s => s())).then(() => { });
-  }
-
-  /**
-   * Get loaded compiler options
-   */
-  static async getCompilerOptions(outputFolder: string, bootTsConfig: string, workspace: string): Promise<ts.CompilerOptions> {
-    const opts: Partial<ts.CompilerOptions> = {};
-    const rootDir = nativeCwd;
-    const projTsconfig = path.resolve('tsconfig.json');
-    // Fallback to base tsconfig if not found in local folder
-    const config = (await fs.stat(projTsconfig).catch(() => false)) ? projTsconfig : bootTsConfig;
-    const base = await this.readTsConfigOptions(config);
-
-    const { type } = PackageUtil.readPackage(workspace);
-
-    if (type !== undefined) {
-      base.module = `${type}`.toLowerCase() === 'commonjs' ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext;
-    }
-
-    return {
-      ...base,
-      resolveJsonModule: true,
-      allowJs: true,
-      outDir: outputFolder,
-      sourceRoot: rootDir,
-      ...opts,
-      rootDir,
-    };
-  }
-
-
   /**
    * Naive hashing
    */

package/support/compiler-entry.ts

@@ -0,0 +1,2 @@
+import { Compiler } from '../src/compiler';
+Compiler.main();

package/support/launcher.ts

@@ -0,0 +1,160 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { Module } from 'module';
+
+import type { ManifestContext } from '@travetto/manifest';
+import { TranspileUtil } from './transpile';
+
+const SOURCE_SEED = ['package.json', 'index.ts', '__index__.ts', 'src', 'support', 'bin'];
+const PRECOMPILE_MODS = ['@travetto/terminal', '@travetto/manifest', '@travetto/transformer', '@travetto/compiler'];
+
+const importManifest = (ctx: ManifestContext): Promise<typeof import('@travetto/manifest')> =>
+  import(path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/manifest/__index__.js'));
+
+/**
+ * Recompile folder if stale
+ */
+async function compileIfStale(ctx: ManifestContext, scope: string, mod: string, seed: string[]): Promise<string[]> {
+  const files = await TranspileUtil.getModuleSources(ctx, mod, seed);
+  const changes = files.filter(x => x.stale).map(x => x.input);
+  const out: string[] = [];
+
+  try {
+    await TranspileUtil.withLogger(scope, { args: [mod], basic: false }, async log => {
+      if (files.some(f => f.stale)) {
+        log('debug', 'Starting');
+        for (const file of files.filter(x => x.stale)) {
+          await TranspileUtil.transpileFile(ctx, file.input, file.output);
+        }
+        if (changes.length) {
+          out.push(...changes.map(x => `${mod}/${x}`));
+          log('debug', `Source changed: ${changes.join(', ')}`);
+        }
+        log('debug', 'Completed');
+      } else {
+        log('debug', 'Skipped');
+      }
+    });
+  } catch (err) {
+    console.error(err);
+  }
+  return out;
+}
+
+/**
+ * Run the compiler
+ */
+export async function compile(ctx: ManifestContext, op?: 'watch' | 'build'): Promise<void> {
+  let changes = 0;
+
+  await TranspileUtil.withLogger('precompile', async () => {
+    for (const mod of PRECOMPILE_MODS) {
+      changes += (await compileIfStale(ctx, 'precompile', mod, SOURCE_SEED)).length;
+    }
+  });
+
+  const { ManifestUtil, ManifestDeltaUtil } = await importManifest(ctx);
+
+  const manifest = await TranspileUtil.withLogger('manifest', async () => ManifestUtil.buildManifest(ctx));
+
+  await TranspileUtil.withLogger('transformers', async () => {
+    for (const mod of Object.values(manifest.modules).filter(m => m.files.$transformer?.length)) {
+      changes += (await compileIfStale(ctx, 'transformers', mod.name, ['package.json', ...mod.files.$transformer!.map(x => x[0])])).length;
+    }
+  });
+
+  const delta = await TranspileUtil.withLogger('delta', async log => {
+    if (changes) {
+      log('debug', 'Skipping, everything changed');
+      return [{ type: 'changed', file: '*', module: ctx.mainModule } as const];
+    } else {
+      return ManifestDeltaUtil.produceDelta(ctx, manifest);
+    }
+  });
+
+  if (changes) {
+    await fs.rm(path.resolve(ctx.workspacePath, ctx.outputFolder), { recursive: true, force: true });
+    TranspileUtil.log('reset', [], 'info', 'Clearing output due to compiler changes');
+  }
+
+  // Write manifest
+  await TranspileUtil.withLogger('manifest', async log => {
+    await ManifestUtil.writeManifest(ctx, manifest);
+    log('debug', `Wrote manifest ${ctx.mainModule}`);
+
+    // Update all manifests
+    if (delta.length && ctx.monoRepo && !ctx.mainFolder) {
+      const names: string[] = [];
+      const mods = Object.values(manifest.modules).filter(x => x.local && x.name !== ctx.mainModule);
+      for (const mod of mods) {
+        await ManifestUtil.rewriteManifest(path.resolve(ctx.workspacePath, mod.sourceFolder));
+        names.push(mod.name);
+      }
+      log('debug', `Changes triggered ${delta.map(x => `${x.type}:${x.module}:${x.file}`)}`);
+      log('debug', `Rewrote monorepo manifests [changes=${delta.length}] ${names.join(', ')}`);
+    }
+  });
+
+  await TranspileUtil.withLogger('compile', { args: [], basic: false }, async log => {
+    const changed = delta.filter(x => x.type === 'added' || x.type === 'changed');
+    log('debug', `Started action=${op} changed=${changed.map(x => `${x.module}/${x.file}`)}`);
+    if (changed.length || op === 'watch') {
+      await TranspileUtil.runCompiler(ctx, manifest, changed, op === 'watch');
+      log('debug', 'Finished');
+    } else {
+      log('debug', 'Skipped');
+    }
+  });
+
+  if (op === 'build') {
+    TranspileUtil.log('build', [], 'info', 'Successfully built');
+  }
+}
+
+/**
+ * Export manifests
+ */
+export async function exportManifest(ctx: ManifestContext, output?: string, env = 'dev'): Promise<string | undefined> {
+  const { ManifestUtil } = await importManifest(ctx);
+  const manifest = await ManifestUtil.buildManifest(ctx);
+
+  // If in prod mode, only include std modules
+  if (/^prod/i.test(env)) {
+    manifest.modules = Object.fromEntries(
+      Object.values(manifest.modules)
+        .filter(x => x.profiles.includes('std'))
+        .map(m => [m.name, m])
+    );
+    // Mark output folder/workspace path as portable
+    manifest.outputFolder = '';
+    manifest.workspacePath = '';
+  }
+  if (output) {
+    if (!output.endsWith('.json')) {
+      output = path.resolve(output, 'manifest.json');
+    }
+
+    await TranspileUtil.writeTextFile(output, JSON.stringify(manifest));
+    TranspileUtil.log('manifest', [], 'info', `Wrote manifest ${output}`);
+    return output;
+  } else {
+    console.log(JSON.stringify(manifest, null, 2));
+    return;
+  }
+}
+
+export async function launchMain(ctx: ManifestContext): Promise<void> {
+  // Rewriting node_path
+  const nodeOut = path.resolve(ctx.workspacePath, ctx.outputFolder, 'node_modules');
+  const og = process.env.NODE_PATH;
+  process.env.NODE_PATH = [nodeOut, og].join(path.delimiter);
+  // @ts-expect-error
+  Module._initPaths();
+  process.env.NODE_PATH = og; // Restore
+
+  process.env.TRV_MANIFEST = path.resolve(nodeOut, ctx.mainModule);
+
+  // TODO: Externalize somehow?
+  const cliMain = path.join(nodeOut, '@travetto/cli/support/cli.js');
+  return await import(cliMain);
+}

package/support/transpile.ts

@@ -0,0 +1,185 @@
+import path from 'path';
+import fs from 'fs/promises';
+import os from 'os';
+import cp from 'child_process';
+import { createRequire } from 'module';
+
+import { DeltaEvent, ManifestContext, ManifestRoot, Package } from '@travetto/manifest';
+
+type ModFile = { input: string, output: string, stale: boolean };
+
+const SRC_REQ = createRequire(path.resolve('node_modules'));
+const recentStat = (stat: { ctimeMs: number, mtimeMs: number }): number => Math.max(stat.ctimeMs, stat.mtimeMs);
+
+export type CompilerLogEvent = [level: 'info' | 'debug' | 'warn', message: string];
+type WithLogger<T> = (log: (...ev: CompilerLogEvent) => void) => Promise<T>;
+type LogConfig = { args?: string[], basic?: boolean };
+const isCompilerLogEvent = (o: unknown): o is CompilerLogEvent => o !== null && o !== undefined && Array.isArray(o);
+
+const LEVELS = { warn: true, debug: /^debug$/.test(process.env.TRV_BUILD ?? ''), info: !/^warn$/.test(process.env.TRV_BUILD ?? '') };
+const SCOPE_MAX = 15;
+
+/**
+ * Transpile utilities for launching
+ */
+export class TranspileUtil {
+  static #optCache: Record<string, {}> = {};
+
+  static log(scope: string, args: string[], ...[level, msg]: CompilerLogEvent): void {
+    const message = msg.replaceAll(process.cwd(), '.');
+    LEVELS[level] && console.debug(new Date().toISOString(), `[${scope.padEnd(SCOPE_MAX, ' ')}]`, ...args, message);
+  }
+
+  /**
+   * With logger
+   */
+  static withLogger<T>(scope: string, op: WithLogger<T>): Promise<T>;
+  static withLogger<T>(scope: string, opts: LogConfig, op: WithLogger<T>): Promise<T>;
+  static withLogger<T>(scope: string, opts: LogConfig | WithLogger<T>, op?: WithLogger<T>): Promise<T> {
+    const cfg = { basic: true, ...typeof opts === 'function' ? {} : opts };
+    const go = typeof opts === 'function' ? opts : op!;
+    const log = this.log.bind(null, scope, cfg.args ?? []);
+    cfg.basic && log('debug', 'Started');
+    return go(log).finally(() => cfg.basic && log('debug', 'Completed'));
+  }
+
+  /**
+   * Write text file, and ensure folder exists
+   */
+  static writeTextFile = (file: string, content: string): Promise<void> =>
+    fs.mkdir(path.dirname(file), { recursive: true }).then(() => fs.writeFile(file, content));
+
+  /**
+   * Returns the compiler options
+   */
+  static async getCompilerOptions(ctx: ManifestContext): Promise<{}> {
+    if (!(ctx.workspacePath in this.#optCache)) {
+      let tsconfig = path.resolve(ctx.workspacePath, 'tsconfig.json');
+
+      if (!await fs.stat(tsconfig).then(_ => true, _ => false)) {
+        tsconfig = SRC_REQ.resolve('@travetto/compiler/tsconfig.trv.json');
+      }
+
+      const ts = (await import('typescript')).default;
+
+      const { options } = ts.parseJsonSourceFileConfigFileContent(
+        ts.readJsonConfigFile(tsconfig, ts.sys.readFile), ts.sys, ctx.workspacePath
+      );
+
+      options.allowJs = true;
+      options.resolveJsonModule = true;
+      options.sourceRoot = ctx.workspacePath;
+      options.rootDir = ctx.workspacePath;
+      options.outDir = path.resolve(ctx.workspacePath);
+
+      try {
+        options.module = ctx.moduleType === 'commonjs' ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext;
+      } catch { }
+
+      this.#optCache[ctx.workspacePath] = options;
+    }
+    return this.#optCache[ctx.workspacePath];
+  }
+
+  /**
+   * Output a file, support for ts, js, and package.json
+   */
+  static async transpileFile(ctx: ManifestContext, inputFile: string, outputFile: string): Promise<void> {
+    if (inputFile.endsWith('.ts') || inputFile.endsWith('.js')) {
+      const compilerOut = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules');
+
+      const text = (await fs.readFile(inputFile, 'utf8'))
+        .replace(/from '([.][^']+)'/g, (_, i) => `from '${i.replace(/[.]js$/, '')}.js'`)
+        .replace(/from '(@travetto\/(.*?))'/g, (_, i, s) => `from '${path.resolve(compilerOut, `${i}${s.includes('/') ? '.js' : '/__index__.js'}`)}'`);
+
+      const ts = (await import('typescript')).default;
+      const content = ts.transpile(text, await this.getCompilerOptions(ctx), inputFile);
+      await this.writeTextFile(outputFile, content);
+    } else if (inputFile.endsWith('package.json')) {
+      const pkg: Package = JSON.parse(await fs.readFile(inputFile, 'utf8'));
+      const main = pkg.main?.replace(/[.]ts$/, '.js');
+      const files = pkg.files?.map(x => x.replace('.ts', '.js'));
+
+      const content = JSON.stringify({ ...pkg, main, type: ctx.moduleType, files }, null, 2);
+      await this.writeTextFile(outputFile, content);
+    }
+  }
+
+  /**
+   * Scan directory to find all project sources for comparison
+   */
+  static async getModuleSources(ctx: ManifestContext, module: string, seed: string[]): Promise<ModFile[]> {
+    const inputFolder = (ctx.mainModule === module) ?
+      process.cwd() :
+      path.dirname(SRC_REQ.resolve(`${module}/package.json`));
+
+    const folders = seed.filter(x => !/[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+    const files = seed.filter(x => /[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+
+    while (folders.length) {
+      const sub = folders.pop();
+      if (!sub) {
+        continue;
+      }
+
+      for (const file of await fs.readdir(sub).catch(() => [])) {
+        if (file.startsWith('.')) {
+          continue;
+        }
+        const resolvedInput = path.resolve(sub, file);
+        const stat = await fs.stat(resolvedInput);
+
+        if (stat.isDirectory()) {
+          folders.push(resolvedInput);
+        } else if (file.endsWith('.d.ts')) {
+          // Do nothing
+        } else if (file.endsWith('.ts') || file.endsWith('.js')) {
+          files.push(resolvedInput);
+        }
+      }
+    }
+
+    const outputFolder = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', module);
+    const out: ModFile[] = [];
+    for (const input of files) {
+      const output = input.replace(inputFolder, outputFolder).replace(/[.]ts$/, '.js');
+      const inputTs = await fs.stat(input).then(recentStat, () => 0);
+      if (inputTs) {
+        const outputTs = await fs.stat(output).then(recentStat, () => 0);
+        await fs.mkdir(path.dirname(output), { recursive: true, });
+        out.push({ input, output, stale: inputTs > outputTs });
+      }
+    }
+
+    return out;
+  }
+
+  /**
+   * Run compiler
+   */
+  static async runCompiler(ctx: ManifestContext, manifest: ManifestRoot, changed: DeltaEvent[], watch = false): Promise<void> {
+    const compiler = path.resolve(ctx.workspacePath, ctx.compilerFolder);
+    const main = path.resolve(compiler, 'node_modules', '@travetto/compiler/support/compiler-entry.js');
+    const deltaFile = path.resolve(os.tmpdir(), `manifest-delta.${Date.now()}.${Math.random()}.json`);
+
+    const changedFiles = changed[0]?.file === '*' ? ['*'] : changed.map(ev =>
+      path.resolve(manifest.workspacePath, manifest.modules[ev.module].sourceFolder, ev.file)
+    );
+
+    await this.writeTextFile(deltaFile, changedFiles.join('\n'));
+
+    await this.withLogger('compiler-exec', async log => {
+      await new Promise((res, rej) =>
+        cp.spawn(process.argv0, [main, deltaFile, `${watch}`], {
+          env: {
+            ...process.env,
+            TRV_MANIFEST: path.resolve(ctx.workspacePath, ctx.outputFolder, 'node_modules', ctx.mainModule),
+          },
+          stdio: [0, 1, 2, 'ipc'],
+        })
+          .on('message', msg => isCompilerLogEvent(msg) && log(...msg))
+          .on('exit', code => (code !== null && code > 0) ? rej() : res(null))
+      );
+    });
+  }
+}

package/tsconfig.trv.json

@@ -17,6 +17,7 @@
     "inlineSourceMap": false,
     "removeComments": true,
     "importHelpers": true,
+    "noEmitHelpers": true,
     "outDir": "build/"
   },
 }

package/bin/transpile.d.ts

@@ -1,32 +0,0 @@
-import type { ManifestContext, Package } from '@travetto/manifest';
-
-declare namespace Transpile {
-  type CompileCommand = 'build' | 'watch' | 'clean' | 'manifest';
-
-  /**
-   * Writes a package json file
-   */
-  function writePackageJson(ctx: ManifestContext, inputFile: string, outputFile: string, transform?: (pkg: Package) => Package): Promise<void>;
-
-  /**
-   * Transpiles a file
-   */
-  function transpileFile(ctx: ManifestContext, inputFile: string, outputFile: string): Promise<void>;
-
-  /**
-   * Write js file
-   */
-  function writeJsFile(ctx: ManifestContext, inputFile: string, outputFile: string): Promise<void>;
-
-  /**
-   * Build an entire package
-   */
-  function buildPackage(ctx: ManifestContext, name: string, sourcePath: string, mainSource: string, extraSource: string[]): Promise<string>;
-
-  /**
-   * Get Context for building
-   */
-  function getContext(folder?: string): Promise<ManifestContext>;
-}
-
-export = Transpile;