@travetto/compiler 3.0.0-rc.2 → 3.0.0-rc.20

package/README.md

@@ -1,7 +1,7 @@
 <!-- This file was generated by @travetto/doc and should not be modified directly -->
-<!-- Please modify https://github.com/travetto/travetto/tree/main/module/compiler/doc.ts and execute "npx trv doc" to rebuild -->
+<!-- Please modify https://github.com/travetto/travetto/tree/main/module/compiler/DOC.ts and execute "npx trv doc" to rebuild -->
 # Compiler
-## Node-integration of Typescript Compiler with advanced functionality for detecting changes in classes and methods.
+## Compiler
 
 **Install: @travetto/compiler**
 ```bash
@@ -17,9 +17,12 @@ This module expands upon [Typescript](https://typescriptlang.org), with suppleme
    *  Support for detecting changes in sources files at runtime
    *  Allows for hot-reloading of classes during development    
       *  Utilizes `es2015` `Proxy`s to allow for swapping out implementation at runtime
-
 Additionally, there is support for common AST transformations via [Transformation](https://github.com/travetto/travetto/tree/main/module/transformer#readme "Functionality for AST transformations, with transformer registration, and general utils")
-
 ## Debugging
-
 When dealing with transformers, logging is somewhat tricky as the compiler executes before the code is loaded.  To that end, the file `compiler.log` is created in the cache directory during the compilation process. This is a location that transformers should be free to log to, for debugging, and any additional feedback.
+
+## CLI
+
+The module provides the ability to clear the compilation cache to handle any inconsistencies that may arise.
+
+TODO: Describe cli behavior

package/__index__.ts

@@ -0,0 +1,6 @@
+export * from './src/compiler';
+export * from './src/state';
+export * from './src/util';
+export * from './src/watch';
+export * from './src/log';
+export * from './src/types';

package/bin/trv.js

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+
+// @ts-check
+import fs from 'fs/promises';
+import path from 'path';
+import { createRequire } from 'module';
+
+import { getManifestContext } from '@travetto/manifest/bin/context.js';
+
+const VALID_OPS = { watch: 'watch', build: 'build', clean: 'clean', manifest: 'manifest' };
+
+const COMPILER_FILES = [...['launcher', 'transpile', 'lock', 'log', 'lock-pinger'].map(x => `support/${x}.js`), 'package.json'];
+
+/**
+ * @param {import('@travetto/manifest').ManifestContext} ctx
+ * @return {Promise<import('@travetto/compiler/support/launcher').launch>}
+ */
+const $getLauncher = async (ctx) => {
+  const compPkg = createRequire(path.resolve('node_modules')).resolve('@travetto/compiler/package.json');
+  const files = [];
+
+  for (const file of COMPILER_FILES) {
+    const target = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/compiler', file);
+    const src = compPkg.replace('package.json', file.replace(/[.]js$/, '.ts'));
+
+    const targetTime = await fs.stat(target).then(s => Math.max(s.mtimeMs, s.ctimeMs)).catch(() => 0);
+    const srcTime = await fs.stat(src).then(s => Math.max(s.mtimeMs, s.ctimeMs));
+    // If stale
+    if (srcTime > targetTime) {
+      const ts = (await import('typescript')).default;
+      const module = ctx.moduleType === 'module' ? ts.ModuleKind.ESNext : ts.ModuleKind.CommonJS;
+      await fs.mkdir(path.dirname(target), { recursive: true });
+      const text = await fs.readFile(src, 'utf8');
+      if (file.endsWith('.js')) {
+        const content = ts.transpile(
+          text,
+          { target: ts.ScriptTarget.ES2020, module, esModuleInterop: true, allowSyntheticDefaultImports: true }
+        )
+          .replace(/^((?:im|ex)port .*from '[.][^']+)(')/mg, (_, a, b) => `${a}.js${b}`)
+          .replace(/^(import [^\n]*from '[^.][^\n/]+[/][^\n/]+[/][^\n']+)(')/mg, (_, a, b) => `${a}.js${b}`);
+        await fs.writeFile(target, content, 'utf8');
+      } else {
+        const pkg = JSON.parse(text);
+        pkg.type = ctx.moduleType;
+        await fs.writeFile(target, JSON.stringify(pkg, null, 2), 'utf8');
+      }
+      // Compile
+    }
+    files.push(target);
+  }
+
+  try { return require(files[0]).launch; }
+  catch { return import(files[0]).then(x => x.launch); }
+};
+
+(async () => {
+  const ctx = await getManifestContext(process.env.TRV_COMPILER_CWD);
+  const [op, args] = [VALID_OPS[process.argv[2]], process.argv.slice(3)];
+
+  if (op === 'clean') {
+    const folders = process.argv.find(x => x === '--all' || x === '-a') ? [ctx.outputFolder, ctx.compilerFolder] : [ctx.outputFolder];
+    for (const f of folders) {
+      await fs.rm(path.resolve(ctx.workspacePath, f), { force: true, recursive: true });
+    }
+    return console.log(`Cleaned ${ctx.workspacePath}: [${folders.join(', ')}]`);
+  }
+
+  const rootCtx = ctx.monoRepo ? await getManifestContext(ctx.workspacePath) : ctx;
+
+  return (await $getLauncher(ctx))(ctx, rootCtx, op, args);
+})();

package/package.json

@@ -1,16 +1,11 @@
 {
   "name": "@travetto/compiler",
-  "displayName": "Compiler",
-  "version": "3.0.0-rc.2",
-  "description": "Node-integration of Typescript Compiler with advanced functionality for detecting changes in classes and methods.",
+  "version": "3.0.0-rc.20",
+  "description": "Compiler",
   "keywords": [
-    "tsc",
-    "typescript",
     "compiler",
-    "caching",
-    "change-detection",
-    "real-time",
-    "travetto"
+    "travetto",
+    "typescript"
   ],
   "homepage": "https://travetto.io",
   "license": "MIT",
@@ -18,21 +13,43 @@
     "email": "travetto.framework@gmail.com",
     "name": "Travetto Framework"
   },
+  "type": "module",
   "files": [
-    "index.ts",
+    "__index__.ts",
     "src",
-    "support"
+    "bin",
+    "support",
+    "tsconfig.trv.json"
   ],
-  "main": "index.ts",
+  "main": "__index__.ts",
+  "bin": {
+    "trv": "bin/trv.js"
+  },
   "repository": {
     "url": "https://github.com/travetto/travetto.git",
     "directory": "module/compiler"
   },
   "dependencies": {
-    "@travetto/base": "^3.0.0-rc.0",
-    "@travetto/transformer": "^3.0.0-rc.2",
-    "@travetto/watch": "^3.0.0-rc.0"
+    "@parcel/watcher": "^2.1.0",
+    "@travetto/manifest": "^3.0.0-rc.13",
+    "@travetto/terminal": "^3.0.0-rc.7",
+    "@travetto/transformer": "^3.0.0-rc.16"
+  },
+  "peerDependencies": {
+    "@travetto/cli": "^3.0.0-rc.15"
+  },
+  "peerDependenciesMeta": {
+    "@travetto/cli": {
+      "optional": true
+    }
+  },
+  "travetto": {
+    "displayName": "Compiler",
+    "profiles": [
+      "compile"
+    ]
   },
+  "private": false,
   "publishConfig": {
     "access": "public"
   }

package/src/compiler.ts

@@ -1,187 +1,146 @@
-import { EventEmitter } from 'events';
-import * as sourceMapSupport from 'source-map-support';
-import * as ts from 'typescript';
+import { install } from 'source-map-support';
+import timers from 'timers/promises';
+import ts from 'typescript';
+import fs from 'fs/promises';
 
-import { PathUtil, EnvUtil, AppCache } from '@travetto/boot';
-import { SourceIndex } from '@travetto/boot/src/internal/source';
-import { ModuleManager } from '@travetto/boot/src/internal/module';
-import { Dynamic } from '@travetto/base/src/internal/dynamic';
-import { TranspileUtil } from '@travetto/boot/src/internal/transpile-util';
+import { GlobalTerminal, TerminalProgressEvent } from '@travetto/terminal';
+import { RootIndex } from '@travetto/manifest';
 
-import { SourceHost } from './host';
-import { TransformerManager } from './transformer';
+import { CompilerUtil } from './util';
+import { CompilerState } from './state';
+import { CompilerWatcher } from './watch';
+import { Log } from './log';
+import { CompileEmitError, CompileEmitEvent, CompileEmitter } from './types';
 
-type FileListener = (name: string) => void;
-type EventType = 'added' | 'removed' | 'changed';
+const PING_THRESHOLD = 1000;
 
 /**
- * Compilation orchestrator, interfaces with watching, unloading, emitting and delegates appropriately
+ * Compilation support
  */
-@Dynamic('@travetto/compiler/support/dynamic.compiler')
-class $Compiler {
-
-  #program: ts.Program | undefined;
-  #transformerManager = new TransformerManager();
-  #emitter = new EventEmitter();
-  #host = new SourceHost();
-
-  active = false;
+export class Compiler {
 
   /**
-   * Build typescript program
-   *
-   * @param forFile If this file is new, force a recompilation
+   * Run compiler as a main entry point
    */
-  #getProgram(forFile?: string): ts.Program {
+  static async main(): Promise<void> {
+    const [dirty, watch] = process.argv.slice(2);
+    install();
+    const dirtyFiles = (await fs.readFile(dirty, 'utf8')).split(/\n/).filter(x => !!x);
+    return new Compiler().init(dirtyFiles).then(c => c.run(watch === 'true'));
+  }
+
+  #state: CompilerState;
+  #dirtyFiles: string[];
 
-    const rootFiles = this.#host.getRootFiles();
+  async init(dirtyFiles: string[]): Promise<this> {
+    this.#state = await CompilerState.get(RootIndex);
+    this.#dirtyFiles = dirtyFiles[0] === '*' ?
+      this.#state.getAllFiles() :
+      dirtyFiles.map(f => this.#state.getBySource(f)!.input);
+
+    return this;
+  }
 
-    if (!this.#program || (forFile && !rootFiles.has(forFile))) {
-      console.debug('Loading program', { size: rootFiles.size, src: forFile });
-      if (forFile) {
-        rootFiles.add(forFile);
+  /**
+   * Watches local modules
+   */
+  #watchLocalModules(emit: CompileEmitter): Promise<() => Promise<void>> {
+    return new CompilerWatcher(this.#state).watchFiles(async file => {
+      const err = await emit(file, true);
+      if (err) {
+        Log.info('Compilation Error', CompilerUtil.buildTranspileError(file, err));
+      } else {
+        Log.info(`Compiled ${file.split('node_modules/')[1]}`);
       }
-      this.#program = ts.createProgram({
-        rootNames: [...rootFiles],
-        options: TranspileUtil.compilerOptions,
-        host: this.#host,
-        oldProgram: this.#program
-      });
-      this.#transformerManager.build(this.#program.getTypeChecker());
-    }
-    return this.#program;
+      return err;
+    });
   }
 
   /**
-   * Perform actual transpilation
+   * Compile in a single pass, only emitting dirty files
    */
-  #transpile(filename: string, force = false): string {
-    if (force || !AppCache.hasEntry(filename)) {
-      console.debug('Emitting', { filename: filename.replace(PathUtil.cwd, '.') });
+  async getCompiler(): Promise<CompileEmitter> {
+    let program: ts.Program;
 
+    const emit = async (inputFile: string, needsNewProgram = program === undefined): Promise<CompileEmitError | undefined> => {
       try {
-        const prog = this.#getProgram(filename);
-
-        const result = prog.emit(
-          prog.getSourceFile(filename),
-          undefined,
-          undefined,
-          false,
-          this.#transformerManager.getTransformers()
-        );
-
-        TranspileUtil.checkTranspileErrors(filename, result.diagnostics);
+        if (needsNewProgram) {
+          program = this.#state.createProgram(program);
+        }
+        const result = this.#state.writeInputFile(program, inputFile);
+        if (result?.diagnostics?.length) {
+          return result.diagnostics;
+        }
       } catch (err) {
-        if (!(err instanceof Error)) {
+        if (err instanceof Error) {
+          return err;
+        } else {
           throw err;
         }
-        const errContent = TranspileUtil.transpileError(filename, err);
-        this.#host.contents.set(filename, errContent);
       }
-      // Save writing for typescript program (`writeFile`)
-    } else {
-      this.#host.fetchFile(filename);
-    }
+    };
 
-    return this.#host.contents.get(filename)!;
-  }
-
-  /**
-   * Get program
-   * @private
-   */
-  getProgram(): ts.Program {
-    return this.#getProgram();
+    return emit;
   }
 
   /**
-   * Initialize the compiler
+   * Emit all files as a stream
    */
-  async init(): Promise<void> {
-    if (this.active) {
-      return;
-    }
-
-    const start = Date.now();
-    this.active = true;
-
-    if (!EnvUtil.isReadonly()) {
-      await this.#transformerManager.init();
-      // Enhance transpilation, with custom transformations
-      ModuleManager.setTranspiler(tsf => this.#transpile(tsf));
+  async * emit(files: string[], emitter: CompileEmitter): AsyncIterable<CompileEmitEvent> {
+    let i = 0;
+    for (const file of files) {
+      const err = await emitter(file);
+      const imp = file.replace(/.*node_modules\//, '');
+      yield { file: imp, i: i += 1, err, total: files.length };
     }
-
-    ModuleManager.onUnload((f, unlink) => this.#host.unload(f, unlink)); // Remove source
-
-    // Update source map support to read from transpiler cache
-    sourceMapSupport.install({
-      retrieveFile: p => this.#host.contents.get(PathUtil.toUnixTs(p))!
-    });
-
-    console.debug('Initialized', { duration: (Date.now() - start) / 1000 });
+    Log.debug(`Compiled ${i} files`);
   }
 
   /**
-   * Reset the compiler
+   * Run the compiler
    */
-  reset(): void {
-    if (!EnvUtil.isReadonly()) {
-      this.#transformerManager.reset();
-      this.#host.reset();
-      this.#program = undefined;
-    }
-    ModuleManager.clearUnloadHandlers();
-    SourceIndex.reset();
-    this.active = false;
-  }
+  async run(watch?: boolean): Promise<void> {
+    Log.debug('Compilation started');
 
-  /**
-   * Notify of an add/remove/change event
-   */
-  notify(type: EventType, filename: string): void {
-    console.debug('File Event', { type, filename: filename.replace(PathUtil.cwd, '.') });
-    this.#emitter.emit(type, filename);
-  }
+    const emitter = await this.getCompiler();
+    let failed = false;
 
-  /**
-   * Listen for events
-   */
-  on(type: EventType, handler: FileListener): this {
-    this.#emitter.on(type, handler);
-    return this;
-  }
+    Log.debug('Compiler loaded');
 
-  /**
-   * Unload if file is known
-   */
-  added(filename: string): void {
-    if (filename in require.cache) { // if already loaded
-      ModuleManager.unload(filename);
+    const resolveEmittedFile = ({ file, total, i, err }: CompileEmitEvent): TerminalProgressEvent => {
+      if (err) {
+        failed = true;
+        console.error(CompilerUtil.buildTranspileError(file, err));
+      }
+      return { idx: i, total, text: `Compiling [%idx/%total] -- ${file}` };
+    };
+
+    if (this.#dirtyFiles.length) {
+      await GlobalTerminal.trackProgress(this.emit(this.#dirtyFiles, emitter), resolveEmittedFile, { position: 'bottom' });
+      if (failed) {
+        Log.debug('Compilation failed');
+        process.exit(1);
+      } else {
+        Log.debug('Compilation succeeded');
+      }
+    } else if (watch) {
+      // Prime compiler before complete
+      const resolved = this.#state.getArbitraryInputFile();
+      await emitter(resolved, true);
     }
-    // Load Synchronously
-    require(filename);
-    this.notify('added', filename);
-  }
 
-  /**
-   * Handle when a file is removed during watch
-   */
-  removed(filename: string): void {
-    ModuleManager.unload(filename, true);
-    this.notify('removed', filename);
-  }
+    process.send?.('build-complete');
 
-  /**
-   * When a file changes during watch
-   */
-  changed(filename: string): void {
-    if (this.#host.hashChanged(filename)) {
-      ModuleManager.unload(filename);
-      // Load Synchronously
-      require(filename);
-      this.notify('changed', filename);
+    if (watch) {
+      Log.info('Watch is ready');
+      await this.#watchLocalModules(emitter);
+      for await (const _ of timers.setInterval(PING_THRESHOLD)) {
+        if (!await fs.stat(this.#state.resolveOutputFile('.')).catch(() => false)) { // Output removed
+          process.send?.('restart');
+        } else {
+          process.send?.('ping');
+        }
+      }
     }
   }
-}
-
-export const Compiler = new $Compiler();
+}

package/src/log.ts

@@ -0,0 +1,18 @@
+import util from 'util';
+
+import type { CompilerLogEvent } from '../support/log';
+
+function log(level: 'info' | 'debug', message: string, ...args: unknown[]): void {
+  if (process.send) {
+    const ev: CompilerLogEvent = [level, util.format(message, ...args)];
+    process.send(ev);
+  } else {
+    // eslint-disable-next-line no-console
+    console[level](message, ...args);
+  }
+}
+
+export const Log = {
+  debug: log.bind(null, 'debug'),
+  info: log.bind(null, 'info')
+};