@travetto/compiler 3.0.0-rc.1 → 3.0.0-rc.12

package/README.md

@@ -1,7 +1,7 @@
 <!-- This file was generated by @travetto/doc and should not be modified directly -->
-<!-- Please modify https://github.com/travetto/travetto/tree/main/module/compiler/doc.ts and execute "npx trv doc" to rebuild -->
+<!-- Please modify https://github.com/travetto/travetto/tree/main/module/compiler/DOC.ts and execute "npx trv doc" to rebuild -->
 # Compiler
-## Node-integration of Typescript Compiler with advanced functionality for detecting changes in classes and methods.
+## Compiler
 
 **Install: @travetto/compiler**
 ```bash
@@ -17,9 +17,12 @@ This module expands upon [Typescript](https://typescriptlang.org), with suppleme
    *  Support for detecting changes in sources files at runtime
    *  Allows for hot-reloading of classes during development    
       *  Utilizes `es2015` `Proxy`s to allow for swapping out implementation at runtime
-
 Additionally, there is support for common AST transformations via [Transformation](https://github.com/travetto/travetto/tree/main/module/transformer#readme "Functionality for AST transformations, with transformer registration, and general utils")
-
 ## Debugging
-
 When dealing with transformers, logging is somewhat tricky as the compiler executes before the code is loaded.  To that end, the file `compiler.log` is created in the cache directory during the compilation process. This is a location that transformers should be free to log to, for debugging, and any additional feedback.
+
+## CLI
+
+The module provides the ability to clear the compilation cache to handle any inconsistencies that may arise.
+
+TODO: Describe cli behavior

package/__index__.ts

@@ -0,0 +1,3 @@
+export * from './src/compiler';
+export * from './src/state';
+export * from './src/util';

package/bin/trv.js

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+// @ts-check
+import fs from 'fs/promises';
+import path from 'path';
+import { createRequire } from 'module';
+
+import { getManifestContext } from '@travetto/manifest/bin/context.js';
+
+const VALID_OPS = { watch: 'watch', build: 'build', clean: 'clean', manifest: 'manifest' };
+
+/**
+ * @param {import('@travetto/manifest').ManifestContext} ctx
+ * @return {Promise<import('@travetto/compiler/support/launcher')>}
+ */
+const $getLauncher = async (ctx) => {
+  const compPkg = createRequire(path.resolve('node_modules')).resolve('@travetto/compiler/package.json');
+  const files = [];
+
+  for (const file of ['support/launcher.js', 'support/transpile.js', 'package.json']) {
+    const target = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/compiler', file);
+    const src = compPkg.replace('package.json', file.replace(/[.]js$/, '.ts'));
+
+    const targetTime = await fs.stat(target).then(s => Math.max(s.mtimeMs, s.ctimeMs)).catch(() => 0);
+    const srcTime = await fs.stat(src).then(s => Math.max(s.mtimeMs, s.ctimeMs));
+    // If stale
+    if (srcTime > targetTime) {
+      const ts = (await import('typescript')).default;
+      const module = ctx.moduleType === 'module' ? ts.ModuleKind.ESNext : ts.ModuleKind.CommonJS;
+      await fs.mkdir(path.dirname(target), { recursive: true });
+      const text = await fs.readFile(src, 'utf8');
+      if (file.endsWith('.js')) {
+        let content = ts.transpile(text, {
+          target: ts.ScriptTarget.ES2020, module, esModuleInterop: true, allowSyntheticDefaultImports: true
+        });
+        if (ctx.moduleType === 'module') {
+          content = content.replace(/^((?:im|ex)port .*from '[.][^']+)(')/mg, (_, a, b) => `${a}.js${b}`)
+            .replace(/^(import [^\n]*from '[^.][^\n/]+[/][^\n/]+[/][^\n']+)(')/mg, (_, a, b) => `${a}.js${b}`);
+        }
+        await fs.writeFile(target, content, 'utf8');
+      } else {
+        const pkg = JSON.parse(text);
+        pkg.type = ctx.moduleType;
+        await fs.writeFile(target, JSON.stringify(pkg, null, 2), 'utf8');
+      }
+      // Compile
+    }
+    files.push(target);
+  }
+
+  try { return await require(files[0]); }
+  catch { return import(files[0]); }
+};
+
+/**
+ * Parse arguments
+ * @param {string[]} args
+ * @returns {{ op?: keyof typeof VALID_OPS, clean?: boolean, outputPath?: string, env?: string }}
+ */
+function parseArgs(args) {
+  const op = VALID_OPS[args.find(x => !x.startsWith('-')) ?? ''];
+  return {
+    op,
+    clean: args.includes('--clean') || args.includes('-c'),
+    ...(op === 'manifest' ? { outputPath: args[1], env: args[2] } : {})
+  };
+}
+
+const exec = async () => {
+  const ctx = await getManifestContext();
+  const { op, outputPath, env, ...flags } = parseArgs(process.argv.slice(2));
+
+  // Clean if needed
+  if (op === 'clean' || (op && flags.clean)) {
+    for (const f of [ctx.outputFolder, ctx.compilerFolder]) {
+      await fs.rm(path.resolve(ctx.workspacePath, f), { force: true, recursive: true });
+    }
+  }
+
+  if (op === 'clean') { // Clean needs to not attempt to compile/load launcher
+    return console.log(`Cleaned ${ctx.workspacePath}: [${ctx.outputFolder}, ${ctx.compilerFolder}]`);
+  }
+
+  const { compile, launchMain, exportManifest } = await $getLauncher(ctx);
+
+  switch (op) {
+    case 'manifest': return exportManifest(ctx, outputPath ?? '', env);
+    case 'watch':
+    case 'build': return compile(ctx, op);
+    default:
+      await compile(ctx, op);
+      return launchMain(ctx);
+  }
+};
+
+exec();

package/package.json

@@ -1,16 +1,11 @@
 {
   "name": "@travetto/compiler",
-  "displayName": "Compiler",
-  "version": "3.0.0-rc.1",
-  "description": "Node-integration of Typescript Compiler with advanced functionality for detecting changes in classes and methods.",
+  "version": "3.0.0-rc.12",
+  "description": "Compiler",
   "keywords": [
-    "tsc",
-    "typescript",
     "compiler",
-    "caching",
-    "change-detection",
-    "real-time",
-    "travetto"
+    "travetto",
+    "typescript"
   ],
   "homepage": "https://travetto.io",
   "license": "MIT",
@@ -18,21 +13,43 @@
     "email": "travetto.framework@gmail.com",
     "name": "Travetto Framework"
   },
+  "type": "module",
   "files": [
-    "index.ts",
+    "__index__.ts",
     "src",
-    "support"
+    "bin",
+    "support",
+    "tsconfig.trv.json"
   ],
-  "main": "index.ts",
+  "main": "__index__.ts",
+  "bin": {
+    "trv": "bin/trv.js"
+  },
   "repository": {
     "url": "https://github.com/travetto/travetto.git",
     "directory": "module/compiler"
   },
   "dependencies": {
-    "@travetto/base": "^3.0.0-rc.0",
-    "@travetto/transformer": "^3.0.0-rc.1",
-    "@travetto/watch": "^3.0.0-rc.0"
+    "@parcel/watcher": "^2.1.0",
+    "@travetto/manifest": "^3.0.0-rc.7",
+    "@travetto/terminal": "^3.0.0-rc.6",
+    "@travetto/transformer": "^3.0.0-rc.10"
+  },
+  "peerDependencies": {
+    "@travetto/cli": "^3.0.0-rc.9"
+  },
+  "peerDependenciesMeta": {
+    "@travetto/cli": {
+      "optional": true
+    }
+  },
+  "travetto": {
+    "displayName": "Compiler",
+    "profiles": [
+      "compile"
+    ]
   },
+  "private": false,
   "publishConfig": {
     "access": "public"
   }

package/src/compiler.ts

@@ -1,187 +1,185 @@
-import { EventEmitter } from 'events';
-import * as sourceMapSupport from 'source-map-support';
-import * as ts from 'typescript';
-
-import { PathUtil, EnvUtil, AppCache } from '@travetto/boot';
-import { SourceIndex } from '@travetto/boot/src/internal/source';
-import { ModuleManager } from '@travetto/boot/src/internal/module';
-import { Dynamic } from '@travetto/base/src/internal/dynamic';
-import { TranspileUtil } from '@travetto/boot/src/internal/transpile-util';
-
-import { SourceHost } from './host';
-import { TransformerManager } from './transformer';
+import util from 'util';
+import { install } from 'source-map-support';
+import ts from 'typescript';
+import fs from 'fs/promises';
+
+import { GlobalTerminal, TerminalProgressEvent } from '@travetto/terminal';
+import { RootIndex, watchFolders } from '@travetto/manifest';
+import { TransformerManager } from '@travetto/transformer';
+
+import { CompilerUtil } from './util';
+import { CompilerState } from './state';
+import type { CompilerLogEvent } from '../support/transpile';
+
+export type TransformerProvider = {
+  init(checker: ts.TypeChecker): void;
+  get(): ts.CustomTransformers | undefined;
+};
+
+type EmitError = Error | readonly ts.Diagnostic[];
+type Emitter = (file: string, newProgram?: boolean) => Promise<EmitError | undefined>;
+type EmitEvent = { file: string, i: number, total: number, err?: EmitError };
+
+function log(level: 'info' | 'debug', message: string, ...args: unknown[]): void {
+  if (process.send) {
+    const ev: CompilerLogEvent = [level, util.format(message, ...args)];
+    process.send(ev);
+  } else {
+    // eslint-disable-next-line no-console
+    console[level](message, ...args);
+  }
+}
 
-type FileListener = (name: string) => void;
-type EventType = 'added' | 'removed' | 'changed';
+const debug = log.bind(null, 'debug');
+const info = log.bind(null, 'info');
 
 /**
- * Compilation orchestrator, interfaces with watching, unloading, emitting and delegates appropriately
+ * Compilation support
  */
-@Dynamic('@travetto/compiler/support/dynamic.compiler')
-class $Compiler {
-
-  #program: ts.Program | undefined;
-  #transformerManager = new TransformerManager();
-  #emitter = new EventEmitter();
-  #host = new SourceHost();
-
-  active = false;
+export class Compiler {
 
   /**
-   * Build typescript program
-   *
-   * @param forFile If this file is new, force a recompilation
+   * Run compiler as a main entry point
    */
-  #getProgram(forFile?: string): ts.Program {
+  static async main(): Promise<void> {
+    const [dirty, watch] = process.argv.slice(2);
+    install();
+    const dirtyFiles = (await fs.readFile(dirty, 'utf8')).split(/\n/).filter(x => !!x);
+    return new Compiler(dirtyFiles).run(watch === 'true');
+  }
 
-    const rootFiles = this.#host.getRootFiles();
+  #state: CompilerState;
+  #dirtyFiles: string[];
 
-    if (!this.#program || (forFile && !rootFiles.has(forFile))) {
-      console.debug('Loading program', { size: rootFiles.size, src: forFile });
-      if (forFile) {
-        rootFiles.add(forFile);
-      }
-      this.#program = ts.createProgram({
-        rootNames: [...rootFiles],
-        options: TranspileUtil.compilerOptions,
-        host: this.#host,
-        oldProgram: this.#program
-      });
-      this.#transformerManager.build(this.#program.getTypeChecker());
-    }
-    return this.#program;
+  constructor(dirtyFiles: string[]) {
+    this.#state = new CompilerState(RootIndex.manifest);
+    this.#dirtyFiles = dirtyFiles[0] === '*' ?
+      this.#state.getAllFiles() :
+      dirtyFiles.map(f => this.#state.resolveInput(f));
+  }
+
+  get state(): CompilerState {
+    return this.#state;
   }
 
   /**
-   * Perform actual transpilation
+   * Watches local modules
    */
-  #transpile(filename: string, force = false): string {
-    if (force || !AppCache.hasEntry(filename)) {
-      console.debug('Emitting', { filename: filename.replace(PathUtil.cwd, '.') });
-
-      try {
-        const prog = this.#getProgram(filename);
-
-        const result = prog.emit(
-          prog.getSourceFile(filename),
-          undefined,
-          undefined,
-          false,
-          this.#transformerManager.getTransformers()
-        );
-
-        TranspileUtil.checkTranspileErrors(filename, result.diagnostics);
-      } catch (err) {
-        if (!(err instanceof Error)) {
-          throw err;
-        }
-        const errContent = TranspileUtil.transpileError(filename, err);
-        this.#host.contents.set(filename, errContent);
+  async #watchLocalModules(emit: Emitter): Promise<() => Promise<void>> {
+    const emitWithError = async (file: string): Promise<void> => {
+      const err = await emit(file, true);
+      if (err) {
+        info('Compilation Error', CompilerUtil.buildTranspileError(file, err));
+      } else {
+        info(`Compiled ${file.split('node_modules/')[1]}`);
       }
-      // Save writing for typescript program (`writeFile`)
-    } else {
-      this.#host.fetchFile(filename);
-    }
-
-    return this.#host.contents.get(filename)!;
+    };
+    const watcher = this.state.getWatcher({
+      create: emitWithError,
+      update: emitWithError,
+      delete: (outputFile) => fs.unlink(outputFile).catch(() => { })
+    });
+    return watchFolders(RootIndex.getLocalInputFolders(), watcher, {
+      filter: ev => ev.file.endsWith('.ts') || ev.file.endsWith('.js'),
+      ignore: ['node_modules']
+    });
   }
 
-  /**
-   * Get program
-   * @private
-   */
-  getProgram(): ts.Program {
-    return this.#getProgram();
+  async createTransformerProvider(): Promise<TransformerProvider> {
+    return TransformerManager.create(this.state.transformers);
   }
 
   /**
-   * Initialize the compiler
+   * Compile in a single pass, only emitting dirty files
    */
-  async init(): Promise<void> {
-    if (this.active) {
-      return;
-    }
-
-    const start = Date.now();
-    this.active = true;
-
-    if (!EnvUtil.isReadonly()) {
-      await this.#transformerManager.init();
-      // Enhance transpilation, with custom transformations
-      ModuleManager.setTranspiler(tsf => this.#transpile(tsf));
-    }
+  async getCompiler(): Promise<Emitter> {
+    let program: ts.Program;
 
-    ModuleManager.onUnload((f, unlink) => this.#host.unload(f, unlink)); // Remove source
+    const transformers = await this.createTransformerProvider();
+    const options = await this.state.getCompilerOptions();
+    const host = this.state.getCompilerHost(options);
 
-    // Update source map support to read from transpiler cache
-    sourceMapSupport.install({
-      retrieveFile: p => this.#host.contents.get(PathUtil.toUnixTs(p))!
-    });
+    const emit = async (file: string, needsNewProgram = program === undefined): Promise<EmitError | undefined> => {
+      try {
+        if (needsNewProgram) {
+          program = ts.createProgram({ rootNames: this.#state.getAllFiles(), host, options, oldProgram: program });
+          transformers.init(program.getTypeChecker());
+        }
+        if (file.endsWith('.json')) {
+          host.writeFile(file, host.readFile(file)!, false);
+        } else if (file.endsWith('.js')) {
+          host.writeFile(file, ts.transpile(host.readFile(file)!, options), false);
+        } else {
+          const result = program.emit(
+            program.getSourceFile(file)!, host.writeFile, undefined, false, transformers.get()
+          );
+
+          if (result.diagnostics?.length) {
+            return result.diagnostics;
+          }
+        }
+      } catch (err) {
+        if (err instanceof Error) {
+          return err;
+        } else {
+          throw err;
+        }
+      }
+    };
 
-    console.debug('Initialized', { duration: (Date.now() - start) / 1000 });
+    return emit;
   }
 
   /**
-   * Reset the compiler
+   * Emit all files as a stream
    */
-  reset(): void {
-    if (!EnvUtil.isReadonly()) {
-      this.#transformerManager.reset();
-      this.#host.reset();
-      this.#program = undefined;
+  async * emit(files: string[], emitter: Emitter): AsyncIterable<EmitEvent> {
+    let i = 0;
+    for (const file of files) {
+      const err = await emitter(file);
+      const imp = file.replace(/.*node_modules\//, '');
+      yield { file: imp, i: i += 1, err, total: files.length };
     }
-    ModuleManager.clearUnloadHandlers();
-    SourceIndex.reset();
-    this.active = false;
+    debug(`Compiled ${i} files`);
   }
 
   /**
-   * Notify of an add/remove/change event
+   * Run the compiler
    */
-  notify(type: EventType, filename: string): void {
-    console.debug('File Event', { type, filename: filename.replace(PathUtil.cwd, '.') });
-    this.#emitter.emit(type, filename);
-  }
+  async run(watch?: boolean): Promise<void> {
+    debug('Compilation started');
 
-  /**
-   * Listen for events
-   */
-  on(type: EventType, handler: FileListener): this {
-    this.#emitter.on(type, handler);
-    return this;
-  }
+    const emitter = await this.getCompiler();
+    let failed = false;
 
-  /**
-   * Unload if file is known
-   */
-  added(filename: string): void {
-    if (filename in require.cache) { // if already loaded
-      ModuleManager.unload(filename);
-    }
-    // Load Synchronously
-    require(filename);
-    this.notify('added', filename);
-  }
+    debug('Compiler loaded');
 
-  /**
-   * Handle when a file is removed during watch
-   */
-  removed(filename: string): void {
-    ModuleManager.unload(filename, true);
-    this.notify('removed', filename);
-  }
+    const resolveEmittedFile = ({ file, total, i, err }: EmitEvent): TerminalProgressEvent => {
+      if (err) {
+        failed = true;
+        console.error(CompilerUtil.buildTranspileError(file, err));
+      }
+      return { idx: i, total, text: `Compiling [%idx/%total] -- ${file}` };
+    };
+
+    if (this.#dirtyFiles.length) {
+      await GlobalTerminal.trackProgress(this.emit(this.#dirtyFiles, emitter), resolveEmittedFile, { position: 'bottom' });
+      if (failed) {
+        debug('Compilation failed');
+        process.exit(1);
+      } else {
+        debug('Compilation succeeded');
+      }
+    }
 
-  /**
-   * When a file changes during watch
-   */
-  changed(filename: string): void {
-    if (this.#host.hashChanged(filename)) {
-      ModuleManager.unload(filename);
-      // Load Synchronously
-      require(filename);
-      this.notify('changed', filename);
+    if (watch) {
+      if (!this.#dirtyFiles.length) {
+        const resolved = this.state.resolveInput(RootIndex.getModule('@travetto/manifest')!.files.src[0].sourceFile);
+        await emitter(resolved, true);
+      }
+      info('Watch is ready');
+      await this.#watchLocalModules(emitter);
+      await new Promise(r => setTimeout(r, 1000 * 60 * 60 * 24));
     }
   }
-}
-
-export const Compiler = new $Compiler();
+}