@travetto/compiler 3.0.0-rc.4 → 3.0.0-rc.7

package/src/compiler.ts

@@ -1,187 +1,187 @@
-import { EventEmitter } from 'events';
-import * as sourceMapSupport from 'source-map-support';
-import * as ts from 'typescript';
+import ts from 'typescript';
+import fs from 'fs/promises';
+import path from 'path';
 
-import { PathUtil, EnvUtil, AppCache } from '@travetto/boot';
-import { SourceIndex } from '@travetto/boot/src/internal/source';
-import { ModuleManager } from '@travetto/boot/src/internal/module';
-import { Dynamic } from '@travetto/base/src/internal/dynamic';
-import { TranspileUtil } from '@travetto/boot/src/internal/transpile-util';
+import { ManifestState } from '@travetto/manifest';
+import { GlobalTerminal, TerminalProgressEvent } from '@travetto/terminal';
 
-import { SourceHost } from './host';
-import { TransformerManager } from './transformer';
+import { CompilerUtil } from './util';
+import { CompilerState } from './state';
 
-type FileListener = (name: string) => void;
-type EventType = 'added' | 'removed' | 'changed';
+export type TransformerProvider = {
+  init(checker: ts.TypeChecker): void;
+  get(): ts.CustomTransformers | undefined;
+};
+
+type EmitError = Error | readonly ts.Diagnostic[];
+type Emitter = (file: string, newProgram?: boolean) => EmitError | undefined;
+type EmitEvent = { file: string, i: number, total: number, err?: EmitError };
 
 /**
- * Compilation orchestrator, interfaces with watching, unloading, emitting and delegates appropriately
+ * Compilation support
  */
-@Dynamic('@travetto/compiler/support/dynamic.compiler')
-class $Compiler {
+export class Compiler {
+
+  #bootTsconfig: string;
+  #state: CompilerState;
+  #transformers: string[];
+
+  init(manifestState: ManifestState): this {
+    this.#state = new CompilerState(manifestState);
+    this.#bootTsconfig = this.#state.resolveModuleFile('@travetto/compiler', 'tsconfig.trv.json');
+
+    this.#transformers = this.state.modules.flatMap(
+      x => (x.files.support ?? [])
+        .filter(([f, type]) => type === 'ts' && f.startsWith('support/transformer.'))
+        .map(([f]) =>
+          path.resolve(
+            this.#state.manifest.workspacePath,
+            this.#state.manifest.compilerFolder,
+            x.output,
+            f.replace(/[.][tj]s$/, '.js')
+          )
+        )
+    );
 
-  #program: ts.Program | undefined;
-  #transformerManager = new TransformerManager();
-  #emitter = new EventEmitter();
-  #host = new SourceHost();
+    return this;
+  }
 
-  active = false;
+  get state(): CompilerState {
+    return this.#state;
+  }
 
   /**
-   * Build typescript program
-   *
-   * @param forFile If this file is new, force a recompilation
+   * Watches local modules
    */
-  #getProgram(forFile?: string): ts.Program {
+  async #watchLocalModules(emit: Emitter): Promise<() => Promise<void>> {
+    const folders = this.state.modules.filter(x => x.local).map(x => x.source);
+    const emitWithError = (file: string): void => {
+      const err = emit(file, true);
+      if (err) {
+        console.error(CompilerUtil.buildTranspileError(file, err));
+      } else {
+        console.error('Compiled', file.split('node_modules/')[1]);
+      }
+    };
+    const watcher = this.state.getWatcher({
+      create: (inputFile) => emitWithError(inputFile),
+      update: (inputFile) => emitWithError(inputFile),
+      delete: (outputFile) => fs.unlink(outputFile).catch(() => { })
+    });
+    return CompilerUtil.fileWatcher(folders, watcher);
+  }
 
-    const rootFiles = this.#host.getRootFiles();
+  async createTransformerProvider(): Promise<TransformerProvider> {
+    const { TransformerManager } = await import('@travetto/transformer');
+    return TransformerManager.create(this.#transformers, this.state.manifest);
+  }
 
-    if (!this.#program || (forFile && !rootFiles.has(forFile))) {
-      console.debug('Loading program', { size: rootFiles.size, src: forFile });
-      if (forFile) {
-        rootFiles.add(forFile);
-      }
-      this.#program = ts.createProgram({
-        rootNames: [...rootFiles],
-        options: TranspileUtil.compilerOptions,
-        host: this.#host,
-        oldProgram: this.#program
-      });
-      this.#transformerManager.build(this.#program.getTypeChecker());
-    }
-    return this.#program;
+  async writeRawFile(file: string, contents: string, mode?: string): Promise<void> {
+    const outFile = path.resolve(
+      this.#state.manifest.workspacePath,
+      this.#state.manifest.outputFolder,
+      file
+    );
+    await fs.mkdir(path.dirname(outFile), { recursive: true });
+    await fs.writeFile(outFile, contents, { encoding: 'utf8', mode });
+  }
+
+  async outputInit(): Promise<void> {
+    // Write manifest
+    await this.writeRawFile(this.#state.manifest.manifestFile, JSON.stringify(this.state.manifest));
+    // TODO: This needs to be isolated, just like in the bootstrap
+    await this.writeRawFile('trv', '#!/bin/sh\nnode node_modules/@travetto/cli/support/main.cli.js $@\n', '755');
+    await this.writeRawFile('trv.cmd', 'node node_modules/@travetto/cli/support/main.cli.js %*\n', '755');
   }
 
   /**
-   * Perform actual transpilation
+   * Compile in a single pass, only emitting dirty files
    */
-  #transpile(filename: string, force = false): string {
-    if (force || !AppCache.hasEntry(filename)) {
-      console.debug('Emitting', { filename: filename.replace(PathUtil.cwd, '.') });
-
+  async getCompiler(): Promise<Emitter> {
+    let program: ts.Program;
+
+    const transformers = await this.createTransformerProvider();
+    const options = await CompilerUtil.getCompilerOptions(
+      path.resolve(
+        this.#state.manifest.workspacePath,
+        this.#state.manifest.outputFolder,
+      ),
+      this.#bootTsconfig,
+      this.#state.manifest.workspacePath
+    );
+    const host = this.state.getCompilerHost(options);
+
+    const emit = (file: string, needsNewProgram = program === undefined): EmitError | undefined => {
+      if (needsNewProgram) {
+        program = ts.createProgram({ rootNames: this.#state.getAllFiles(), host, options, oldProgram: program });
+        transformers.init(program.getTypeChecker());
+      }
       try {
-        const prog = this.#getProgram(filename);
-
-        const result = prog.emit(
-          prog.getSourceFile(filename),
-          undefined,
-          undefined,
-          false,
-          this.#transformerManager.getTransformers()
+        const result = program.emit(
+          program.getSourceFile(file)!, host.writeFile, undefined, false, transformers.get()
         );
 
-        TranspileUtil.checkTranspileErrors(filename, result.diagnostics);
+        if (result.diagnostics?.length) {
+          return result.diagnostics;
+        }
       } catch (err) {
-        if (!(err instanceof Error)) {
+        if (err instanceof Error) {
+          return err;
+        } else {
           throw err;
         }
-        const errContent = TranspileUtil.transpileError(filename, err);
-        this.#host.contents.set(filename, errContent);
       }
-      // Save writing for typescript program (`writeFile`)
-    } else {
-      this.#host.fetchFile(filename);
-    }
-
-    return this.#host.contents.get(filename)!;
-  }
-
-  /**
-   * Get program
-   * @private
-   */
-  getProgram(): ts.Program {
-    return this.#getProgram();
-  }
-
-  /**
-   * Initialize the compiler
-   */
-  async init(): Promise<void> {
-    if (this.active) {
-      return;
-    }
-
-    const start = Date.now();
-    this.active = true;
-
-    if (!EnvUtil.isReadonly()) {
-      await this.#transformerManager.init();
-      // Enhance transpilation, with custom transformations
-      ModuleManager.setTranspiler(tsf => this.#transpile(tsf));
-    }
-
-    ModuleManager.onUnload((f, unlink) => this.#host.unload(f, unlink)); // Remove source
-
-    // Update source map support to read from transpiler cache
-    sourceMapSupport.install({
-      retrieveFile: p => this.#host.contents.get(PathUtil.toUnixTs(p))!
-    });
+    };
 
-    console.debug('Initialized', { duration: (Date.now() - start) / 1000 });
+    return emit;
   }
 
   /**
-   * Reset the compiler
+   * Emit all files as a stream
    */
-  reset(): void {
-    if (!EnvUtil.isReadonly()) {
-      this.#transformerManager.reset();
-      this.#host.reset();
-      this.#program = undefined;
+  async * emit(files: string[], emitter: Emitter): AsyncIterable<EmitEvent> {
+    let i = 0;
+    const manifest = this.#state.manifest;
+    for (const file of files) {
+      const err = emitter(file);
+      const outputFile = file
+        .replace(/[.]ts$/, '.js')
+        .replace(manifest.compilerFolder, manifest.outputFolder);
+      yield { file: outputFile, i: i += 1, err, total: files.length };
     }
-    ModuleManager.clearUnloadHandlers();
-    SourceIndex.reset();
-    this.active = false;
   }
 
   /**
-   * Notify of an add/remove/change event
+   * Run the compiler
    */
-  notify(type: EventType, filename: string): void {
-    console.debug('File Event', { type, filename: filename.replace(PathUtil.cwd, '.') });
-    this.#emitter.emit(type, filename);
-  }
-
-  /**
-   * Listen for events
-   */
-  on(type: EventType, handler: FileListener): this {
-    this.#emitter.on(type, handler);
-    return this;
-  }
+  async run(watch?: boolean): Promise<void> {
+    await this.outputInit();
+    const emitter = await this.getCompiler();
+    let failed = false;
+
+    const resolveEmittedFile = ({ file, total, i, err }: EmitEvent): TerminalProgressEvent => {
+      if (err) {
+        failed = true;
+        console.error(CompilerUtil.buildTranspileError(file, err));
+      }
+      return { idx: i, total, text: `Compiling [%idx/%total] -- ${file.split('node_modules/')[1]}` };
+    };
 
-  /**
-   * Unload if file is known
-   */
-  added(filename: string): void {
-    if (filename in require.cache) { // if already loaded
-      ModuleManager.unload(filename);
+    let files = this.state.getDirtyFiles();
+    if (!watch && !files.length) {
+      files = this.state.getAllFiles();
     }
-    // Load Synchronously
-    require(filename);
-    this.notify('added', filename);
-  }
 
-  /**
-   * Handle when a file is removed during watch
-   */
-  removed(filename: string): void {
-    ModuleManager.unload(filename, true);
-    this.notify('removed', filename);
-  }
+    if (files.length) {
+      await GlobalTerminal.trackProgress(this.emit(files, emitter), resolveEmittedFile, { position: 'bottom' });
+      if (failed) {
+        process.exit(1);
+      }
+    }
 
-  /**
-   * When a file changes during watch
-   */
-  changed(filename: string): void {
-    if (this.#host.hashChanged(filename)) {
-      ModuleManager.unload(filename);
-      // Load Synchronously
-      require(filename);
-      this.notify('changed', filename);
+    if (watch) {
+      await this.#watchLocalModules(emitter);
+      await new Promise(r => setTimeout(r, 1000 * 60 * 60 * 24));
     }
   }
-}
-
-export const Compiler = new $Compiler();
+}

package/src/state.ts

@@ -0,0 +1,225 @@
+import ts from 'typescript';
+import { mkdirSync, readFileSync, writeFile } from 'fs';
+
+import {
+  path,
+  ManifestModuleUtil, ManifestDelta, ManifestModule,
+  ManifestModuleFileType, ManifestRoot, ManifestState
+} from '@travetto/manifest';
+
+import { CompilerUtil, FileWatchEvent } from './util';
+
+const validFile = (type: ManifestModuleFileType): boolean => type === 'ts' || type === 'package-json' || type === 'js';
+
+export class CompilerState {
+
+  #inputFiles: Set<string>;
+  #relativeInputToSource = new Map<string, { source: string, module: ManifestModule }>();
+  #inputToSource = new Map<string, string>();
+  #inputToOutput = new Map<string, string | undefined>();
+  #inputDirectoryToSource = new Map<string, string>();
+  #sourceInputOutput = new Map<string, { input: string, output?: string, relativeInput: string, module: ManifestModule }>();
+
+  #sourceContents = new Map<string, string | undefined>();
+  #sourceFileObjects = new Map<string, ts.SourceFile>();
+  #sourceHashes = new Map<string, number>();
+
+  #manifest: ManifestRoot;
+  #delta: ManifestDelta;
+  #modules: ManifestModule[];
+
+  constructor({ manifest, delta }: ManifestState) {
+    this.#manifest = manifest;
+    this.#delta = delta;
+    this.#modules = Object.values(this.#manifest.modules);
+    this.#inputFiles = new Set(this.#modules.flatMap(
+      x => [
+        ...x.files.bin ?? [],
+        ...x.files.src ?? [],
+        ...x.files.support ?? [],
+        ...x.files.doc ?? [],
+        ...x.files.test ?? [],
+        ...x.files.$index ?? [],
+        ...x.files.$package ?? []
+      ]
+        .filter(([file, type]) => validFile(type) || type === 'typings')
+        .map(([f]) => this.registerInput(x, f))
+    ));
+
+    return this;
+  }
+
+  registerInput(module: ManifestModule, moduleFile: string): string {
+    const relativeInput = `${module.output}/${moduleFile}`;
+    const sourceFile = `${module.source}/${moduleFile}`;
+    const sourceFolder = path.dirname(sourceFile);
+    const inputFile = path.resolve(relativeInput);
+    const inputFolder = path.dirname(inputFile);
+    const fileType = ManifestModuleUtil.getFileType(moduleFile);
+    const outputFile = fileType === 'typings' ?
+      undefined :
+      path.resolve(
+        this.#manifest.workspacePath,
+        this.#manifest.outputFolder,
+        (fileType === 'ts' ? relativeInput.replace(/[.]ts$/, '.js') : relativeInput)
+      );
+
+    this.#inputToSource.set(inputFile, sourceFile);
+    this.#sourceInputOutput.set(sourceFile, { input: inputFile, output: outputFile, relativeInput, module });
+    this.#inputToOutput.set(inputFile, outputFile);
+    this.#inputDirectoryToSource.set(inputFolder, sourceFolder);
+    this.#relativeInputToSource.set(relativeInput, { source: sourceFile, module });
+
+    return inputFile;
+  }
+
+  removeInput(inputFile: string): void {
+    const source = this.#inputToSource.get(inputFile)!;
+    const { relativeInput } = this.#sourceInputOutput.get(source)!;
+    this.#sourceInputOutput.delete(source);
+    this.#inputToSource.delete(inputFile);
+    this.#inputToOutput.delete(inputFile);
+    this.#relativeInputToSource.delete(relativeInput);
+    this.#inputFiles.delete(inputFile);
+  }
+
+  resetInputSource(inputFile: string): void {
+    this.#sourceFileObjects.delete(inputFile);
+    this.#sourceContents.delete(inputFile);
+  }
+
+  get manifest(): ManifestRoot {
+    return this.#manifest;
+  }
+
+  get modules(): ManifestModule[] {
+    return this.#modules;
+  }
+
+  getDirtyFiles(): string[] {
+    if (this.#delta && Object.keys(this.#delta).length) { // If we have any changes
+      const files: string[] = [];
+      for (const [modName, events] of Object.entries(this.#delta)) {
+        const mod = this.#manifest.modules[modName];
+        for (const { file } of events) {
+          const fileType = ManifestModuleUtil.getFileType(file);
+          if (validFile(fileType)) {
+            files.push(path.resolve(mod.output, file));
+          }
+        }
+      }
+      return files;
+    } else {
+      return [];
+    }
+  }
+
+  getAllFiles(): string[] {
+    return [...this.#inputFiles];
+  }
+
+  resolveModuleFile(module: string, file: string): string {
+    return `${this.modules.find(m => m.name === module)!.source}/${file}`;
+  }
+
+  // Build watcher
+  getWatcher(handler: {
+    create: (inputFile: string) => void;
+    update: (inputFile: string) => void;
+    delete: (outputFile: string) => void;
+  }): (ev: FileWatchEvent, folder: string) => void {
+    const mods = Object.fromEntries(this.modules.map(x => [x.source, x]));
+    return ({ path: sourceFile, type }: FileWatchEvent, folder: string): void => {
+      const mod = mods[folder];
+      const moduleFile = sourceFile.replace(`${mod.source}/`, '');
+      switch (type) {
+        case 'create': {
+          const fileType = ManifestModuleUtil.getFileType(moduleFile);
+          if (validFile(fileType)) {
+            const hash = CompilerUtil.naiveHash(readFileSync(sourceFile, 'utf8'));
+            const input = this.registerInput(mod, moduleFile);
+            this.#sourceHashes.set(sourceFile, hash);
+            handler.create(input);
+          }
+          break;
+        }
+        case 'update': {
+          const io = this.#sourceInputOutput.get(sourceFile);
+          if (io) {
+            const hash = CompilerUtil.naiveHash(readFileSync(sourceFile, 'utf8'));
+            if (this.#sourceHashes.get(sourceFile) !== hash) {
+              this.resetInputSource(io.input);
+              this.#sourceHashes.set(sourceFile, hash);
+              handler.update(io.input);
+            }
+          }
+          break;
+        }
+        case 'delete': {
+          const io = this.#sourceInputOutput.get(sourceFile);
+          if (io) {
+            this.removeInput(io.input);
+            if (io.output) {
+              handler.delete(io.output);
+            }
+          }
+        }
+      }
+
+      // Update manifest on every change
+      writeFile(
+        path.resolve(
+          this.#manifest.workspacePath,
+          this.#manifest.outputFolder,
+          this.#manifest.manifestFile
+        ),
+        JSON.stringify(this.#manifest),
+        () => { });
+    };
+  }
+
+  // ts.CompilerHost
+  getCompilerHost(options: ts.CompilerOptions): ts.CompilerHost {
+    const host: ts.CompilerHost = {
+      getCanonicalFileName: (file: string): string => file,
+      getCurrentDirectory: path.cwd,
+      getDefaultLibFileName: (opts: ts.CompilerOptions): string => ts.getDefaultLibFileName(opts),
+      getNewLine: (): string => ts.sys.newLine,
+      useCaseSensitiveFileNames: (): boolean => ts.sys.useCaseSensitiveFileNames,
+      getDefaultLibLocation: (): string => path.dirname(ts.getDefaultLibFilePath(options)),
+      fileExists: (inputFile: string): boolean => this.#inputToSource.has(inputFile) || ts.sys.fileExists(inputFile),
+      directoryExists: (inputFolder: string): boolean => this.#inputDirectoryToSource.has(inputFolder) || ts.sys.directoryExists(inputFolder),
+      readFile: (inputFile: string): string | undefined => {
+        const res = this.#sourceContents.get(inputFile) ?? ts.sys.readFile(this.#inputToSource.get(inputFile) ?? inputFile);
+        this.#sourceContents.set(inputFile, res);
+        return res;
+      },
+      writeFile: (
+        outputFile: string,
+        text: string,
+        bom: boolean,
+        onError?: (message: string) => void,
+        sourceFiles?: readonly ts.SourceFile[],
+        data?: ts.WriteFileCallbackData
+      ): void => {
+        mkdirSync(path.dirname(outputFile), { recursive: true });
+        if (outputFile.endsWith('package.json')) {
+          text = CompilerUtil.rewritePackageJSON(this.manifest, text, options);
+        } else if (!options.inlineSourceMap && options.sourceMap && outputFile.endsWith('.map')) {
+          text = CompilerUtil.rewriteSourceMap(text, f => this.#relativeInputToSource.get(f));
+        } else if (options.inlineSourceMap && CompilerUtil.isSourceMapUrlPosData(data)) {
+          text = CompilerUtil.rewriteInlineSourceMap(text, f => this.#relativeInputToSource.get(f), data);
+        }
+        ts.sys.writeFile(outputFile, text, bom);
+      },
+      getSourceFile: (inputFile: string, language: ts.ScriptTarget, __onErr?: unknown): ts.SourceFile => {
+        if (!this.#sourceFileObjects.has(inputFile)) {
+          const content = host.readFile(inputFile)!;
+          this.#sourceFileObjects.set(inputFile, ts.createSourceFile(inputFile, content ?? '', language));
+        }
+        return this.#sourceFileObjects.get(inputFile)!;
+      }
+    };
+    return host;
+  }
+}