@travetto/compiler 3.0.0-rc.9 → 3.0.0

package/src/state.ts

@@ -1,39 +1,61 @@
 import ts from 'typescript';
-import { mkdirSync, readFileSync, writeFile } from 'fs';
 
-import {
-  path,
-  ManifestModuleUtil, ManifestDelta, ManifestModule,
-  ManifestModuleFileType, ManifestRoot, ManifestState
-} from '@travetto/manifest';
+import { path, ManifestModuleUtil, ManifestModule, ManifestRoot, ManifestIndex } from '@travetto/manifest';
+import { TransformerManager } from '@travetto/transformer';
 
-import { CompilerUtil, FileWatchEvent } from './util';
+import { CompilerUtil } from './util';
+import { TranspileUtil } from '../support/transpile';
+import { CompileStateEntry } from './types';
 
-const validFile = (type: ManifestModuleFileType): boolean => type === 'ts' || type === 'package-json' || type === 'js';
+function folderMapper(root: string, prefix: string): { dir: string, translate: (val: string) => string } {
+  let matched: string = '~~';
+  prefix = `/${prefix}`;
+  const final = path.resolve(root).replace(/\/[^\/]+/, m => {
+    matched = m;
+    return prefix;
+  });
+  return { dir: final, translate: (file: string) => file.replace(prefix, matched) };
+}
 
-export class CompilerState {
+export class CompilerState implements ts.CompilerHost {
 
-  #inputFiles: Set<string>;
-  #relativeInputToSource = new Map<string, { source: string, module: ManifestModule }>();
-  #inputToSource = new Map<string, string>();
-  #inputToOutput = new Map<string, string | undefined>();
+  static async get(idx: ManifestIndex): Promise<CompilerState> {
+    return new CompilerState().init(idx);
+  }
+
+  private constructor() { }
+
+  #rootDir: string;
+  #inputPathToSource: (file: string) => string;
+  #outputPath: string;
+  #inputFiles = new Set<string>();
   #inputDirectoryToSource = new Map<string, string>();
-  #sourceInputOutput = new Map<string, { input: string, output?: string, relativeInput: string, module: ManifestModule }>();
+  #inputToEntry = new Map<string, CompileStateEntry>();
+  #sourceToEntry = new Map<string, CompileStateEntry>();
+  #outputToEntry = new Map<string, CompileStateEntry>();
 
   #sourceContents = new Map<string, string | undefined>();
   #sourceFileObjects = new Map<string, ts.SourceFile>();
-  #sourceHashes = new Map<string, number>();
 
+  #manifestIndex: ManifestIndex;
   #manifest: ManifestRoot;
-  #delta: ManifestDelta;
   #modules: ManifestModule[];
+  #transformerManager: TransformerManager;
+  #compilerOptions: ts.CompilerOptions;
+
+  async init(idx: ManifestIndex): Promise<this> {
+    this.#manifestIndex = idx;
+    this.#manifest = idx.manifest;
+    const mapper = folderMapper(this.#manifest.workspacePath, '##');
+    this.#rootDir = mapper.dir;
+    this.#inputPathToSource = mapper.translate;
 
-  constructor({ manifest, delta }: ManifestState) {
-    this.#manifest = manifest;
-    this.#delta = delta;
+    this.#outputPath = path.resolve(this.#manifest.workspacePath, this.#manifest.outputFolder);
     this.#modules = Object.values(this.#manifest.modules);
-    this.#inputFiles = new Set(this.#modules.flatMap(
-      x => [
+
+    // Register all inputs
+    for (const x of this.#modules) {
+      const files = [
         ...x.files.bin ?? [],
         ...x.files.src ?? [],
         ...x.files.support ?? [],
@@ -41,45 +63,98 @@ export class CompilerState {
         ...x.files.test ?? [],
         ...x.files.$index ?? [],
         ...x.files.$package ?? []
-      ]
-        .filter(([file, type]) => validFile(type) || type === 'typings')
-        .map(([f]) => this.registerInput(x, f))
-    ));
+      ];
+      for (const [file, type] of files) {
+        if (CompilerUtil.validFile(type) || type === 'typings') {
+          this.registerInput(x, file);
+        }
+      }
+    }
+
+    this.#transformerManager = await TransformerManager.create(this.#manifestIndex);
+
+    this.#compilerOptions = {
+      ...await TranspileUtil.getCompilerOptions(this.#manifest),
+      rootDir: this.#rootDir,
+      outDir: this.#outputPath
+    };
 
     return this;
   }
 
+  get manifest(): ManifestRoot {
+    return this.#manifest;
+  }
+
+  get manifestIndex(): ManifestIndex {
+    return this.#manifestIndex;
+  }
+
+  resolveOutputFile(file: string): string {
+    return path.resolve(this.#manifest.workspacePath, this.#manifest.outputFolder, file);
+  }
+
+  getArbitraryInputFile(): string {
+    return this.getBySource(this.#manifestIndex.getModule('@travetto/manifest')!.files.src[0].sourceFile)!.input;
+  }
+
+  createProgram(oldProgram?: ts.Program): ts.Program {
+    const prog = ts.createProgram({ rootNames: this.getAllFiles(), host: this, options: this.#compilerOptions, oldProgram });
+    this.#transformerManager.init(prog.getTypeChecker());
+    return prog;
+  }
+
+  writeInputFile(program: ts.Program, inputFile: string): ts.EmitResult | undefined {
+    if (inputFile.endsWith('.json')) {
+      this.writeFile(this.#inputToEntry.get(inputFile)!.output!, this.readFile(inputFile)!, false);
+    } else if (inputFile.endsWith('.js')) {
+      this.writeFile(this.#inputToEntry.get(inputFile)!.output!, ts.transpile(this.readFile(inputFile)!, this.#compilerOptions), false);
+    } else if (inputFile.endsWith('.ts')) {
+      return program.emit(
+        program.getSourceFile(inputFile)!,
+        (...args) => this.writeFile(...args), undefined, false,
+        this.#transformerManager.get()
+      );
+    }
+  }
+
+  getBySource(sourceFile: string): CompileStateEntry | undefined {
+    return this.#sourceToEntry.get(sourceFile);
+  }
+
   registerInput(module: ManifestModule, moduleFile: string): string {
-    const relativeInput = `${module.output}/${moduleFile}`;
-    const sourceFile = `${module.source}/${moduleFile}`;
+    const relativeInput = `${module.outputFolder}/${moduleFile}`;
+    const sourceFile = path.toPosix(path.resolve(this.#manifest.workspacePath, module.sourceFolder, moduleFile));
     const sourceFolder = path.dirname(sourceFile);
-    const inputFile = path.resolve(relativeInput);
+    const inputFile = path.resolve(this.#rootDir, relativeInput); // Ensure input is isolated
     const inputFolder = path.dirname(inputFile);
     const fileType = ManifestModuleUtil.getFileType(moduleFile);
     const outputFile = fileType === 'typings' ?
       undefined :
-      path.resolve(
-        this.#manifest.workspacePath,
-        this.#manifest.outputFolder,
-        (fileType === 'ts' ? relativeInput.replace(/[.]ts$/, '.js') : relativeInput)
-      );
+      path.resolve(this.#outputPath, CompilerUtil.inputToOutput(relativeInput));
+
+    const entry = { source: sourceFile, input: inputFile, output: outputFile, module, relativeInput };
 
-    this.#inputToSource.set(inputFile, sourceFile);
-    this.#sourceInputOutput.set(sourceFile, { input: inputFile, output: outputFile, relativeInput, module });
-    this.#inputToOutput.set(inputFile, outputFile);
+    this.#inputToEntry.set(inputFile, entry);
+    this.#sourceToEntry.set(sourceFile, entry);
     this.#inputDirectoryToSource.set(inputFolder, sourceFolder);
-    this.#relativeInputToSource.set(relativeInput, { source: sourceFile, module });
+
+    if (outputFile) {
+      this.#outputToEntry.set(outputFile, entry);
+    }
+
+    this.#inputFiles.add(inputFile);
 
     return inputFile;
   }
 
   removeInput(inputFile: string): void {
-    const source = this.#inputToSource.get(inputFile)!;
-    const { relativeInput } = this.#sourceInputOutput.get(source)!;
-    this.#sourceInputOutput.delete(source);
-    this.#inputToSource.delete(inputFile);
-    this.#inputToOutput.delete(inputFile);
-    this.#relativeInputToSource.delete(relativeInput);
+    const { output, source } = this.#inputToEntry.get(inputFile)!;
+    if (output) {
+      this.#outputToEntry.delete(output);
+    }
+    this.#sourceToEntry.delete(source);
+    this.#inputToEntry.delete(inputFile);
     this.#inputFiles.delete(inputFile);
   }
 
@@ -88,138 +163,57 @@ export class CompilerState {
     this.#sourceContents.delete(inputFile);
   }
 
-  get manifest(): ManifestRoot {
-    return this.#manifest;
+  getAllFiles(): string[] {
+    return [...this.#inputFiles];
   }
 
-  get modules(): ManifestModule[] {
-    return this.#modules;
-  }
+  /* Start Compiler Host */
+  getCanonicalFileName(file: string): string { return file; }
+  getCurrentDirectory(): string { return this.#rootDir; }
+  getDefaultLibFileName(opts: ts.CompilerOptions): string { return ts.getDefaultLibFileName(opts); }
+  getNewLine(): string { return ts.sys.newLine; }
+  useCaseSensitiveFileNames(): boolean { return ts.sys.useCaseSensitiveFileNames; }
+  getDefaultLibLocation(): string { return path.dirname(ts.getDefaultLibFilePath(this.#compilerOptions)); }
 
-  getDirtyFiles(): string[] {
-    if (this.#delta && Object.keys(this.#delta).length) { // If we have any changes
-      const files: string[] = [];
-      for (const [modName, events] of Object.entries(this.#delta)) {
-        const mod = this.#manifest.modules[modName];
-        for (const { file } of events) {
-          const fileType = ManifestModuleUtil.getFileType(file);
-          if (validFile(fileType)) {
-            files.push(path.resolve(mod.output, file));
-          }
-        }
-      }
-      return files;
-    } else {
-      return [];
-    }
+  fileExists(inputFile: string): boolean {
+    return this.#inputToEntry.has(inputFile) || ts.sys.fileExists(this.#inputPathToSource(inputFile));
   }
 
-  getAllFiles(): string[] {
-    return [...this.#inputFiles];
+  directoryExists(inputDir: string): boolean {
+    return this.#inputDirectoryToSource.has(inputDir) || ts.sys.directoryExists(this.#inputPathToSource(inputDir));
   }
 
-  resolveModuleFile(module: string, file: string): string {
-    return `${this.modules.find(m => m.name === module)!.source}/${file}`;
-  }
-
-  // Build watcher
-  getWatcher(handler: {
-    create: (inputFile: string) => void;
-    update: (inputFile: string) => void;
-    delete: (outputFile: string) => void;
-  }): (ev: FileWatchEvent, folder: string) => void {
-    const mods = Object.fromEntries(this.modules.map(x => [x.source, x]));
-    return ({ path: sourceFile, type }: FileWatchEvent, folder: string): void => {
-      const mod = mods[folder];
-      const moduleFile = sourceFile.replace(`${mod.source}/`, '');
-      switch (type) {
-        case 'create': {
-          const fileType = ManifestModuleUtil.getFileType(moduleFile);
-          if (validFile(fileType)) {
-            const hash = CompilerUtil.naiveHash(readFileSync(sourceFile, 'utf8'));
-            const input = this.registerInput(mod, moduleFile);
-            this.#sourceHashes.set(sourceFile, hash);
-            handler.create(input);
-          }
-          break;
-        }
-        case 'update': {
-          const io = this.#sourceInputOutput.get(sourceFile);
-          if (io) {
-            const hash = CompilerUtil.naiveHash(readFileSync(sourceFile, 'utf8'));
-            if (this.#sourceHashes.get(sourceFile) !== hash) {
-              this.resetInputSource(io.input);
-              this.#sourceHashes.set(sourceFile, hash);
-              handler.update(io.input);
-            }
-          }
-          break;
-        }
-        case 'delete': {
-          const io = this.#sourceInputOutput.get(sourceFile);
-          if (io) {
-            this.removeInput(io.input);
-            if (io.output) {
-              handler.delete(io.output);
-            }
-          }
-        }
-      }
+  writeFile(
+    outputFile: string,
+    text: string,
+    bom: boolean,
+    onError?: (message: string) => void,
+    sourceFiles?: readonly ts.SourceFile[],
+    data?: ts.WriteFileCallbackData
+  ): void {
+    if (outputFile.endsWith('package.json')) {
+      text = CompilerUtil.rewritePackageJSON(this.#manifest, text);
+    } else if (!this.#compilerOptions.inlineSourceMap && this.#compilerOptions.sourceMap && outputFile.endsWith('.map')) {
+      text = CompilerUtil.rewriteSourceMap(this.#manifest, text, f => this.#outputToEntry.get(f.replace(/[.]map$/, ''))!);
+    } else if (this.#compilerOptions.inlineSourceMap && CompilerUtil.isSourceMapUrlPosData(data)) {
+      text = CompilerUtil.rewriteInlineSourceMap(this.#manifest, text, f => this.#outputToEntry.get(f)!, data);
+    }
+    ts.sys.writeFile(outputFile, text, bom);
+  }
 
-      // Update manifest on every change
-      writeFile(
-        path.resolve(
-          this.#manifest.workspacePath,
-          this.#manifest.outputFolder,
-          this.#manifest.manifestFile
-        ),
-        JSON.stringify(this.#manifest),
-        () => { });
-    };
+  readFile(inputFile: string): string | undefined {
+    const res = this.#sourceContents.get(inputFile) ?? ts.sys.readFile(
+      this.#inputToEntry.get(inputFile)?.source ?? this.#inputPathToSource(inputFile)
+    );
+    this.#sourceContents.set(inputFile, res);
+    return res;
   }
 
-  // ts.CompilerHost
-  getCompilerHost(options: ts.CompilerOptions): ts.CompilerHost {
-    const host: ts.CompilerHost = {
-      getCanonicalFileName: (file: string): string => file,
-      getCurrentDirectory: path.cwd,
-      getDefaultLibFileName: (opts: ts.CompilerOptions): string => ts.getDefaultLibFileName(opts),
-      getNewLine: (): string => ts.sys.newLine,
-      useCaseSensitiveFileNames: (): boolean => ts.sys.useCaseSensitiveFileNames,
-      getDefaultLibLocation: (): string => path.dirname(ts.getDefaultLibFilePath(options)),
-      fileExists: (inputFile: string): boolean => this.#inputToSource.has(inputFile) || ts.sys.fileExists(inputFile),
-      directoryExists: (inputFolder: string): boolean => this.#inputDirectoryToSource.has(inputFolder) || ts.sys.directoryExists(inputFolder),
-      readFile: (inputFile: string): string | undefined => {
-        const res = this.#sourceContents.get(inputFile) ?? ts.sys.readFile(this.#inputToSource.get(inputFile) ?? inputFile);
-        this.#sourceContents.set(inputFile, res);
-        return res;
-      },
-      writeFile: (
-        outputFile: string,
-        text: string,
-        bom: boolean,
-        onError?: (message: string) => void,
-        sourceFiles?: readonly ts.SourceFile[],
-        data?: ts.WriteFileCallbackData
-      ): void => {
-        mkdirSync(path.dirname(outputFile), { recursive: true });
-        if (outputFile.endsWith('package.json')) {
-          text = CompilerUtil.rewritePackageJSON(this.manifest, text, options);
-        } else if (!options.inlineSourceMap && options.sourceMap && outputFile.endsWith('.map')) {
-          text = CompilerUtil.rewriteSourceMap(text, f => this.#relativeInputToSource.get(f));
-        } else if (options.inlineSourceMap && CompilerUtil.isSourceMapUrlPosData(data)) {
-          text = CompilerUtil.rewriteInlineSourceMap(text, f => this.#relativeInputToSource.get(f), data);
-        }
-        ts.sys.writeFile(outputFile, text, bom);
-      },
-      getSourceFile: (inputFile: string, language: ts.ScriptTarget, __onErr?: unknown): ts.SourceFile => {
-        if (!this.#sourceFileObjects.has(inputFile)) {
-          const content = host.readFile(inputFile)!;
-          this.#sourceFileObjects.set(inputFile, ts.createSourceFile(inputFile, content ?? '', language));
-        }
-        return this.#sourceFileObjects.get(inputFile)!;
-      }
-    };
-    return host;
+  getSourceFile(inputFile: string, language: ts.ScriptTarget): ts.SourceFile {
+    if (!this.#sourceFileObjects.has(inputFile)) {
+      const content = this.readFile(inputFile)!;
+      this.#sourceFileObjects.set(inputFile, ts.createSourceFile(inputFile, content ?? '', language));
+    }
+    return this.#sourceFileObjects.get(inputFile)!;
   }
 }

package/src/types.ts

@@ -0,0 +1,14 @@
+import type ts from 'typescript';
+
+import type { ManifestModule } from '@travetto/manifest';
+
+export type CompileEmitError = Error | readonly ts.Diagnostic[];
+export type CompileEmitter = (file: string, newProgram?: boolean) => Promise<CompileEmitError | undefined>;
+export type CompileEmitEvent = { file: string, i: number, total: number, err?: CompileEmitError };
+export type CompileStateEntry = { source: string, input: string, relativeInput: string, output?: string, module: ManifestModule };
+
+export type CompileWatcherHandler = {
+  create: (inputFile: string) => void;
+  update: (inputFile: string) => void;
+  delete: (outputFile: string) => void;
+};

package/src/util.ts

@@ -1,10 +1,8 @@
 import ts from 'typescript';
-import fs from 'fs/promises';
-import { readdirSync } from 'fs';
 
-import { ManifestModule, ManifestRoot, Package, PackageUtil, path } from '@travetto/manifest';
+import { ManifestContext, ManifestModuleFileType, ManifestRoot, Package, path } from '@travetto/manifest';
 
-type InputToSource = (inputFile: string) => ({ source: string, module: ManifestModule } | undefined);
+type OutputToSource = (outputFile: string) => ({ source: string } | undefined);
 export type FileWatchEvent = { type: 'create' | 'delete' | 'update', path: string };
 
 const nativeCwd = process.cwd();
@@ -14,6 +12,20 @@ const nativeCwd = process.cwd();
  */
 export class CompilerUtil {
 
+  /**
+   * Determine if this is a manifest file we care about
+   */
+  static validFile = (type: ManifestModuleFileType): boolean => type === 'ts' || type === 'package-json' || type === 'js';
+
+  /**
+   * Map input file to output format, generally converting ts extensions to js
+   * @param file
+   * @returns
+   */
+  static inputToOutput(file: string): string {
+    return file.replace(/[.][tj]s$/, '.js');
+  }
+
   /**
    * Determines if write callback data has sourcemap information
    * @param data
@@ -27,35 +39,35 @@ export class CompilerUtil {
    * Rewrite's sourcemap locations to real folders
    * @returns
    */
-  static rewriteSourceMap(text: string, inputToSource: InputToSource): string {
-    const data: { sourceRoot: string, sources: string[] } = JSON.parse(text);
-    const [src] = data.sources;
-
-    const { source: file, module } = inputToSource(src) ?? {};
-    if (file && module) {
-      data.sourceRoot = module.source;
-      data.sources = [file.replace(`${module.source}/`, '')];
+  static rewriteSourceMap(ctx: ManifestContext, text: string, outputToSource: OutputToSource): string {
+    const data: { sourceRoot?: string, sources: string[] } = JSON.parse(text);
+    const output = this.inputToOutput(path.resolve(ctx.workspacePath, ctx.outputFolder, data.sources[0]));
+    const { source: file } = outputToSource(output) ?? {};
+
+    if (file) {
+      delete data.sourceRoot;
+      data.sources = [file];
       text = JSON.stringify(data);
     }
-
     return text;
   }
 
   /**
    * Rewrite's inline sourcemap locations to real folders
    * @param text
-   * @param inputToSource
+   * @param outputToSource
    * @param writeData
    * @returns
    */
   static rewriteInlineSourceMap(
+    ctx: ManifestContext,
     text: string,
-    inputToSource: InputToSource,
+    outputToSource: OutputToSource,
     { sourceMapUrlPos }: ts.WriteFileCallbackData & { sourceMapUrlPos: number }
   ): string {
     const sourceMapUrl = text.substring(sourceMapUrlPos);
     const [prefix, sourceMapData] = sourceMapUrl.split('base64,');
-    const rewritten = this.rewriteSourceMap(Buffer.from(sourceMapData, 'base64url').toString('utf8'), inputToSource);
+    const rewritten = this.rewriteSourceMap(ctx, Buffer.from(sourceMapData, 'base64url').toString('utf8'), outputToSource);
     return [
       text.substring(0, sourceMapUrlPos),
       prefix,
@@ -71,15 +83,15 @@ export class CompilerUtil {
    * @param text
    * @returns
    */
-  static rewritePackageJSON(manifest: ManifestRoot, text: string, opts: ts.CompilerOptions): string {
+  static rewritePackageJSON(manifest: ManifestRoot, text: string): string {
     const pkg: Package = JSON.parse(text);
     if (pkg.files) {
-      pkg.files = pkg.files.map(x => x.replace(/[.]ts$/, '.js'));
+      pkg.files = pkg.files.map(x => this.inputToOutput(x));
     }
     if (pkg.main) {
-      pkg.main = pkg.main.replace(/[.]ts$/, '.js');
+      pkg.main = this.inputToOutput(pkg.main);
     }
-    pkg.type = opts.module !== ts.ModuleKind.CommonJS ? 'module' : 'commonjs';
+    pkg.type = manifest.moduleType;
     for (const key of ['devDependencies', 'dependencies', 'peerDependencies'] as const) {
       if (key in pkg) {
         for (const dep of Object.keys(pkg[key] ?? {})) {
@@ -92,20 +104,6 @@ export class CompilerUtil {
     return JSON.stringify(pkg, null, 2);
   }
 
-  /**
-   * Read the given tsconfig.json values for the project
-   * @param path
-   * @returns
-   */
-  static async readTsConfigOptions(file: string): Promise<ts.CompilerOptions> {
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    const { options } = ts.parseJsonSourceFileConfigFileContent(
-      ts.readJsonConfigFile(file, ts.sys.readFile), ts.sys, nativeCwd
-    );
-    options.target = ts.ScriptTarget.ESNext;
-    return options;
-  }
-
   /**
    * Build transpilation error
    * @param filename The name of the file
@@ -133,59 +131,6 @@ export class CompilerUtil {
     return new Error(`Transpiling ${filename.replace(nativeCwd, '.')} failed: \n${errors.join('\n')}`);
   }
 
-  /**
-   * Allows for watching of explicit folders
-   *
-   * @param onEvent
-   * @returns
-   */
-  static async fileWatcher(
-    folders: string[],
-    onEvent: (ev: FileWatchEvent, folder: string) => void
-  ): Promise<() => Promise<void>> {
-    const watcher = await import('@parcel/watcher');
-    const subs: (() => Promise<void>)[] = [];
-    for (const folder of folders) {
-      const sub = await watcher.subscribe(folder, (err, events) => {
-        for (const ev of events) {
-          onEvent(ev, folder);
-        }
-      }, { ignore: ['node_modules', ...readdirSync(folder).filter(x => x.startsWith('.') && x.length > 2)] });
-      subs.push(() => sub.unsubscribe());
-    }
-    const readiedSubs = await Promise.all(subs);
-    return () => Promise.all(readiedSubs.map(s => s())).then(() => { });
-  }
-
-  /**
-   * Get loaded compiler options
-   */
-  static async getCompilerOptions(outputFolder: string, bootTsConfig: string, workspace: string): Promise<ts.CompilerOptions> {
-    const opts: Partial<ts.CompilerOptions> = {};
-    const rootDir = nativeCwd;
-    const projTsconfig = path.resolve('tsconfig.json');
-    // Fallback to base tsconfig if not found in local folder
-    const config = (await fs.stat(projTsconfig).catch(() => false)) ? projTsconfig : bootTsConfig;
-    const base = await this.readTsConfigOptions(config);
-
-    const { type } = PackageUtil.readPackage(workspace);
-
-    if (type !== undefined) {
-      base.module = `${type}`.toLowerCase() === 'commonjs' ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext;
-    }
-
-    return {
-      ...base,
-      resolveJsonModule: true,
-      allowJs: true,
-      outDir: outputFolder,
-      sourceRoot: rootDir,
-      ...opts,
-      rootDir,
-    };
-  }
-
-
   /**
    * Naive hashing
    */