@travetto/compiler 3.0.0-rc.1 → 3.0.0-rc.12

package/src/state.ts

@@ -0,0 +1,211 @@
+import ts from 'typescript';
+import { readFileSync } from 'fs';
+
+import { path, ManifestModuleUtil, ManifestModule, ManifestModuleFileType, ManifestRoot, WatchEvent } from '@travetto/manifest';
+
+import { CompilerUtil } from './util';
+import { TranspileUtil } from '../support/transpile';
+
+const validFile = (type: ManifestModuleFileType): boolean => type === 'ts' || type === 'package-json' || type === 'js';
+
+export class CompilerState {
+
+  #inputFiles: Set<string>;
+  #inputToSource = new Map<string, string>();
+  #stagedOutputToOutput = new Map<string, string>();
+  #inputToOutput = new Map<string, string | undefined>();
+  #inputDirectoryToSource = new Map<string, string>();
+  #sourceInputOutput = new Map<string, { source: string, input: string, stagedOutput?: string, output?: string, module: ManifestModule }>();
+
+  #sourceContents = new Map<string, string | undefined>();
+  #sourceFileObjects = new Map<string, ts.SourceFile>();
+  #sourceHashes = new Map<string, number>();
+
+  #manifest: ManifestRoot;
+  #modules: ManifestModule[];
+  #transformers: string[];
+
+  constructor(manifest: ManifestRoot) {
+    this.#manifest = manifest;
+    this.#modules = Object.values(this.#manifest.modules);
+    this.#inputFiles = new Set(this.#modules.flatMap(
+      x => [
+        ...x.files.bin ?? [],
+        ...x.files.src ?? [],
+        ...x.files.support ?? [],
+        ...x.files.doc ?? [],
+        ...x.files.test ?? [],
+        ...x.files.$index ?? [],
+        ...x.files.$package ?? []
+      ]
+        .filter(([file, type]) => validFile(type) || type === 'typings')
+        .map(([f]) => this.registerInput(x, f))
+    ));
+
+    this.#transformers = this.#modules.flatMap(
+      x => (x.files.$transformer ?? []).map(([f]) =>
+        path.resolve(manifest.workspacePath, x.sourceFolder, f)
+      )
+    );
+  }
+
+  async getCompilerOptions(): Promise<ts.CompilerOptions> {
+    return {
+      ...await TranspileUtil.getCompilerOptions(this.#manifest),
+      outDir: this.#manifest.workspacePath, // Force to root
+    };
+  }
+
+  resolveInput(file: string): string {
+    return this.#sourceInputOutput.get(file)!.input;
+  }
+
+  registerInput(module: ManifestModule, moduleFile: string): string {
+    const relativeInput = `${module.outputFolder}/${moduleFile}`;
+    const sourceFile = path.toPosix(path.resolve(this.#manifest.workspacePath, module.sourceFolder, moduleFile));
+    const sourceFolder = path.dirname(sourceFile);
+    const inputFile = path.resolve(this.#manifest.workspacePath, '##', relativeInput); // Ensure input is isolated
+    const inputFolder = path.dirname(inputFile);
+    const fileType = ManifestModuleUtil.getFileType(moduleFile);
+    const outputFile = fileType === 'typings' ?
+      undefined :
+      path.resolve(
+        this.#manifest.workspacePath,
+        this.#manifest.outputFolder,
+        CompilerUtil.inputToOutput(relativeInput)
+      );
+
+    // Rewrite stagedOutput to final output form
+    const stagedOutputFile = CompilerUtil.inputToOutput(inputFile);
+
+    this.#inputToSource.set(inputFile, sourceFile);
+    this.#sourceInputOutput.set(sourceFile, { source: sourceFile, input: inputFile, stagedOutput: stagedOutputFile, output: outputFile, module });
+    this.#inputToOutput.set(inputFile, outputFile);
+    this.#inputDirectoryToSource.set(inputFolder, sourceFolder);
+
+    if (stagedOutputFile) {
+      this.#stagedOutputToOutput.set(stagedOutputFile, outputFile!);
+      this.#stagedOutputToOutput.set(`${stagedOutputFile}.map`, `${outputFile!}.map`);
+    }
+
+    return inputFile;
+  }
+
+  removeInput(inputFile: string): void {
+    const source = this.#inputToSource.get(inputFile)!;
+    const { stagedOutput } = this.#sourceInputOutput.get(source)!;
+    this.#stagedOutputToOutput.delete(stagedOutput!);
+    this.#sourceInputOutput.delete(source);
+    this.#inputToSource.delete(inputFile);
+    this.#inputToOutput.delete(inputFile);
+    this.#inputFiles.delete(inputFile);
+  }
+
+  resetInputSource(inputFile: string): void {
+    this.#sourceFileObjects.delete(inputFile);
+    this.#sourceContents.delete(inputFile);
+  }
+
+  get modules(): ManifestModule[] {
+    return this.#modules;
+  }
+
+  get transformers(): string[] {
+    return this.#transformers;
+  }
+
+  getAllFiles(): string[] {
+    return [...this.#inputFiles];
+  }
+
+  // Build watcher
+  getWatcher(handler: {
+    create: (inputFile: string) => void;
+    update: (inputFile: string) => void;
+    delete: (outputFile: string) => void;
+  }): (ev: WatchEvent, folder: string) => void {
+    const mods = Object.fromEntries(this.modules.map(x => [path.resolve(this.#manifest.workspacePath, x.sourceFolder), x]));
+    return ({ file: sourceFile, action }: WatchEvent, folder: string): void => {
+      const mod = mods[folder];
+      const moduleFile = sourceFile.includes(mod.sourceFolder) ? sourceFile.split(`${mod.sourceFolder}/`)[1] : sourceFile;
+      switch (action) {
+        case 'create': {
+          const fileType = ManifestModuleUtil.getFileType(moduleFile);
+          if (validFile(fileType)) {
+            const hash = CompilerUtil.naiveHash(readFileSync(sourceFile, 'utf8'));
+            const input = this.registerInput(mod, moduleFile);
+            this.#sourceHashes.set(sourceFile, hash);
+            handler.create(input);
+          }
+          break;
+        }
+        case 'update': {
+          const io = this.#sourceInputOutput.get(sourceFile);
+          if (io) {
+            const hash = CompilerUtil.naiveHash(readFileSync(sourceFile, 'utf8'));
+            if (this.#sourceHashes.get(sourceFile) !== hash) {
+              this.resetInputSource(io.input);
+              this.#sourceHashes.set(sourceFile, hash);
+              handler.update(io.input);
+            }
+          }
+          break;
+        }
+        case 'delete': {
+          const io = this.#sourceInputOutput.get(sourceFile);
+          if (io) {
+            this.removeInput(io.input);
+            if (io.output) {
+              handler.delete(io.output);
+            }
+          }
+        }
+      }
+    };
+  }
+
+  // ts.CompilerHost
+  getCompilerHost(options: ts.CompilerOptions): ts.CompilerHost {
+    const host: ts.CompilerHost = {
+      getCanonicalFileName: (file: string): string => file,
+      getCurrentDirectory: path.cwd,
+      getDefaultLibFileName: (opts: ts.CompilerOptions): string => ts.getDefaultLibFileName(opts),
+      getNewLine: (): string => ts.sys.newLine,
+      useCaseSensitiveFileNames: (): boolean => ts.sys.useCaseSensitiveFileNames,
+      getDefaultLibLocation: (): string => path.dirname(ts.getDefaultLibFilePath(options)),
+      fileExists: (inputFile: string): boolean => this.#inputToSource.has(inputFile) || ts.sys.fileExists(inputFile),
+      directoryExists: (inputFolder: string): boolean => this.#inputDirectoryToSource.has(inputFolder) || ts.sys.directoryExists(inputFolder),
+      readFile: (inputFile: string): string | undefined => {
+        const res = this.#sourceContents.get(inputFile) ?? ts.sys.readFile(this.#inputToSource.get(inputFile) ?? inputFile);
+        this.#sourceContents.set(inputFile, res);
+        return res;
+      },
+      writeFile: (
+        outputFile: string,
+        text: string,
+        bom: boolean,
+        onError?: (message: string) => void,
+        sourceFiles?: readonly ts.SourceFile[],
+        data?: ts.WriteFileCallbackData
+      ): void => {
+        if (outputFile.endsWith('package.json')) {
+          text = CompilerUtil.rewritePackageJSON(this.#manifest, text, options);
+        } else if (!options.inlineSourceMap && options.sourceMap && outputFile.endsWith('.map')) {
+          text = CompilerUtil.rewriteSourceMap(this.#manifest.workspacePath, text, f => this.#sourceInputOutput.get(this.#inputToSource.get(f)!));
+        } else if (options.inlineSourceMap && CompilerUtil.isSourceMapUrlPosData(data)) {
+          text = CompilerUtil.rewriteInlineSourceMap(this.#manifest.workspacePath, text, f => this.#sourceInputOutput.get(this.#inputToSource.get(f)!), data);
+        }
+        outputFile = this.#stagedOutputToOutput.get(outputFile) ?? outputFile;
+        ts.sys.writeFile(outputFile, text, bom);
+      },
+      getSourceFile: (inputFile: string, language: ts.ScriptTarget, __onErr?: unknown): ts.SourceFile => {
+        if (!this.#sourceFileObjects.has(inputFile)) {
+          const content = host.readFile(inputFile)!;
+          this.#sourceFileObjects.set(inputFile, ts.createSourceFile(inputFile, content ?? '', language));
+        }
+        return this.#sourceFileObjects.get(inputFile)!;
+      }
+    };
+    return host;
+  }
+}

package/src/util.ts

@@ -0,0 +1,143 @@
+import ts from 'typescript';
+
+import { ManifestRoot, Package, path } from '@travetto/manifest';
+
+type InputToSource = (inputFile: string) => ({ source: string } | undefined);
+export type FileWatchEvent = { type: 'create' | 'delete' | 'update', path: string };
+
+const nativeCwd = process.cwd();
+
+/**
+ * Standard utilities for compiler
+ */
+export class CompilerUtil {
+
+  /**
+   * Map input file to output format, generally converting ts extensions to js
+   * @param file
+   * @returns
+   */
+  static inputToOutput(file: string): string {
+    return file.replace(/[.][tj]s$/, '.js');
+  }
+
+  /**
+   * Determines if write callback data has sourcemap information
+   * @param data
+   * @returns
+   */
+  static isSourceMapUrlPosData(data?: ts.WriteFileCallbackData): data is { sourceMapUrlPos: number } {
+    return data !== undefined && data !== null && typeof data === 'object' && ('sourceMapUrlPos' in data);
+  }
+
+  /**
+   * Rewrite's sourcemap locations to real folders
+   * @returns
+   */
+  static rewriteSourceMap(root: string, text: string, inputToSource: InputToSource): string {
+    const data: { sourceRoot: string, sources: string[] } = JSON.parse(text);
+    const src = path.resolve(data.sourceRoot, data.sources[0]);
+
+    const { source: file } = inputToSource(src) ?? {};
+    if (file) {
+      data.sourceRoot = root;
+      data.sources = [file];
+      text = JSON.stringify(data);
+    }
+
+    return text;
+  }
+
+  /**
+   * Rewrite's inline sourcemap locations to real folders
+   * @param text
+   * @param inputToSource
+   * @param writeData
+   * @returns
+   */
+  static rewriteInlineSourceMap(
+    root: string,
+    text: string,
+    inputToSource: InputToSource,
+    { sourceMapUrlPos }: ts.WriteFileCallbackData & { sourceMapUrlPos: number }
+  ): string {
+    const sourceMapUrl = text.substring(sourceMapUrlPos);
+    const [prefix, sourceMapData] = sourceMapUrl.split('base64,');
+    const rewritten = this.rewriteSourceMap(root, Buffer.from(sourceMapData, 'base64url').toString('utf8'), inputToSource);
+    return [
+      text.substring(0, sourceMapUrlPos),
+      prefix,
+      'base64,',
+      Buffer.from(rewritten, 'utf8').toString('base64url')
+    ].join('');
+  }
+
+  /**
+   * Rewrites the package.json to target .js files instead of .ts files, and pins versions
+   * @param manifest
+   * @param file
+   * @param text
+   * @returns
+   */
+  static rewritePackageJSON(manifest: ManifestRoot, text: string, opts: ts.CompilerOptions): string {
+    const pkg: Package = JSON.parse(text);
+    if (pkg.files) {
+      pkg.files = pkg.files.map(x => this.inputToOutput(x));
+    }
+    if (pkg.main) {
+      pkg.main = this.inputToOutput(pkg.main);
+    }
+    pkg.type = opts.module !== ts.ModuleKind.CommonJS ? 'module' : 'commonjs';
+    for (const key of ['devDependencies', 'dependencies', 'peerDependencies'] as const) {
+      if (key in pkg) {
+        for (const dep of Object.keys(pkg[key] ?? {})) {
+          if (dep in manifest.modules) {
+            pkg[key]![dep] = manifest.modules[dep].version;
+          }
+        }
+      }
+    }
+    return JSON.stringify(pkg, null, 2);
+  }
+
+  /**
+   * Build transpilation error
+   * @param filename The name of the file
+   * @param diagnostics The diagnostic errors
+   */
+  static buildTranspileError(filename: string, diagnostics: Error | readonly ts.Diagnostic[]): Error {
+    if (diagnostics instanceof Error) {
+      return diagnostics;
+    }
+
+    const errors: string[] = diagnostics.slice(0, 5).map(diag => {
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      const message = ts.flattenDiagnosticMessageText(diag.messageText, '\n');
+      if (diag.file) {
+        const { line, character } = diag.file.getLineAndCharacterOfPosition(diag.start!);
+        return ` @ ${diag.file.fileName.replace(nativeCwd, '.')}(${line + 1}, ${character + 1}): ${message}`;
+      } else {
+        return ` ${message}`;
+      }
+    });
+
+    if (diagnostics.length > 5) {
+      errors.push(`${diagnostics.length - 5} more ...`);
+    }
+    return new Error(`Transpiling ${filename.replace(nativeCwd, '.')} failed: \n${errors.join('\n')}`);
+  }
+
+  /**
+   * Naive hashing
+   */
+  static naiveHash(text: string): number {
+    let hash = 5381;
+
+    for (let i = 0; i < text.length; i++) {
+      // eslint-disable-next-line no-bitwise
+      hash = (hash * 33) ^ text.charCodeAt(i);
+    }
+
+    return Math.abs(hash);
+  }
+}

package/support/compiler-entry.ts

@@ -0,0 +1,2 @@
+import { Compiler } from '../src/compiler';
+Compiler.main();

package/support/launcher.ts

@@ -0,0 +1,160 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { Module } from 'module';
+
+import type { ManifestContext } from '@travetto/manifest';
+import { TranspileUtil } from './transpile';
+
+const SOURCE_SEED = ['package.json', 'index.ts', '__index__.ts', 'src', 'support', 'bin'];
+const PRECOMPILE_MODS = ['@travetto/terminal', '@travetto/manifest', '@travetto/transformer', '@travetto/compiler'];
+
+const importManifest = (ctx: ManifestContext): Promise<typeof import('@travetto/manifest')> =>
+  import(path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/manifest/__index__.js'));
+
+/**
+ * Recompile folder if stale
+ */
+async function compileIfStale(ctx: ManifestContext, scope: string, mod: string, seed: string[]): Promise<string[]> {
+  const files = await TranspileUtil.getModuleSources(ctx, mod, seed);
+  const changes = files.filter(x => x.stale).map(x => x.input);
+  const out: string[] = [];
+
+  try {
+    await TranspileUtil.withLogger(scope, { args: [mod], basic: false }, async log => {
+      if (files.some(f => f.stale)) {
+        log('debug', 'Starting');
+        for (const file of files.filter(x => x.stale)) {
+          await TranspileUtil.transpileFile(ctx, file.input, file.output);
+        }
+        if (changes.length) {
+          out.push(...changes.map(x => `${mod}/${x}`));
+          log('debug', `Source changed: ${changes.join(', ')}`);
+        }
+        log('debug', 'Completed');
+      } else {
+        log('debug', 'Skipped');
+      }
+    });
+  } catch (err) {
+    console.error(err);
+  }
+  return out;
+}
+
+/**
+ * Run the compiler
+ */
+export async function compile(ctx: ManifestContext, op?: 'watch' | 'build'): Promise<void> {
+  let changes = 0;
+
+  await TranspileUtil.withLogger('precompile', async () => {
+    for (const mod of PRECOMPILE_MODS) {
+      changes += (await compileIfStale(ctx, 'precompile', mod, SOURCE_SEED)).length;
+    }
+  });
+
+  const { ManifestUtil, ManifestDeltaUtil } = await importManifest(ctx);
+
+  const manifest = await TranspileUtil.withLogger('manifest', async () => ManifestUtil.buildManifest(ctx));
+
+  await TranspileUtil.withLogger('transformers', async () => {
+    for (const mod of Object.values(manifest.modules).filter(m => m.files.$transformer?.length)) {
+      changes += (await compileIfStale(ctx, 'transformers', mod.name, ['package.json', ...mod.files.$transformer!.map(x => x[0])])).length;
+    }
+  });
+
+  const delta = await TranspileUtil.withLogger('delta', async log => {
+    if (changes) {
+      log('debug', 'Skipping, everything changed');
+      return [{ type: 'changed', file: '*', module: ctx.mainModule } as const];
+    } else {
+      return ManifestDeltaUtil.produceDelta(ctx, manifest);
+    }
+  });
+
+  if (changes) {
+    await fs.rm(path.resolve(ctx.workspacePath, ctx.outputFolder), { recursive: true, force: true });
+    TranspileUtil.log('reset', [], 'info', 'Clearing output due to compiler changes');
+  }
+
+  // Write manifest
+  await TranspileUtil.withLogger('manifest', async log => {
+    await ManifestUtil.writeManifest(ctx, manifest);
+    log('debug', `Wrote manifest ${ctx.mainModule}`);
+
+    // Update all manifests
+    if (delta.length && ctx.monoRepo && !ctx.mainFolder) {
+      const names: string[] = [];
+      const mods = Object.values(manifest.modules).filter(x => x.local && x.name !== ctx.mainModule);
+      for (const mod of mods) {
+        await ManifestUtil.rewriteManifest(path.resolve(ctx.workspacePath, mod.sourceFolder));
+        names.push(mod.name);
+      }
+      log('debug', `Changes triggered ${delta.map(x => `${x.type}:${x.module}:${x.file}`)}`);
+      log('debug', `Rewrote monorepo manifests [changes=${delta.length}] ${names.join(', ')}`);
+    }
+  });
+
+  await TranspileUtil.withLogger('compile', { args: [], basic: false }, async log => {
+    const changed = delta.filter(x => x.type === 'added' || x.type === 'changed');
+    log('debug', `Started action=${op} changed=${changed.map(x => `${x.module}/${x.file}`)}`);
+    if (changed.length || op === 'watch') {
+      await TranspileUtil.runCompiler(ctx, manifest, changed, op === 'watch');
+      log('debug', 'Finished');
+    } else {
+      log('debug', 'Skipped');
+    }
+  });
+
+  if (op === 'build') {
+    TranspileUtil.log('build', [], 'info', 'Successfully built');
+  }
+}
+
+/**
+ * Export manifests
+ */
+export async function exportManifest(ctx: ManifestContext, output?: string, env = 'dev'): Promise<string | undefined> {
+  const { ManifestUtil } = await importManifest(ctx);
+  const manifest = await ManifestUtil.buildManifest(ctx);
+
+  // If in prod mode, only include std modules
+  if (/^prod/i.test(env)) {
+    manifest.modules = Object.fromEntries(
+      Object.values(manifest.modules)
+        .filter(x => x.profiles.includes('std'))
+        .map(m => [m.name, m])
+    );
+    // Mark output folder/workspace path as portable
+    manifest.outputFolder = '';
+    manifest.workspacePath = '';
+  }
+  if (output) {
+    if (!output.endsWith('.json')) {
+      output = path.resolve(output, 'manifest.json');
+    }
+
+    await TranspileUtil.writeTextFile(output, JSON.stringify(manifest));
+    TranspileUtil.log('manifest', [], 'info', `Wrote manifest ${output}`);
+    return output;
+  } else {
+    console.log(JSON.stringify(manifest, null, 2));
+    return;
+  }
+}
+
+export async function launchMain(ctx: ManifestContext): Promise<void> {
+  // Rewriting node_path
+  const nodeOut = path.resolve(ctx.workspacePath, ctx.outputFolder, 'node_modules');
+  const og = process.env.NODE_PATH;
+  process.env.NODE_PATH = [nodeOut, og].join(path.delimiter);
+  // @ts-expect-error
+  Module._initPaths();
+  process.env.NODE_PATH = og; // Restore
+
+  process.env.TRV_MANIFEST = path.resolve(nodeOut, ctx.mainModule);
+
+  // TODO: Externalize somehow?
+  const cliMain = path.join(nodeOut, '@travetto/cli/support/cli.js');
+  return await import(cliMain);
+}