@travetto/compiler 3.0.0-rc.3 → 3.0.0-rc.6

package/src/util.ts

@@ -0,0 +1,202 @@
+import ts from 'typescript';
+import fs from 'fs/promises';
+import { readdirSync } from 'fs';
+
+import { ManifestModule, ManifestRoot, Package, PackageUtil, path } from '@travetto/manifest';
+
+type InputToSource = (inputFile: string) => ({ source: string, module: ManifestModule } | undefined);
+export type FileWatchEvent = { type: 'create' | 'delete' | 'update', path: string };
+
+const nativeCwd = process.cwd();
+
+/**
+ * Standard utilities for compiler
+ */
+export class CompilerUtil {
+
+  /**
+   * Determines if write callback data has sourcemap information
+   * @param data
+   * @returns
+   */
+  static isSourceMapUrlPosData(data?: ts.WriteFileCallbackData): data is { sourceMapUrlPos: number } {
+    return data !== undefined && data !== null && typeof data === 'object' && ('sourceMapUrlPos' in data);
+  }
+
+  /**
+   * Rewrite's sourcemap locations to real folders
+   * @returns
+   */
+  static rewriteSourceMap(text: string, inputToSource: InputToSource): string {
+    const data: { sourceRoot: string, sources: string[] } = JSON.parse(text);
+    const [src] = data.sources;
+
+    const { source: file, module } = inputToSource(src) ?? {};
+    if (file && module) {
+      data.sourceRoot = module.source;
+      data.sources = [file.replace(`${module.source}/`, '')];
+      text = JSON.stringify(data);
+    }
+
+    return text;
+  }
+
+  /**
+   * Rewrite's inline sourcemap locations to real folders
+   * @param text
+   * @param inputToSource
+   * @param writeData
+   * @returns
+   */
+  static rewriteInlineSourceMap(
+    text: string,
+    inputToSource: InputToSource,
+    { sourceMapUrlPos }: ts.WriteFileCallbackData & { sourceMapUrlPos: number }
+  ): string {
+    const sourceMapUrl = text.substring(sourceMapUrlPos);
+    const [prefix, sourceMapData] = sourceMapUrl.split('base64,');
+    const rewritten = this.rewriteSourceMap(Buffer.from(sourceMapData, 'base64url').toString('utf8'), inputToSource);
+    return [
+      text.substring(0, sourceMapUrlPos),
+      prefix,
+      'base64,',
+      Buffer.from(rewritten, 'utf8').toString('base64url')
+    ].join('');
+  }
+
+  /**
+   * Rewrites the package.json to target .js files instead of .ts files, and pins versions
+   * @param manifest
+   * @param file
+   * @param text
+   * @returns
+   */
+  static rewritePackageJSON(manifest: ManifestRoot, text: string, opts: ts.CompilerOptions): string {
+    const pkg: Package = JSON.parse(text);
+    if (pkg.files) {
+      pkg.files = pkg.files.map(x => x.replace(/[.]ts$/, '.js'));
+    }
+    if (pkg.main) {
+      pkg.main = pkg.main.replace(/[.]ts$/, '.js');
+    }
+    pkg.type = opts.module !== ts.ModuleKind.CommonJS ? 'module' : 'commonjs';
+    for (const key of ['devDependencies', 'dependencies', 'peerDependencies'] as const) {
+      if (key in pkg) {
+        for (const dep of Object.keys(pkg[key] ?? {})) {
+          if (dep in manifest.modules) {
+            pkg[key]![dep] = manifest.modules[dep].version;
+          }
+        }
+      }
+    }
+    return JSON.stringify(pkg, null, 2);
+  }
+
+  /**
+   * Read the given tsconfig.json values for the project
+   * @param path
+   * @returns
+   */
+  static async readTsConfigOptions(file: string): Promise<ts.CompilerOptions> {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const { options } = ts.parseJsonSourceFileConfigFileContent(
+      ts.readJsonConfigFile(file, ts.sys.readFile), ts.sys, nativeCwd
+    );
+    options.target = ts.ScriptTarget.ESNext;
+    return options;
+  }
+
+  /**
+   * Build transpilation error
+   * @param filename The name of the file
+   * @param diagnostics The diagnostic errors
+   */
+  static buildTranspileError(filename: string, diagnostics: Error | readonly ts.Diagnostic[]): Error {
+    if (diagnostics instanceof Error) {
+      return diagnostics;
+    }
+
+    const errors: string[] = diagnostics.slice(0, 5).map(diag => {
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      const message = ts.flattenDiagnosticMessageText(diag.messageText, '\n');
+      if (diag.file) {
+        const { line, character } = diag.file.getLineAndCharacterOfPosition(diag.start!);
+        return ` @ ${diag.file.fileName.replace(nativeCwd, '.')}(${line + 1}, ${character + 1}): ${message}`;
+      } else {
+        return ` ${message}`;
+      }
+    });
+
+    if (diagnostics.length > 5) {
+      errors.push(`${diagnostics.length - 5} more ...`);
+    }
+    return new Error(`Transpiling ${filename.replace(nativeCwd, '.')} failed: \n${errors.join('\n')}`);
+  }
+
+  /**
+   * Allows for watching of explicit folders
+   *
+   * @param onEvent
+   * @returns
+   */
+  static async fileWatcher(
+    folders: string[],
+    onEvent: (ev: FileWatchEvent, folder: string) => void
+  ): Promise<() => Promise<void>> {
+    const watcher = await import('@parcel/watcher');
+    const subs: (() => Promise<void>)[] = [];
+    for (const folder of folders) {
+      const sub = await watcher.subscribe(folder, (err, events) => {
+        for (const ev of events) {
+          onEvent(ev, folder);
+        }
+      }, { ignore: ['node_modules', ...readdirSync(folder).filter(x => x.startsWith('.') && x.length > 2)] });
+      subs.push(() => sub.unsubscribe());
+    }
+    const readiedSubs = await Promise.all(subs);
+    return () => Promise.all(readiedSubs.map(s => s())).then(() => { });
+  }
+
+  /**
+   * Get loaded compiler options
+   */
+  static async getCompilerOptions(outputFolder: string, bootTsConfig: string, workspace: string): Promise<ts.CompilerOptions> {
+    const opts: Partial<ts.CompilerOptions> = {};
+    const rootDir = nativeCwd;
+    const projTsconfig = path.resolve('tsconfig.json');
+    // Fallback to base tsconfig if not found in local folder
+    const config = (await fs.stat(projTsconfig).catch(() => false)) ? projTsconfig : bootTsConfig;
+    const base = await this.readTsConfigOptions(config);
+
+    const { type } = PackageUtil.readPackage(workspace);
+
+    if (type !== undefined) {
+      base.module = `${type}`.toLowerCase() === 'commonjs' ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext;
+    }
+
+    return {
+      ...base,
+      resolveJsonModule: true,
+      allowJs: true,
+      outDir: outputFolder,
+      sourceRoot: rootDir,
+      ...opts,
+      rootDir,
+    };
+  }
+
+
+  /**
+   * Naive hashing
+   */
+  static naiveHash(text: string): number {
+    let hash = 5381;
+
+    for (let i = 0; i < text.length; i++) {
+      // eslint-disable-next-line no-bitwise
+      hash = (hash * 33) ^ text.charCodeAt(i);
+    }
+
+    return Math.abs(hash);
+  }
+}

package/support/bin/compiler-bootstrap.ts

@@ -0,0 +1,151 @@
+import path from 'path';
+import fs from 'fs/promises';
+import cp from 'child_process';
+
+import type { ManifestState, ManifestContext, ManifestRoot } from '@travetto/manifest';
+
+import { log, compileIfStale, getProjectSources, addNodePath, importManifest } from './utils';
+
+const PRECOMPILE_MODS = [
+  '@travetto/terminal',
+  '@travetto/manifest',
+  '@travetto/transformer',
+  '@travetto/compiler'
+];
+
+let manifestTemp;
+
+/**
+ * Step 0
+ */
+export async function precompile(ctx: ManifestContext): Promise<void> {
+  for (const mod of PRECOMPILE_MODS) {
+    await compileIfStale(ctx, `[0] Compiling ${mod}`, await getProjectSources(ctx, mod),);
+  }
+}
+
+export async function writeManifest(ctx: ManifestContext, manifest: ManifestRoot): Promise<void> {
+  const { ManifestUtil } = await importManifest(ctx);
+  return ManifestUtil.writeManifest(ctx, manifest);
+}
+
+async function rewriteManifests(ctx: ManifestContext, state: ManifestState): Promise<void> {
+  const { ManifestUtil } = await importManifest(ctx);
+
+  // Write out all changed manifests
+  const { getContext } = await import('../../bin/transpile');
+
+  const dirtyModules = [...Object.entries(state.delta)].filter(x => x[1].length > 0).map(([mod]) => mod);
+  for (const module of dirtyModules) {
+    const subCtx = await getContext(state.manifest.modules[module].source);
+    await ManifestUtil.createAndWriteManifest(subCtx);
+  }
+}
+
+/**
+ *  Step 1
+ */
+export async function buildManifest(ctx: ManifestContext): Promise<ManifestState> {
+  log('[1] Manifest Generation');
+  const { ManifestUtil } = await importManifest(ctx);
+  return ManifestUtil.produceState(ctx);
+}
+
+function shouldRebuildCompiler({ delta }: ManifestState): { total: boolean, transformers: [string, string][] } {
+  // Did enough things change to re-stage and build the compiler
+  const transformersChanged = Object.entries(delta)
+    .flatMap(([mod, files]) => files.map(x => [mod, x.file]))
+    .filter((ev): ev is [string, string] => ev[1].startsWith('support/transform'));
+  const transformerChanged = (delta['@travetto/transformer'] ?? []);
+  const compilerChanged = delta['@travetto/compiler'] ?? [];
+
+  const changed = transformerChanged.length || transformersChanged.length || compilerChanged.length;
+  if (changed) {
+    if (compilerChanged.length) {
+      log('[2] Compiler source changed @travetto/compiler', compilerChanged);
+    }
+    if (transformerChanged.length) {
+      log('[2] Compiler source changed @travetto/transformer', transformerChanged);
+    }
+    if (transformersChanged.length) {
+      log('[2] Compiler source changed */support/transform', transformersChanged);
+    }
+  }
+  return { total: changed > 0, transformers: transformersChanged };
+}
+
+/**
+ *  Step 2
+ */
+async function buildCompiler(state: ManifestState, ctx: ManifestContext): Promise<void> {
+  const changed = shouldRebuildCompiler(state);
+
+  if (changed.transformers.length) {
+    state = await buildManifest(ctx);
+    let x = 0;
+    for (const [mod, file] of changed.transformers) {
+      await compileIfStale(
+        ctx,
+        `[2.${x += 1}] ${file} Bootstrapping`,
+        await getProjectSources(ctx, mod, ['package.json', file])
+      );
+    }
+  }
+
+  log('[2] Compiler Ready');
+}
+
+/**
+ *  Step 4
+ */
+async function compileOutput(state: ManifestState, ctx: ManifestContext, watch?: boolean): Promise<void> {
+  let changes = Object.values(state.delta).flat();
+
+  // Remove files that should go away
+  await Promise.all(changes.filter(x => x.type === 'removed')
+    .map(x => fs.unlink(path.resolve(ctx.workspacePath, ctx.outputFolder, x.file)).catch(() => { })));
+
+  changes = changes.filter(x => x.type !== 'removed');
+
+  const { ManifestUtil } = await importManifest(ctx);
+  const resolve = ManifestUtil.resolveFile.bind(null, ctx, state.manifest, '@travetto/compiler');
+
+  manifestTemp ??= await ManifestUtil.writeState(state);
+  const cwd = path.resolve(ctx.workspacePath, ctx.compilerFolder);
+
+  if (changes.length) {
+    log('[3] Changed Sources', changes);
+
+    // Blocking call, compile only
+    const res = cp.spawnSync(process.argv0,
+      [resolve('support/main.output'), manifestTemp],
+      { cwd, stdio: 'inherit', encoding: 'utf8' }
+    );
+
+    if (res.status) {
+      throw new Error(res.stderr);
+    }
+
+    await rewriteManifests(ctx, state);
+  }
+
+  if (watch) {
+    // Rewrite state with updated manifest
+    const newState = await ManifestUtil.produceState(ctx);
+    manifestTemp = await ManifestUtil.writeState(newState);
+
+    // Run with watching
+    cp.spawnSync(process.argv0,
+      [resolve('support/main.output'), manifestTemp, 'true'], { cwd, stdio: 'inherit' }
+    );
+  }
+}
+
+export async function compile(ctx: ManifestContext, watch?: boolean): Promise<ManifestState> {
+  await precompile(ctx); // Step 0
+  const state = await buildManifest(ctx); // Step 1
+  await buildCompiler(state, ctx); // Step 2
+  await compileOutput(state, ctx, watch); // Step 3
+  await addNodePath(path.resolve(ctx.workspacePath, ctx.outputFolder));
+  return state;
+}

package/support/bin/utils.ts

@@ -0,0 +1,116 @@
+import fs from 'fs/promises';
+
+import path from 'path';
+import { type Stats } from 'fs';
+import { createRequire } from 'module';
+
+import type { ManifestContext, ManifestUtil } from '@travetto/manifest';
+
+import { transpileFile, writePackageJson } from '../../bin/transpile';
+
+const req = createRequire(`${process.cwd()}/node_modules`);
+
+type ModFile = { input: string, output: string, stale: boolean };
+
+const SOURCE_SEED = ['package.json', 'index.ts', '__index__.ts', 'src', 'support', 'bin'];
+
+export const IS_DEBUG = /\b([*]|build)\b/.test(process.env.DEBUG ?? '');
+
+const resolveImport = (lib: string): string => req.resolve(lib);
+const recentStat = (stat: Stats): number => Math.max(stat.ctimeMs, stat.mtimeMs);
+
+/**
+ * Common logging support
+ */
+export const log = IS_DEBUG ?
+  (...args: unknown[]): void => console.debug(new Date().toISOString(), ...args) :
+  (): void => { };
+
+/**
+ * Scan directory to find all project sources for comparison
+ */
+export async function getProjectSources(
+  ctx: ManifestContext, module: string, seed: string[] = SOURCE_SEED
+): Promise<ModFile[]> {
+  const inputFolder = (ctx.mainModule === module) ?
+    process.cwd() :
+    path.dirname(resolveImport(`${module}/package.json`));
+
+  const folders = seed.filter(x => !/[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+  const files = seed.filter(x => /[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+
+  while (folders.length) {
+    const sub = folders.pop();
+    if (!sub) {
+      continue;
+    }
+
+    for (const file of await fs.readdir(sub).catch(() => [])) {
+      if (file.startsWith('.')) {
+        continue;
+      }
+      const resolvedInput = path.resolve(sub, file);
+      const stat = await fs.stat(resolvedInput);
+
+      if (stat.isDirectory()) {
+        folders.push(resolvedInput);
+      } else if (file.endsWith('.d.ts')) {
+        // Do nothing
+      } else if (file.endsWith('.ts') || file.endsWith('.js')) {
+        files.push(resolvedInput);
+      }
+    }
+  }
+
+  const outputFolder = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', module);
+  const out: ModFile[] = [];
+  for (const input of files) {
+    const output = input.replace(inputFolder, outputFolder).replace(/[.]ts$/, '.js');
+    const inputTs = await fs.stat(input).then(recentStat, () => 0);
+    if (inputTs) {
+      const outputTs = await fs.stat(output).then(recentStat, () => 0);
+      await fs.mkdir(path.dirname(output), { recursive: true, });
+      out.push({ input, output, stale: inputTs > outputTs });
+    }
+  }
+
+  return out;
+}
+
+/**
+ * Recompile folder if stale
+ */
+export async function compileIfStale(ctx: ManifestContext, prefix: string, files: ModFile[]): Promise<void> {
+  try {
+    if (files.some(f => f.stale)) {
+      log(`${prefix} Starting`);
+      for (const file of files.filter(x => x.stale)) {
+        if (file.input.endsWith('package.json')) {
+          await writePackageJson(ctx, file.input, file.output);
+        } else {
+          await transpileFile(ctx, file.input, file.output);
+        }
+      }
+    } else {
+      log(`${prefix} Skipped`);
+    }
+  } catch (err) {
+    console.error(err);
+  }
+}
+
+/**
+ * Add node path at runtime
+ */
+export async function addNodePath(folder: string): Promise<void> {
+  process.env.NODE_PATH = [`${folder}/node_modules`, process.env.NODE_PATH].join(path.delimiter);
+  const { Module } = await import('module');
+  // @ts-expect-error
+  Module._initPaths();
+}
+
+/**
+ * Import the manifest utils once compiled
+ */
+export const importManifest = (ctx: ManifestContext): Promise<{ ManifestUtil: typeof ManifestUtil }> =>
+  import(path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto', 'manifest', '__index__.js'));

package/support/main.output.ts

@@ -0,0 +1,11 @@
+import { readFileSync } from 'fs';
+import { install } from 'source-map-support';
+
+import { ManifestState } from '@travetto/manifest';
+
+import { Compiler } from '../src/compiler';
+
+install();
+const [manifestState, watch] = process.argv.slice(2);
+const state: ManifestState = JSON.parse(readFileSync(manifestState, 'utf8'));
+new Compiler().init(state).run(watch === 'true');

package/tsconfig.trv.json

@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "module": "CommonJS",
+    "target": "esnext",
+    "moduleResolution": "node",
+    "lib": [
+      "es2022"
+    ],
+    "strict": true,
+    "esModuleInterop": true,
+    "strictPropertyInitialization": false,
+    "experimentalDecorators": true,
+    "noEmitOnError": false,
+    "noErrorTruncation": true,
+    "resolveJsonModule": true,
+    "sourceMap": true,
+    "inlineSourceMap": false,
+    "removeComments": true,
+    "importHelpers": true,
+    "outDir": "build/"
+  },
+}

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2020 ArcSine Technologies
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/index.ts

@@ -1,3 +0,0 @@
-export * from './src/host';
-export * from './src/compiler';
-export * from './src/transformer';

package/src/host.ts

@@ -1,142 +0,0 @@
-import * as ts from 'typescript';
-import { readFileSync } from 'fs';
-import * as path from 'path';
-
-import { PathUtil, AppCache } from '@travetto/boot';
-import { SourceUtil } from '@travetto/boot/src/internal/source-util';
-import { SystemUtil } from '@travetto/boot/src/internal/system';
-import { TranspileUtil } from '@travetto/boot/src/internal/transpile-util';
-import { SourceIndex } from '@travetto/boot/src/internal/source';
-import { AppManifest } from '@travetto/base';
-
-/**
- * Manages the source code and typescript relationship.
- */
-export class SourceHost implements ts.CompilerHost {
-
-  #rootFiles = new Set<string>();
-  #hashes = new Map<string, number>();
-  #sources = new Map<string, ts.SourceFile>();
-  readonly contents = new Map<string, string>();
-
-  #trackFile(filename: string, content: string): void {
-    this.contents.set(filename, content);
-    this.#hashes.set(filename, SystemUtil.naiveHash(readFileSync(filename, 'utf8'))); // Get og content for hashing
-  }
-
-  getCanonicalFileName: (file: string) => string = (f: string) => f;
-  getCurrentDirectory: () => string = () => PathUtil.cwd;
-  getDefaultLibFileName: (opts: ts.CompilerOptions) => string = (opts: ts.CompilerOptions) => ts.getDefaultLibFileName(opts);
-  getNewLine: () => string = () => ts.sys.newLine;
-  useCaseSensitiveFileNames: () => boolean = () => ts.sys.useCaseSensitiveFileNames;
-  getDefaultLibLocation(): string {
-    return path.dirname(ts.getDefaultLibFilePath(TranspileUtil.compilerOptions));
-  }
-
-  /**
-   * Get root files
-   */
-  getRootFiles(): Set<string> {
-    if (!this.#rootFiles.size) {
-      // Only needed for compilation
-      this.#rootFiles = new Set(SourceIndex.findByFolders(AppManifest.source, 'required').map(x => x.file));
-    }
-    return this.#rootFiles;
-  }
-
-  /**
-   * Read file from disk, using the transpile pre-processor on .ts files
-   */
-  readFile(filename: string): string {
-    filename = PathUtil.toUnixTs(filename);
-    let content = ts.sys.readFile(filename);
-    if (content === undefined) {
-      throw new Error(`Unable to read file ${filename}`);
-    }
-    if (filename.endsWith(SourceUtil.EXT) && !filename.endsWith('.d.ts')) {
-      content = SourceUtil.preProcess(filename, content);
-    }
-    return content;
-  }
-
-  /**
-   * Write file to disk, and set value in cache as well
-   */
-  writeFile(filename: string, content: string): void {
-    filename = PathUtil.toUnixTs(filename);
-    this.#trackFile(filename, content);
-    AppCache.writeEntry(filename, content);
-  }
-
-  /**
-   * Fetch file
-   */
-  fetchFile(filename: string): void {
-    filename = PathUtil.toUnixTs(filename);
-    const cached = AppCache.readEntry(filename);
-    this.#trackFile(filename, cached);
-  }
-
-  /**
-   * Get a source file on demand
-   * @returns
-   */
-  getSourceFile(filename: string, __tgt: unknown, __onErr: unknown, force?: boolean): ts.SourceFile {
-    if (!this.#sources.has(filename) || force) {
-      const content = this.readFile(filename)!;
-      this.#sources.set(filename, ts.createSourceFile(filename, content ?? '',
-        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-        (TranspileUtil.compilerOptions as ts.CompilerOptions).target!
-      ));
-    }
-    return this.#sources.get(filename)!;
-  }
-
-  /**
-   * See if a file exists
-   */
-  fileExists(filename: string): boolean {
-    filename = PathUtil.toUnixTs(filename);
-    return this.contents.has(filename) || ts.sys.fileExists(filename);
-  }
-
-  /**
-   * See if a file's hash code has changed
-   */
-  hashChanged(filename: string, content?: string): boolean {
-    content ??= readFileSync(filename, 'utf8');
-    // Let's see if they are really different
-    const hash = SystemUtil.naiveHash(content);
-    if (hash === this.#hashes.get(filename)) {
-      console.debug('Contents Unchanged', { filename });
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Unload a file from the transpiler
-   */
-  unload(filename: string, unlink = true): void {
-    if (this.contents.has(filename)) {
-      AppCache.removeExpiredEntry(filename, unlink);
-
-      if (unlink && this.#hashes.has(filename)) {
-        this.#hashes.delete(filename);
-      }
-      this.#rootFiles.delete(filename);
-      this.contents.delete(filename);
-      this.#sources.delete(filename);
-    }
-  }
-
-  /**
-   * Reset the transpiler
-   */
-  reset(): void {
-    this.contents.clear();
-    this.#rootFiles.clear();
-    this.#hashes.clear();
-    this.#sources.clear();
-  }
-}