@travetto/compiler 3.0.0-rc.9 → 3.0.0

package/src/watch.ts

@@ -0,0 +1,142 @@
+import { readFileSync } from 'fs';
+import fs from 'fs/promises';
+
+import {
+  ManifestContext, ManifestModuleUtil, ManifestUtil, WatchEvent, ManifestModuleFolderType,
+  ManifestModuleFileType, path, ManifestModule, watchFolders, WatchEventListener
+} from '@travetto/manifest';
+import { getManifestContext } from '@travetto/manifest/bin/context';
+
+import { CompilerState } from './state';
+import { CompilerUtil } from './util';
+import { CompileEmitter, CompileWatcherHandler } from './types';
+
+/**
+ * Utils for watching
+ */
+export class CompilerWatcher {
+
+  #sourceHashes = new Map<string, number>();
+  #manifestContexts = new Map<string, ManifestContext>();
+  #dirtyFiles: { modFolder: string, mod: string, moduleFile?: string, folderKey?: ManifestModuleFolderType, type?: ManifestModuleFileType }[] = [];
+  #state: CompilerState;
+
+  constructor(state: CompilerState) {
+    this.#state = state;
+  }
+
+  async #rebuildManifestsIfNeeded(): Promise<void> {
+    if (!this.#dirtyFiles.length) {
+      return;
+    }
+    const mods = [...new Set(this.#dirtyFiles.map(x => x.modFolder))];
+    const contexts = await Promise.all(mods.map(async folder => {
+      if (!this.#manifestContexts.has(folder)) {
+        const ctx = await getManifestContext(folder);
+        this.#manifestContexts.set(folder, ctx);
+      }
+      return this.#manifestContexts.get(folder)!;
+    }));
+
+    const files = this.#dirtyFiles;
+    this.#dirtyFiles = [];
+
+    for (const ctx of [...contexts, this.#state.manifest]) {
+      const newManifest = await ManifestUtil.buildManifest(ctx);
+      for (const file of files) {
+        if (file.folderKey && file.moduleFile && file.type && file.mod in newManifest.modules) {
+          newManifest.modules[file.mod].files[file.folderKey] ??= [];
+          newManifest.modules[file.mod].files[file.folderKey]!.push([file.moduleFile, file.type, Date.now()]);
+        }
+      }
+      await ManifestUtil.writeManifest(ctx, newManifest);
+    }
+    // Reindex
+    this.#state.manifestIndex.init(this.#state.manifestIndex.manifestFile);
+  }
+
+  #getModuleMap(): Record<string, ManifestModule> {
+    return Object.fromEntries(
+      Object.values(this.#state.manifest.modules).map(x => [path.resolve(this.#state.manifest.workspacePath, x.sourceFolder), x])
+    );
+  }
+
+  /**
+   * Get a watcher for a given compiler state
+   * @param state
+   * @param handler
+   * @returns
+   */
+  #getWatcher(handler: CompileWatcherHandler): WatchEventListener {
+    const mods = this.#getModuleMap();
+
+    return async ({ file: sourceFile, action }: WatchEvent, folder: string): Promise<void> => {
+      const mod = mods[folder];
+      const moduleFile = mod.sourceFolder ?
+        (sourceFile.includes(mod.sourceFolder) ? sourceFile.split(`${mod.sourceFolder}/`)[1] : sourceFile) :
+        sourceFile.replace(`${this.#state.manifest.workspacePath}/`, '');
+      switch (action) {
+        case 'create': {
+          const fileType = ManifestModuleUtil.getFileType(moduleFile);
+          this.#dirtyFiles.push({
+            mod: mod.name,
+            modFolder: folder,
+            moduleFile,
+            folderKey: ManifestModuleUtil.getFolderKey(moduleFile),
+            type: ManifestModuleUtil.getFileType(moduleFile)
+          });
+          if (CompilerUtil.validFile(fileType)) {
+            await this.#rebuildManifestsIfNeeded();
+
+            const hash = CompilerUtil.naiveHash(readFileSync(sourceFile, 'utf8'));
+            const input = this.#state.registerInput(mod, moduleFile);
+            this.#sourceHashes.set(sourceFile, hash);
+            handler.create(input);
+          }
+          break;
+        }
+        case 'update': {
+          await this.#rebuildManifestsIfNeeded();
+          const entry = this.#state.getBySource(sourceFile);
+          if (entry) {
+            const hash = CompilerUtil.naiveHash(readFileSync(sourceFile, 'utf8'));
+            if (this.#sourceHashes.get(sourceFile) !== hash) {
+              this.#state.resetInputSource(entry.input);
+              this.#sourceHashes.set(sourceFile, hash);
+              handler.update(entry.input);
+            }
+          }
+          break;
+        }
+        case 'delete': {
+          const entry = this.#state.getBySource(sourceFile);
+          if (entry) {
+            this.#state.removeInput(entry.input);
+            if (entry.output) {
+              this.#dirtyFiles.push({ mod: mod.name, modFolder: folder });
+              handler.delete(entry.output);
+            }
+          }
+        }
+      }
+    };
+  }
+
+  /**
+   * Watch files based on root index
+   */
+  watchFiles(emit: CompileEmitter): Promise<() => Promise<void>> {
+    return watchFolders(
+      this.#state.manifestIndex.getLocalInputFolderMapping(),
+      this.#getWatcher({
+        create: emit,
+        update: emit,
+        delete: (outputFile) => fs.rm(outputFile, { force: true })
+      }),
+      {
+        filter: ev => ev.file.endsWith('.ts') || ev.file.endsWith('.js') || ev.file.endsWith('package.json'),
+        ignore: ['node_modules']
+      }
+    );
+  }
+}

package/support/compiler-entry.ts

@@ -0,0 +1,2 @@
+import { Compiler } from '../src/compiler';
+Compiler.main();

package/support/launcher.ts

@@ -0,0 +1,152 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import type { ManifestContext } from '@travetto/manifest';
+
+import { TranspileUtil, CompileResult } from './transpile';
+import { LockManager } from './lock';
+import { LogUtil } from './log';
+
+const SOURCE_SEED = ['package.json', 'index.ts', '__index__.ts', 'src', 'support', 'bin'];
+const PRECOMPILE_MODS = ['@travetto/terminal', '@travetto/manifest', '@travetto/transformer', '@travetto/compiler'];
+
+const importManifest = (ctx: ManifestContext): Promise<typeof import('@travetto/manifest')> =>
+  import(path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/manifest/__index__.js'));
+
+/**
+ * Run the compiler
+ */
+async function compile(ctx: ManifestContext, op: 'watch' | 'build' | undefined, onMessage: (msg: unknown) => void): Promise<CompileResult> {
+  let changes = 0;
+
+  await LogUtil.withLogger('precompile', async () => {
+    for (const mod of PRECOMPILE_MODS) {
+      const count = (await TranspileUtil.compileIfStale(ctx, 'precompile', mod, SOURCE_SEED)).length;
+      if (mod !== '@travetto/terminal') {
+        changes += count;
+      }
+    }
+  });
+
+  const { ManifestUtil, ManifestDeltaUtil, PackageUtil } = await importManifest(ctx);
+
+  PackageUtil.clearCache();
+
+  const manifest = await LogUtil.withLogger('manifest', async () => ManifestUtil.buildManifest(ctx));
+
+  await LogUtil.withLogger('transformers', async () => {
+    for (const mod of Object.values(manifest.modules).filter(m => m.files.$transformer?.length)) {
+      changes += (await TranspileUtil.compileIfStale(ctx, 'transformers', mod.name, ['package.json', ...mod.files.$transformer!.map(x => x[0])])).length;
+    }
+  });
+
+  const delta = await LogUtil.withLogger('delta', async log => {
+    if (changes) {
+      log('debug', 'Skipping, everything changed');
+      return [{ type: 'changed', file: '*', module: ctx.mainModule } as const];
+    } else {
+      return ManifestDeltaUtil.produceDelta(ctx, manifest);
+    }
+  });
+
+  if (changes) {
+    await LogUtil.withLogger('reset', async log => {
+      await fs.rm(path.resolve(ctx.workspacePath, ctx.outputFolder), { recursive: true, force: true });
+      log('info', 'Clearing output due to compiler changes');
+    }, false);
+  }
+
+  // Write manifest
+  await LogUtil.withLogger('manifest', async log => {
+    await ManifestUtil.writeManifest(ctx, manifest);
+    log('debug', `Wrote manifest ${ctx.mainModule}`);
+
+    // Update all manifests
+    if (delta.length && ctx.monoRepo && !ctx.mainFolder) {
+      const names: string[] = [];
+      const mods = Object.values(manifest.modules).filter(x => x.local && x.name !== ctx.mainModule);
+      for (const mod of mods) {
+        await ManifestUtil.rewriteManifest(path.resolve(ctx.workspacePath, mod.sourceFolder));
+        names.push(mod.name);
+      }
+      log('debug', `Changes triggered ${delta.slice(0, 10).map(x => `${x.type}:${x.module}:${x.file}`)}`);
+      log('debug', `Rewrote monorepo manifests [changes=${delta.length}] ${names.slice(0, 10).join(', ')}`);
+    }
+  });
+
+  return await LogUtil.withLogger('compile', async log => {
+    const changed = delta.filter(x => x.type === 'added' || x.type === 'changed');
+    log('debug', `Started action=${op} changed=${changed.slice(0, 10).map(x => `${x.module}/${x.file}`)}`);
+    if (changed.length || op === 'watch') {
+      const res = await TranspileUtil.runCompiler(ctx, manifest, changed, op === 'watch', onMessage);
+      log('debug', 'Finished');
+      return res;
+    } else {
+      log('debug', 'Skipped');
+      return 'skipped';
+    }
+  }, false);
+}
+
+/**
+ * Export manifests
+ */
+async function exportManifest(ctx: ManifestContext, output?: string, env = 'dev'): Promise<void> {
+  const { ManifestUtil } = await importManifest(ctx);
+  let manifest = await ManifestUtil.buildManifest(ctx);
+
+  // If in prod mode, only include std modules
+  if (/^prod/i.test(env)) {
+    manifest = ManifestUtil.createProductionManifest(manifest);
+  }
+  if (output) {
+    output = await ManifestUtil.writeManifestToFile(output, manifest);
+    LogUtil.log('manifest', [], 'info', `Wrote manifest ${output}`);
+  } else {
+    console.log(JSON.stringify(manifest, null, 2));
+  }
+}
+
+/**
+ * Launch
+ */
+export async function launch(ctx: ManifestContext, root: ManifestContext, op?: 'build' | 'watch' | 'manifest', args: (string | undefined)[] = []): Promise<void> {
+  // If quiet enabled, turn off all output by default
+  LogUtil.level = process.env.TRV_BUILD ?? (process.env.TRV_QUIET ? 'none' : (!op ? 'warn' : 'info'));
+
+  if (op !== 'manifest' && await LockManager.getCompileAction(root, op) === 'build') {
+
+    // Ready signal
+    process.send?.('ready');
+
+    await LockManager.withLocks(root, async (acquire, release) => {
+      let action: CompileResult;
+      do {
+        acquire(op ?? 'build');
+        if (op === 'watch') {
+          acquire('build');
+        }
+        action = await compile(root, op, msg => {
+          switch (msg) {
+            case 'build-complete': {
+              release('build');
+              break;
+            }
+          }
+        });
+      } while (action === 'restart');
+    });
+  }
+
+  switch (op) {
+    case 'manifest': return exportManifest(ctx, ...args);
+    case 'build': return LogUtil.log('build', [], 'info', 'Successfully built');
+    case undefined: {
+      // TODO: Externalize somehow?
+      const outputPath = path.resolve(ctx.workspacePath, ctx.outputFolder);
+      process.env.TRV_MANIFEST = path.resolve(outputPath, 'node_modules', ctx.mainModule);
+      const cliMain = path.join(outputPath, 'node_modules', '@travetto/cli/support/cli.js');
+      return import(cliMain);
+    }
+  }
+}

package/support/lock-pinger.ts

@@ -0,0 +1,25 @@
+import { workerData, parentPort } from 'worker_threads';
+import { utimesSync } from 'fs';
+
+const data: { files: string[], interval: number } = workerData;
+const files = data.files;
+
+const interval = setInterval(() => {
+  const now = Date.now() / 1000;
+  for (const file of files) {
+    try {
+      utimesSync(file, now, now);
+    } catch { }
+  }
+}, data.interval);
+
+
+parentPort?.on('message', val => {
+  if (val === 'stop') {
+    files.splice(0, files.length);
+    clearInterval(interval);
+  } else if (val && typeof val === 'object' && 'files' in val && Array.isArray(val.files)) {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    files.splice(0, files.length, ...val.files as string[]);
+  }
+});

package/support/lock.ts

@@ -0,0 +1,234 @@
+import { Stats, watchFile, unwatchFile, rmSync, mkdirSync, writeFileSync, existsSync } from 'fs';
+import fs from 'fs/promises';
+import { Worker } from 'worker_threads';
+import path from 'path';
+
+import type { ManifestContext } from '@travetto/manifest';
+
+import { CompilerLogger, LogUtil } from './log';
+
+type LockStatus = 'complete' | 'stale';
+type LockDetails = {
+  pid: number | undefined;
+  file: string;
+};
+
+export type LockType = 'build' | 'watch';
+export type LockCompileAction = 'skip' | 'build';
+type LockAction = LockCompileAction | 'retry';
+
+const STALE_THRESHOLD = 1000;
+
+/**
+ * Manager for all lock activity
+ */
+export class LockManager {
+
+  /**
+   * Get the lock file name
+   */
+  static #getFileName(ctx: ManifestContext, type: LockType): string {
+    return path.resolve(ctx.workspacePath, ctx.toolFolder, `${type}.lock`);
+  }
+
+  /**
+   * Determine if the given stats are stale for modification time
+   */
+  static #isStale(stat?: Stats): boolean {
+    return !!stat && stat.mtimeMs < (Date.now() - STALE_THRESHOLD * 1.1);
+  }
+
+  /**
+   * Get the lock file details
+   */
+  static async #getDetails(ctx: ManifestContext, type: LockType): Promise<LockDetails> {
+    const file = this.#getFileName(ctx, type);
+    const stat = await fs.stat(file).catch(() => undefined);
+    const stale = this.#isStale(stat);
+    let pid: number | undefined;
+    if (stat) {
+      const { pid: filePid } = JSON.parse(await fs.readFile(file, 'utf8'));
+      if (stale) {
+        LogUtil.log('lock', [], 'debug', `${type} file is stale: ${stat.mtimeMs} vs ${Date.now()}`);
+      } else {
+        pid = filePid;
+      }
+    }
+    return { pid, file };
+  }
+
+  /**
+   * Acquire the lock file, and register a cleanup on exit
+   */
+  static #acquireFile(ctx: ManifestContext, type: LockType): void {
+    const file = this.#getFileName(ctx, type);
+    mkdirSync(path.dirname(file), { recursive: true });
+    LogUtil.log('lock', [], 'debug', `Acquiring ${type}`);
+    writeFileSync(file, JSON.stringify({ pid: process.pid }), 'utf8');
+  }
+
+  /**
+   * Release the lock file (i.e. deleting)
+   */
+  static #releaseFile(ctx: ManifestContext, type: LockType): void {
+    const file = this.#getFileName(ctx, type);
+    if (existsSync(file)) {
+      rmSync(file, { force: true });
+      LogUtil.log('lock', [], 'debug', `Releasing ${type}`);
+    }
+  }
+
+  /**
+   * Wait until a lock file is released, or it becomes stale
+   */
+  static async #waitForRelease(ctx: ManifestContext, type: LockType): Promise<LockStatus> {
+    const file = this.#getFileName(ctx, type);
+    let remove: (() => void) | undefined = undefined;
+
+    const prom = new Promise<LockStatus>(resolve => {
+      let timer: NodeJS.Timeout | undefined = undefined;
+      const handler = async (): Promise<void> => {
+        if (timer) {
+          clearTimeout(timer);
+        }
+        const stats = await fs.stat(file).catch(() => undefined);
+        if (!stats) {
+          resolve('complete');
+        } else if (this.#isStale(stats)) {
+          resolve('stale');
+        } else {
+          timer = setTimeout(handler, STALE_THRESHOLD * 1.1);
+        }
+      };
+
+      watchFile(file, handler);
+      handler();
+
+      remove = (): void => {
+        clearInterval(timer);
+        unwatchFile(file, handler);
+      };
+    });
+
+    return prom.finally(remove);
+  }
+
+  /**
+   * Read the watch lock file and determine its result, communicating with the user as necessary
+   */
+  static async #getWatchAction(ctx: ManifestContext, log: CompilerLogger, lockType: LockType | undefined, buildState: LockDetails): Promise<LockAction> {
+    if (lockType === 'watch') {
+      log('info', 'Already running');
+      return 'skip';
+    } else {
+      if (buildState.pid) {
+        log('warn', 'Already running, waiting for build to finish');
+        switch (await this.#waitForRelease(ctx, 'build')) {
+          case 'complete': {
+            log('info', 'Completed build');
+            return 'skip';
+          }
+          case 'stale': {
+            log('info', 'Became stale, retrying');
+            return 'retry';
+          }
+        }
+      } else {
+        log('info', 'Already running, and has built');
+        return 'skip';
+      }
+    }
+  }
+
+  /**
+   * Read the build lock file and determine its result, communicating with the user as necessary
+   */
+  static async #getBuildAction(ctx: ManifestContext, log: CompilerLogger, lockType: LockType | undefined): Promise<LockAction> {
+    if (lockType === 'watch') {
+      log('warn', 'Build already running, waiting to begin watch');
+      const res = await this.#waitForRelease(ctx, 'build');
+      log('info', `Finished with status of ${res}, retrying`);
+      return 'retry';
+    } else {
+      log('warn', 'Already running, waiting for completion');
+      switch (await this.#waitForRelease(ctx, lockType ?? 'build')) {
+        case 'complete': {
+          log('info', 'Completed');
+          return 'skip';
+        }
+        case 'stale': {
+          log('info', 'Became stale, retrying');
+          return 'retry';
+        }
+      }
+    }
+  }
+
+  /**
+   * Run code with support for lock acquire and release
+   */
+  static async withLocks(ctx: ManifestContext, fn: (acquire: (type: LockType) => void, release: (type: LockType) => void) => Promise<unknown>): Promise<void> {
+    const activeLockTypes = new Set<LockType>();
+
+    const pinger = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/compiler/support/lock-pinger.js');
+    const worker = new Worker(pinger, {
+      workerData: {
+        interval: STALE_THRESHOLD,
+        files: []
+      }
+    });
+
+    const notify = (): void => worker.postMessage({ files: [...activeLockTypes].map(t => this.#getFileName(ctx, t)) });
+
+    const stop = (): void => {
+      worker.postMessage('stop');
+      for (const type of activeLockTypes) {
+        this.#releaseFile(ctx, type);
+      }
+      worker.terminate().then(() => { });
+    };
+
+    process.on('SIGINT', stop);
+    process.on('exit', stop);
+
+    try {
+      await new Promise(r => worker.on('online', r));
+      await fn(
+        type => {
+          if (!activeLockTypes.has(type)) {
+            activeLockTypes.add(type);
+            this.#acquireFile(ctx, type);
+            notify();
+          }
+        },
+        type => {
+          if (activeLockTypes.has(type)) {
+            activeLockTypes.delete(type);
+            this.#releaseFile(ctx, type);
+            notify();
+          }
+        }
+      );
+    } finally {
+      stop();
+    }
+  }
+
+  /**
+   * Reads the lock file states (build + watch) to determine what action should be taken for the compiler
+   */
+  static async getCompileAction(ctx: ManifestContext, lockType: LockType | undefined): Promise<LockCompileAction> {
+    let result: LockAction;
+    do {
+      result = 'build';
+      const buildState = await this.#getDetails(ctx, 'build');
+      const watchState = await this.#getDetails(ctx, 'watch');
+      if (watchState.pid) { // Existing watch operation
+        result = await LogUtil.withLogger('lock', log => this.#getWatchAction(ctx, log, lockType, buildState), true, ['watch', `pid=${watchState.pid}`]);
+      } else if (buildState.pid) { // Existing build operation
+        result = await LogUtil.withLogger('lock', log => this.#getBuildAction(ctx, log, lockType), true, ['build', `pid=${buildState.pid}`]);
+      }
+    } while (result === 'retry');
+    return result;
+  }
+}

package/support/log.ts

@@ -0,0 +1,51 @@
+export type CompilerLogEvent = [level: 'info' | 'debug' | 'warn', message: string];
+export type CompilerLogger = (...args: CompilerLogEvent) => void;
+export type WithLogger<T> = (log: CompilerLogger) => Promise<T>;
+
+const SCOPE_MAX = 15;
+
+export class LogUtil {
+
+  static levels: {
+    debug: boolean;
+    info: boolean;
+    warn: boolean;
+  };
+
+  static set level(value: string) {
+    this.levels = {
+      warn: /^(debug|info|warn)$/.test(value),
+      info: /^(debug|info)$/.test(value),
+      debug: /^debug$/.test(value),
+    };
+  }
+
+  /**
+   * Is object a log event
+   */
+  static isLogEvent = (o: unknown): o is CompilerLogEvent => o !== null && o !== undefined && Array.isArray(o);
+
+  /**
+   * Log message with filtering by level
+   */
+  static log(scope: string, args: string[], ...[level, msg]: CompilerLogEvent): void {
+    const message = msg.replaceAll(process.cwd(), '.');
+    if (LogUtil.levels[level]) {
+      const params = [`[${scope.padEnd(SCOPE_MAX, ' ')}]`, ...args, message];
+      if (!/(0|false|off|no)$/i.test(process.env.TRV_LOG_TIME ?? '')) {
+        params.unshift(new Date().toISOString());
+      }
+      // eslint-disable-next-line no-console
+      console[level]!(...params);
+    }
+  }
+
+  /**
+   * With logger
+   */
+  static withLogger<T>(scope: string, op: WithLogger<T>, basic = true, args: string[] = []): Promise<T> {
+    const log = this.log.bind(null, scope, args);
+    basic && log('debug', 'Started');
+    return op(log).finally(() => basic && log('debug', 'Completed'));
+  }
+}