@travetto/compiler 3.3.2 → 3.4.0-rc.1

package/support/server/server.ts

@@ -0,0 +1,189 @@
+import http from 'http';
+import fs from 'fs/promises';
+import path from 'path';
+
+import type { ManifestContext } from '@travetto/manifest';
+
+import type { CompilerMode, CompilerOp, CompilerServerEvent, CompilerServerEventType, CompilerServerInfo } from '../types';
+import { LogUtil } from '../log';
+import { CompilerClientUtil } from './client';
+import { CommonUtil } from '../util';
+
+const log = LogUtil.log.bind(LogUtil, 'compiler-server');
+
+/**
+ * Compiler Server Class
+ */
+export class CompilerServer {
+
+  #ctx: ManifestContext;
+  #server: http.Server;
+  #listeners: { res: http.ServerResponse, type: CompilerServerEventType }[] = [];
+  #shutdown = new AbortController();
+  signal = this.#shutdown.signal;
+  info: CompilerServerInfo;
+
+  constructor(ctx: ManifestContext, op: CompilerOp) {
+    this.#ctx = ctx;
+    this.info = {
+      state: 'startup',
+      iteration: Date.now(),
+      mode: op === 'run' ? 'build' : op,
+      serverPid: process.pid,
+      compilerPid: -1,
+      path: ctx.workspacePath,
+      url: ctx.compilerUrl
+    };
+
+    this.#server = http.createServer({
+      keepAlive: true,
+      requestTimeout: 1000 * 60 * 60,
+      keepAliveTimeout: 1000 * 60 * 60,
+    }, (req, res) => this.#onRequest(req, res));
+
+    // Connect
+    process.on('SIGINT', () => this.#shutdown.abort());
+  }
+
+  get mode(): CompilerMode {
+    return this.info.mode;
+  }
+
+  isResetEvent(ev: CompilerServerEvent): boolean {
+    return ev.type === 'state' && ev.payload.state === 'reset';
+  }
+
+  async #tryListen(attempt = 0): Promise<'ok' | 'running'> {
+    const output = await new Promise<'ok' | 'running' | 'retry'>((resolve, reject) => {
+      this.#server
+        .on('listening', () => resolve('ok'))
+        .on('error', async err => {
+          if ('code' in err && err.code === 'EADDRINUSE') {
+            const info = await CompilerClientUtil.getServerInfo(this.#ctx);
+            resolve((info && info.mode === 'build' && this.mode === 'watch') ? 'retry' : 'running');
+          } else {
+            reject(err);
+          }
+        });
+
+      const url = new URL(this.#ctx.compilerUrl);
+      setTimeout(() => this.#server.listen(+url.port, url.hostname), 1); // Run async
+    });
+
+    if (output === 'retry') {
+      if (attempt >= 5) {
+        throw new Error('Unable to verify compilation server');
+      }
+      log('info', 'Waiting for build to finish, before retrying');
+      // Let the server finish
+      await CompilerClientUtil.waitForState(this.#ctx, ['close'], this.signal);
+      return this.#tryListen(attempt + 1);
+    }
+
+    return output;
+  }
+
+  async #addListener(type: string, res: http.ServerResponse): Promise<void> {
+    res.writeHead(200);
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    this.#listeners.push({ res, type: type as 'change' });
+    await new Promise(resolve => res.on('close', resolve));
+    this.#listeners.splice(this.#listeners.findIndex(x => x.res === res), 1);
+    res.end();
+  }
+
+  #emitEvent(ev: CompilerServerEvent): void {
+    const msg = `${JSON.stringify(ev.payload)}\n`;
+    for (const el of this.#listeners) {
+      if (!el.res.closed && el.type === ev.type) {
+        el.res.write(msg);
+      }
+    }
+  }
+
+  async #disconnectActive(): Promise<void> {
+    log('info', 'Server disconnect requested');
+    this.info.iteration = Date.now();
+    await new Promise(r => setTimeout(r, 20));
+    this.#server.closeAllConnections(); // Force reconnects
+  }
+
+  async #clean(): Promise<{ clean: boolean }> {
+    await Promise.all([this.#ctx.compilerFolder, this.#ctx.outputFolder]
+      .map(f => fs.rm(path.resolve(this.#ctx.workspacePath, f), { recursive: true, force: true })));
+    return { clean: true };
+  }
+
+  /**
+   * Request handler
+   */
+  async #onRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    res.setHeader('Content-Type', 'application/json');
+
+    const [, action, subAction] = new URL(`${this.#ctx.compilerUrl}${req.url}`).pathname.split('/');
+
+    log('debug', 'Receive request', { action, subAction });
+
+    let out: unknown;
+    switch (action) {
+      case 'event': return await this.#addListener(subAction, res);
+      case 'stop': out = await this.close(); break;
+      case 'clean': out = await this.#clean(); break;
+      case 'info':
+      default: out = this.info ?? {}; break;
+    }
+    res.end(JSON.stringify(out));
+  }
+
+  /**
+   * Process events
+   */
+  async processEvents(src: (signal: AbortSignal) => AsyncIterable<CompilerServerEvent>): Promise<void> {
+
+    CompilerClientUtil.streamLogs(this.#ctx, this.signal); // Send logs to stdout
+
+    for await (const ev of CommonUtil.restartableEvents(src, this.signal, this.isResetEvent)) {
+      this.#emitEvent(ev);
+
+      if (ev.type === 'state') {
+        this.info.state = ev.payload.state;
+        if (ev.payload.state === 'init' && ev.payload.extra && 'pid' in ev.payload.extra && typeof ev.payload.extra.pid === 'number') {
+          this.info.compilerPid = ev.payload.extra.pid;
+        }
+        log('info', `State changed: ${this.info.state}`);
+      }
+      if (this.isResetEvent(ev)) {
+        await this.#disconnectActive();
+      }
+    }
+
+    // Terminate, after letting all remaining events emit
+    setImmediate(() => this.close());
+  }
+
+  /**
+   * Close server
+   */
+  async close(): Promise<unknown> {
+    log('info', 'Closing down server');
+    setTimeout(async () => {
+      this.#shutdown.abort();
+      this.#emitEvent({ type: 'state', payload: { state: 'close' } });
+      this.#server.unref();
+      await new Promise(r => {
+        this.#server.close(r);
+        setImmediate(() => this.#server.closeAllConnections());
+      });
+    }, 10);
+    return { closing: true };
+  }
+
+  /**
+   * Start the server listening
+   */
+  async listen(): Promise<CompilerServer | undefined> {
+    const running = await this.#tryListen() === 'ok';
+    log('info', running ? 'Starting server' : 'Server already running under a different process', this.#ctx.compilerUrl);
+    return running ? this : undefined;
+  }
+}

package/support/setup.ts

@@ -0,0 +1,229 @@
+import path from 'path';
+import fs from 'fs/promises';
+
+import { type DeltaEvent, type ManifestContext, type ManifestRoot, Package } from '@travetto/manifest';
+
+import { LogUtil } from './log';
+import { CommonUtil } from './util';
+
+type ModFile = { input: string, output: string, stale: boolean };
+
+const SOURCE_SEED = ['package.json', 'index.ts', '__index__.ts', 'src', 'support', 'bin'];
+const PRECOMPILE_MODS = ['@travetto/terminal', '@travetto/manifest', '@travetto/transformer', '@travetto/compiler'];
+const RECENT_STAT = (stat: { ctimeMs: number, mtimeMs: number }): number => Math.max(stat.ctimeMs, stat.mtimeMs);
+
+/**
+ * Compiler Setup Utilities
+ */
+export class CompilerSetup {
+
+  /**
+   * Import a compiled manifest
+   */
+  static #importManifest = (ctx: ManifestContext): Promise<typeof import('@travetto/manifest')> =>
+    import(path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/manifest/__index__.js'));
+
+  /**  Convert a file to a given ext */
+  static #sourceToExtension(inputFile: string, ext: string): string {
+    return inputFile.replace(/[.][tj]sx?$/, ext);
+  }
+
+  /**
+   * Get the output file name for a given input
+   */
+  static #sourceToOutputExt(inputFile: string): string {
+    return this.#sourceToExtension(inputFile, '.js');
+  }
+
+  /**
+   * Output a file, support for ts, js, and package.json
+   */
+  static async #transpileFile(ctx: ManifestContext, inputFile: string, outputFile: string): Promise<void> {
+    const type = CommonUtil.getFileType(inputFile);
+    if (type === 'js' || type === 'ts') {
+      const compilerOut = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules');
+
+      const text = (await fs.readFile(inputFile, 'utf8'))
+        .replace(/from '([.][^']+)'/g, (_, i) => `from '${i.replace(/[.]js$/, '')}.js'`)
+        .replace(/from '(@travetto\/(.*?))'/g, (_, i, s) => `from '${path.resolve(compilerOut, `${i}${s.includes('/') ? '.js' : '/__index__.js'}`)}'`);
+
+      const ts = (await import('typescript')).default;
+      const content = ts.transpile(text, {
+        ...await CommonUtil.getCompilerOptions(ctx),
+        sourceMap: false,
+        inlineSourceMap: true,
+      }, inputFile);
+      await CommonUtil.writeTextFile(outputFile, content);
+    } else if (type === 'package-json') {
+      const pkg: Package = JSON.parse(await fs.readFile(inputFile, 'utf8'));
+      const main = pkg.main ? this.#sourceToOutputExt(pkg.main) : undefined;
+      const files = pkg.files?.map(x => this.#sourceToOutputExt(x));
+
+      const content = JSON.stringify({ ...pkg, main, type: ctx.moduleType, files }, null, 2);
+      await CommonUtil.writeTextFile(outputFile, content);
+    }
+  }
+
+  /**
+   * Scan directory to find all project sources for comparison
+   */
+  static async #getModuleSources(ctx: ManifestContext, module: string, seed: string[]): Promise<ModFile[]> {
+    const inputFolder = (ctx.mainModule === module) ?
+      process.cwd() :
+      CommonUtil.resolveModuleFolder(module);
+
+    const folders = seed.filter(x => !/[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+    const files = seed.filter(x => /[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+
+    while (folders.length) {
+      const sub = folders.pop();
+      if (!sub) {
+        continue;
+      }
+
+      for (const file of await fs.readdir(sub).catch(() => [])) {
+        if (file.startsWith('.')) {
+          continue;
+        }
+        const resolvedInput = path.resolve(sub, file);
+        const stat = await fs.stat(resolvedInput);
+
+        if (stat.isDirectory()) {
+          folders.push(resolvedInput);
+        } else {
+          switch (CommonUtil.getFileType(file)) {
+            case 'js': case 'ts': files.push(resolvedInput);
+          }
+        }
+      }
+    }
+
+    const outputFolder = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', module);
+    const out: ModFile[] = [];
+    for (const input of files) {
+      const output = this.#sourceToOutputExt(input.replace(inputFolder, outputFolder));
+      const inputTs = await fs.stat(input).then(RECENT_STAT, () => 0);
+      if (inputTs) {
+        const outputTs = await fs.stat(output).then(RECENT_STAT, () => 0);
+        await fs.mkdir(path.dirname(output), { recursive: true, });
+        out.push({ input, output, stale: inputTs > outputTs });
+      }
+    }
+
+    return out;
+  }
+
+  /**
+   * Recompile folder if stale
+   */
+  static async #compileIfStale(ctx: ManifestContext, scope: string, mod: string, seed: string[]): Promise<string[]> {
+    const files = await this.#getModuleSources(ctx, mod, seed);
+    const changes = files.filter(x => x.stale).map(x => x.input);
+    const out: string[] = [];
+
+    try {
+      await LogUtil.withLogger(scope, async log => {
+        if (files.some(f => f.stale)) {
+          log('debug', 'Starting', mod);
+          for (const file of files.filter(x => x.stale)) {
+            await this.#transpileFile(ctx, file.input, file.output);
+          }
+          if (changes.length) {
+            out.push(...changes.map(x => `${mod}/${x}`));
+            log('debug', `Source changed: ${changes.join(', ')}`, mod);
+          }
+          log('debug', 'Completed', mod);
+        } else {
+          log('debug', 'Skipped', mod);
+        }
+      }, false);
+    } catch (err) {
+      console.error(err);
+    }
+    return out;
+  }
+
+  /**
+   * Export manifest
+   */
+  static async exportManifest(ctx: ManifestContext, output?: string, env = 'dev'): Promise<void> {
+    const { ManifestUtil } = await this.#importManifest(ctx);
+    let manifest = await ManifestUtil.buildManifest(ctx);
+
+    // If in prod mode, only include std modules
+    if (/^prod/i.test(env)) {
+      manifest = ManifestUtil.createProductionManifest(manifest);
+    }
+    if (output) {
+      output = await ManifestUtil.writeManifestToFile(output, manifest);
+    } else {
+      console.log(JSON.stringify(manifest, null, 2));
+    }
+  }
+
+  /**
+   * Sets up compiler, and produces a manifest and set of changes that need to be processed
+   */
+  static async setup(ctx: ManifestContext): Promise<{ manifest: ManifestRoot, changed: DeltaEvent[] }> {
+    let changes = 0;
+
+    await LogUtil.withLogger('precompile', async () => {
+      for (const mod of PRECOMPILE_MODS) {
+        const count = (await this.#compileIfStale(ctx, 'precompile', mod, SOURCE_SEED)).length;
+        if (mod !== '@travetto/terminal') {
+          changes += count;
+        }
+      }
+    });
+
+    const { ManifestUtil, ManifestDeltaUtil, PackageUtil } = await this.#importManifest(ctx);
+
+    PackageUtil.clearCache();
+
+    const manifest = await LogUtil.withLogger('manifest', () => ManifestUtil.buildManifest(ctx));
+
+    await LogUtil.withLogger('transformers', async () => {
+      for (const mod of Object.values(manifest.modules).filter(m => m.files.$transformer?.length)) {
+        changes += (await this.#compileIfStale(ctx, 'transformers', mod.name, ['package.json', ...mod.files.$transformer!.map(x => x[0])])).length;
+      }
+    });
+
+    const delta = await LogUtil.withLogger('delta', async log => {
+      if (changes) {
+        log('debug', 'Skipping, everything changed');
+        return [{ type: 'changed', file: '*', module: ctx.mainModule } as const];
+      } else {
+        return ManifestDeltaUtil.produceDelta(ctx, manifest);
+      }
+    });
+
+    if (changes) {
+      await LogUtil.withLogger('reset', async log => {
+        await fs.rm(path.resolve(ctx.workspacePath, ctx.outputFolder), { recursive: true, force: true });
+        log('info', 'Clearing output due to compiler changes');
+      }, false);
+    }
+
+    // Write manifest
+    await LogUtil.withLogger('manifest', async log => {
+      await ManifestUtil.writeManifest(ctx, manifest);
+      log('debug', `Wrote manifest ${ctx.mainModule}`);
+
+      // Update all manifests
+      if (delta.length && ctx.monoRepo && !ctx.mainFolder) {
+        const names: string[] = [];
+        const mods = Object.values(manifest.modules).filter(x => x.local && x.name !== ctx.mainModule);
+        for (const mod of mods) {
+          await ManifestUtil.rewriteManifest(path.resolve(ctx.workspacePath, mod.sourceFolder));
+          names.push(mod.name);
+        }
+        log('debug', `Changes triggered ${delta.slice(0, 10).map(x => `${x.type}:${x.module}:${x.file}`)}`);
+        log('debug', `Rewrote monorepo manifests [changes=${delta.length}] ${names.slice(0, 10).join(', ')}`);
+      }
+    });
+
+    const changed = delta.filter(x => x.type === 'added' || x.type === 'changed');
+
+    return { manifest, changed };
+  }
+}

package/support/types.ts

@@ -0,0 +1,27 @@
+export type CompilerMode = 'build' | 'watch';
+export type CompilerOp = CompilerMode | 'run';
+
+export type CompilerStateType = 'startup' | 'init' | 'compile-start' | 'compile-end' | 'watch-start' | 'watch-end' | 'reset' | 'close';
+export type CompilerChangeEvent = { file: string, action: 'create' | 'update' | 'delete', folder: string, output: string, module: string, time: number };
+export type CompilerLogLevel = 'info' | 'debug' | 'warn' | 'error';
+export type CompilerLogEvent = { level: CompilerLogLevel, message: string, time: number, args?: unknown[], scope?: string };
+export type CompilerProgressEvent = { idx: number, total: number, message: string, operation: 'compile', complete?: boolean };
+export type CompilerStateEvent = { state: CompilerStateType, extra?: Record<string, unknown> };
+
+export type CompilerServerEvent =
+  { type: 'change', payload: CompilerChangeEvent } |
+  { type: 'log', payload: CompilerLogEvent } |
+  { type: 'progress', payload: CompilerProgressEvent } |
+  { type: 'state', payload: CompilerStateEvent };
+
+export type CompilerServerEventType = CompilerServerEvent['type'];
+
+export type CompilerServerInfo = {
+  path: string;
+  serverPid: number;
+  compilerPid: number;
+  state: CompilerStateType;
+  mode: CompilerMode,
+  iteration: number;
+  url: string;
+};

package/support/util.ts

@@ -0,0 +1,108 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { createRequire } from 'module';
+
+import type { ManifestContext } from '@travetto/manifest';
+
+import { LogUtil } from './log';
+
+const OPT_CACHE: Record<string, import('typescript').CompilerOptions> = {};
+const SRC_REQ = createRequire(path.resolve('node_modules'));
+
+export class CommonUtil {
+  /**
+   * Returns the compiler options
+   */
+  static async getCompilerOptions(ctx: ManifestContext): Promise<{}> {
+    if (!(ctx.workspacePath in OPT_CACHE)) {
+      let tsconfig = path.resolve(ctx.workspacePath, 'tsconfig.json');
+
+      if (!await fs.stat(tsconfig).then(_ => true, _ => false)) {
+        tsconfig = SRC_REQ.resolve('@travetto/compiler/tsconfig.trv.json');
+      }
+
+      const ts = (await import('typescript')).default;
+
+      const { options } = ts.parseJsonSourceFileConfigFileContent(
+        ts.readJsonConfigFile(tsconfig, ts.sys.readFile), ts.sys, ctx.workspacePath
+      );
+
+      OPT_CACHE[ctx.workspacePath] = {
+        ...options,
+        allowJs: true,
+        resolveJsonModule: true,
+        sourceRoot: ctx.workspacePath,
+        rootDir: ctx.workspacePath,
+        outDir: path.resolve(ctx.workspacePath),
+        module: ctx.moduleType === 'commonjs' ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext,
+      };
+    }
+    return OPT_CACHE[ctx.workspacePath];
+  }
+
+  /**
+   * Resolve module location
+   */
+  static resolveModuleFolder(mod: string): string {
+    return path.dirname(SRC_REQ.resolve(`${mod}/package.json`));
+  }
+
+  /**
+   * Determine file type
+   */
+  static getFileType(file: string): 'ts' | 'js' | 'package-json' | 'typings' | undefined {
+    return file.endsWith('package.json') ? 'package-json' :
+      (file.endsWith('.js') ? 'js' :
+        (file.endsWith('.d.ts') ? 'typings' : (/[.]tsx?$/.test(file) ? 'ts' : undefined)));
+  }
+
+  /**
+   * Write text file, and ensure folder exists
+   */
+  static writeTextFile = (file: string, content: string): Promise<void> =>
+    fs.mkdir(path.dirname(file), { recursive: true }).then(() => fs.writeFile(file, content, 'utf8'));
+
+  /**
+   * Restartable Event Stream
+   */
+  static async * restartableEvents<T>(src: (signal: AbortSignal) => AsyncIterable<T>, parent: AbortSignal, shouldRestart: (item: T) => boolean): AsyncIterable<T> {
+    outer: while (!parent.aborted) {
+      const controller = new AbortController();
+      // Chain
+      parent.addEventListener('abort', () => controller.abort());
+
+      const comp = src(controller.signal);
+
+      LogUtil.log('event-stream', 'debug', 'Started event stream');
+
+      // Wait for all events, close at the end
+      for await (const ev of comp) {
+        yield ev;
+        if (shouldRestart(ev)) {
+          controller.abort(); // Ensure terminated of process
+          continue outer;
+        }
+      }
+
+      LogUtil.log('event-stream', 'debug', 'Finished event stream');
+
+      // Natural exit, we done
+      if (!controller.signal.aborted) { // Shutdown source if still running
+        controller.abort();
+      }
+      return;
+    }
+  }
+
+  /**
+   * Create a module loader given a context, and assuming build is complete
+   * @param ctx
+   */
+  static moduleLoader(ctx: ManifestContext): (mod: string) => Promise<unknown> {
+    return (mod) => {
+      const outputRoot = path.resolve(ctx.workspacePath, ctx.outputFolder);
+      process.env.TRV_MANIFEST = path.resolve(outputRoot, 'node_modules', ctx.mainModule); // Setup for running
+      return import(path.join(outputRoot, 'node_modules', mod)); // Return function to run import on a module
+    };
+  }
+}

package/tsconfig.trv.json

@@ -24,8 +24,7 @@
   "watchOptions": {
     "excludeDirectories": [
       "**/node_modules",
-      "**/.trv_output",
-      "**/.trv_compiler",
+      "**/.trv",
       "**/.git",
     ]
   }