@travetto/compiler 3.4.4 → 4.0.0-rc.1

package/support/log.ts

@@ -1,9 +1,11 @@
 import type { ManifestContext } from '@travetto/manifest';
-import type { CompilerLogEvent, CompilerLogLevel } from './types';
+import type { CompilerLogEvent, CompilerLogLevel, CompilerProgressEvent } from './types';
 
 export type CompilerLogger = (level: CompilerLogLevel, message: string, ...args: unknown[]) => void;
 export type WithLogger<T> = (log: CompilerLogger) => Promise<T>;
 
+type ProgressWriter = (ev: CompilerProgressEvent) => (unknown | Promise<unknown>);
+
 const LEVEL_TO_PRI: Record<CompilerLogLevel, number> = { debug: 1, info: 2, warn: 3, error: 4 };
 
 const SCOPE_MAX = 15;
@@ -14,16 +16,37 @@ export class LogUtil {
 
   static logLevel: CompilerLogLevel = 'error';
 
+  static logProgress?: ProgressWriter;
+
+  static linePartial = false;
+
+  static #rewriteLine(text: string): Promise<void> | void {
+    // Move to 1st position, and clear after text
+    const done = process.stdout.write(`\x1b[1G${text}\x1b[0K`);
+    this.linePartial = !!text;
+    if (!done) {
+      return new Promise<void>(r => process.stdout.once('drain', r));
+    }
+  }
+
+
   /**
    * Set level for operation
    */
   static initLogs(ctx: ManifestContext, defaultLevel: CompilerLogLevel): void {
-    // Listen only if we aren't in quiet
-    if ((process.env.TRV_BUILD || !process.env.TRV_QUIET)) {
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      this.logLevel = (process.env.TRV_BUILD as 'debug') || defaultLevel;
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const build = process.env.TRV_BUILD as CompilerLogLevel | 'none';
+    if (build !== 'none' && process.env.TRV_QUIET !== 'true') {
+      this.logLevel = build || defaultLevel;
     }
-    this.root = ctx.workspacePath;
+    this.root = ctx.workspace.path;
+    this.logProgress = (this.isLevelActive('info') && process.stdout.isTTY) ? this.#logProgressEvent : undefined;
+  }
+
+  static #logProgressEvent(ev: CompilerProgressEvent): Promise<void> | void {
+    const pct = Math.trunc(ev.idx * 100 / ev.total);
+    const text = ev.complete ? '' : `Compiling [${'#'.repeat(Math.trunc(pct / 10)).padEnd(10, ' ')}] [${ev.idx}/${ev.total}] ${ev.message}`;
+    return this.#rewriteLine(text);
   }
 
   /**
@@ -34,36 +57,44 @@ export class LogUtil {
   }
 
   /**
-   * Log message with filtering by level
+   * Log event with filtering by level
    */
-  static log(scope: string, level: CompilerLogLevel, message: string, ...args: unknown[]): void {
-    LogUtil.sendLogEventToConsole({ level, scope, message, args, time: Date.now() });
+  static logEvent(ev: CompilerLogEvent): void {
+    if (this.isLevelActive(ev.level)) {
+      const params = [ev.message, ...ev.args ?? []].map(x => typeof x === 'string' ? x.replaceAll(this.root, '.') : x);
+      if (ev.scope) {
+        params.unshift(`[${ev.scope.padEnd(SCOPE_MAX, ' ')}]`);
+      }
+      params.unshift(new Date().toISOString(), `${ev.level.padEnd(5)}`);
+      if (this.linePartial) {
+        this.#rewriteLine(''); // Clear out progress line
+      }
+      // eslint-disable-next-line no-console
+      console[ev.level]!(...params);
+    }
   }
 
   /**
    * With logger
    */
   static withLogger<T>(scope: string, op: WithLogger<T>, basic = true): Promise<T> {
-    const log = this.log.bind(null, scope);
+    const log = this.logger(scope);
     basic && log('debug', 'Started');
     return op(log).finally(() => basic && log('debug', 'Completed'));
   }
 
   /**
-   * Compiler log event to console
+   * With scope
    */
-  static sendLogEventToConsole(ev: CompilerLogEvent): void {
-    if (this.isLevelActive(ev.level)) {
-      const params = [ev.message, ...ev.args ?? []].map(x => typeof x === 'string' ? x.replaceAll(LogUtil.root, '.') : x);
-      if (ev.scope) {
-        params.unshift(`[${ev.scope.padEnd(SCOPE_MAX, ' ')}]`);
-      }
-      params.unshift(`${ev.level.padEnd(5)}`);
-      if (!/(0|false|off|no)$/i.test(process.env.TRV_LOG_TIME ?? '')) {
-        params.unshift(new Date().toISOString());
-      }
-      // eslint-disable-next-line no-console
-      console[ev.level]!(...params);
-    }
+  static logger(scope: string): CompilerLogger {
+    return (level, message, ...args) => this.logEvent({ scope, message, level, args, time: Date.now() });
+  }
+
+  /**
+   * Write all progress events if active
+   */
+  static async consumeProgressEvents(src: () => AsyncIterable<CompilerProgressEvent>): Promise<void> {
+    if (!this.logProgress) { return; }
+    for await (const item of src()) { await this.logProgress?.(item); }
   }
 }

package/support/queue.ts

@@ -13,6 +13,9 @@ export class AsyncQueue<X> implements AsyncIterator<X>, AsyncIterable<X> {
 
   constructor(signal?: AbortSignal) {
     signal?.addEventListener('abort', () => this.close());
+    if (signal?.aborted) {
+      this.close();
+    }
   }
 
   [Symbol.asyncIterator](): AsyncIterator<X> { return this; }
@@ -25,12 +28,14 @@ export class AsyncQueue<X> implements AsyncIterator<X>, AsyncIterable<X> {
     return { value: (this.#queue.length ? this.#queue.shift() : undefined)!, done: this.#done };
   }
 
-  add(item: X, immediate = false): void {
-    if (!immediate) {
-      this.#queue.push(item);
-    } else {
-      this.#queue.unshift(item);
-    }
+  async throw(e?: Error): Promise<IteratorResult<X>> {
+    this.#done = true;
+    this.#ready.reject(e);
+    return { value: undefined, done: this.#done };
+  }
+
+  add(item: X): void {
+    this.#queue.push(item);
     this.#ready.resolve();
   }
 

package/support/server/client.ts

@@ -1,118 +1,159 @@
-import rl from 'readline/promises';
-import { Readable } from 'stream';
+import rl from 'node:readline/promises';
+import timers from 'node:timers/promises';
+import { Readable } from 'node:stream';
 
-import type { ManifestContext } from '@travetto/manifest';
-import type { CompilerServerEvent, CompilerServerEventType, CompilerServerInfo, CompilerStateType } from '../types';
+import { ManifestContext } from '@travetto/manifest';
 
-import { LogUtil } from '../log';
+import type { CompilerEvent, CompilerEventType, CompilerServerInfo, CompilerStateType } from '../types';
+import type { CompilerLogger } from '../log';
+import { ProcessHandle } from './process-handle';
 
-declare global {
-  interface RequestInit { timeout?: number }
-}
-
-const log = LogUtil.log.bind(LogUtil, 'compiler-client');
-
-function getSignal(input?: AbortSignal): AbortSignal {
-  // Ensure we capture end of process at least
-  if (!input) {
-    const ctrl = new AbortController();
-    process.on('SIGINT', () => ctrl.abort());
-    input = ctrl.signal;
-  }
-  return input;
-}
+type FetchEventsConfig<T> = {
+  signal?: AbortSignal;
+  until?: (ev: T) => boolean;
+  enforceIteration?: boolean;
+};
 
 /**
  * Compiler Client Operations
  */
-export class CompilerClientUtil {
+export class CompilerClient {
+
+  #url: string;
+  #log: CompilerLogger;
+  #handle: Record<'compiler' | 'server', ProcessHandle>;
+
+  constructor(ctx: ManifestContext, log: CompilerLogger) {
+    this.#url = ctx.build.compilerUrl.replace('localhost', '127.0.0.1');
+    this.#log = log;
+    this.#handle = { compiler: new ProcessHandle(ctx, 'compiler'), server: new ProcessHandle(ctx, 'server') };
+  }
+
+  toString(): string {
+    return `[${this.constructor.name} url=${this.#url}]`;
+  }
+
+  get url(): string {
+    return this.#url;
+  }
 
-  /**
-   * Get server information, if server is running
-   */
-  static async getServerInfo(ctx: ManifestContext): Promise<CompilerServerInfo | undefined> {
+  async #fetch(rel: string, opts?: RequestInit & { timeout?: number }): Promise<Response> {
+    const ctrl = new AbortController();
+    opts?.signal?.addEventListener('abort', () => ctrl.abort());
+    const timeoutId = setTimeout(() => {
+      this.#log('error', `Timeout on request to ${this.#url}${rel}`);
+      ctrl.abort('TIMEOUT');
+    }, 100).unref();
     try {
+      return await fetch(`${this.#url}${rel}`, { ...opts, signal: ctrl.signal });
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  }
+
+  /** Get server information, if server is running */
+  info(): Promise<CompilerServerInfo | undefined> {
+    return this.#fetch('/info').then(v => v.json(), () => undefined)
       // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      const res = (await fetch(`${ctx.compilerUrl}/info`).then(v => v.json())) as CompilerServerInfo;
-      return res;
-    } catch (err) {
-      return undefined;
+      .then(v => v as CompilerServerInfo);
+  }
+
+  async isWatching(): Promise<boolean> {
+    return (await this.info())?.state === 'watch-start';
+  }
+
+  /** Clean the server */
+  clean(): Promise<boolean> {
+    return this.#fetch('/clean').then(v => v.ok, () => false);
+  }
+
+  /** Stop server and wait for shutdown */
+  async stop(): Promise<boolean> {
+    const info = await this.info();
+    if (!info) {
+      this.#log('debug', 'Stopping server, info not found, manual killing');
+      return Promise.all([this.#handle.server.kill(), this.#handle.compiler.kill()])
+        .then(v => v.some(x => x));
     }
+
+    await this.#fetch('/stop').catch(() => { }); // Trigger
+    this.#log('debug', 'Waiting for compiler to exit');
+    await this.#handle.compiler.ensureKilled();
+    return true;
   }
 
-  /**
-   * Fetch compiler events
-   */
-  static async * fetchEvents<
-    V extends CompilerServerEventType,
-    T extends (CompilerServerEvent & { type: V })['payload']
-  >(ctx: ManifestContext, type: V, signal?: AbortSignal, until?: (ev: T) => boolean): AsyncIterable<T> {
-    log('debug', `Starting watch for events of type "${type}"`);
+  /** Fetch compiler events */
+  async * fetchEvents<
+    V extends CompilerEventType,
+    T extends (CompilerEvent & { type: V })['payload']
+  >(type: V, cfg: FetchEventsConfig<T> = {}): AsyncIterable<T> {
+    let info = await this.info();
+    if (!info) {
+      return;
+    }
+
+    this.#log('debug', `Starting watch for events of type "${type}"`);
 
-    signal = getSignal(signal);
+    let signal = cfg.signal;
+
+    // Ensure we capture end of process at least
+    if (!signal) {
+      const ctrl = new AbortController();
+      process.on('SIGINT', () => ctrl.abort());
+      signal = ctrl.signal;
+    }
+
+    const { iteration } = info;
 
     for (; ;) {
       const ctrl = new AbortController();
+      const quit = (): void => ctrl.abort();
       try {
-        signal.addEventListener('abort', () => ctrl.abort());
-        const stream = await fetch(`${ctx.compilerUrl}/event/${type}`, {
-          signal: ctrl.signal,
-          timeout: 1000 * 60 * 60
-        });
+        signal.addEventListener('abort', quit);
+        const stream = await this.#fetch(`/event/${type}`, { signal: ctrl.signal, keepalive: true });
+
         for await (const line of rl.createInterface(Readable.fromWeb(stream.body!))) {
           if (line.trim().charAt(0) === '{') {
             const val = JSON.parse(line);
-            if (until?.(val)) {
-              setTimeout(() => ctrl.abort(), 1);
+            if (cfg.until?.(val)) {
+              await timers.setTimeout(1);
+              ctrl.abort();
             }
             yield val;
           }
         }
-      } catch (err) { }
+      } catch (err) {
+        if (!ctrl.signal.aborted) { throw err; }
+      }
+      signal.removeEventListener('abort', quit);
+
+      await timers.setTimeout(1);
+
+      info = await this.info();
 
-      if (ctrl.signal.aborted || !(await this.getServerInfo(ctx))) { // If health check fails, or aborted
-        log('debug', `Stopping watch for events of type "${type}"`);
+      if (ctrl.signal.reason === 'TIMEOUT') {
+        this.#log('debug', 'Failed due to timeout');
+        return;
+      }
+
+      if (ctrl.signal.aborted || !info || (cfg.enforceIteration && info.iteration !== iteration)) { // If health check fails, or aborted
+        this.#log('debug', `Stopping watch for events of type "${type}"`);
         return;
       } else {
-        log('debug', `Restarting watch for events of type "${type}"`);
+        this.#log('debug', `Restarting watch for events of type "${type}"`);
       }
     }
   }
 
-  /**
-   * Wait for one of N states to be achieved
-   */
-  static async waitForState(ctx: ManifestContext, states: CompilerStateType[], signal?: AbortSignal): Promise<void> {
+  /** Wait for one of N states to be achieved */
+  async waitForState(states: CompilerStateType[], message?: string, signal?: AbortSignal): Promise<void> {
     const set = new Set(states);
-    const existing = await this.getServerInfo(ctx);
-    log('debug', `Existing: ${JSON.stringify(existing)}`);
-    if (existing && set.has(existing.state)) {
-      log('debug', `Waited for state, ${existing.state} in server info`);
-      return;
-    }
     // Loop until
-    log('debug', `Waiting for states, ${states.join(', ')}`);
-    for await (const _ of this.fetchEvents(ctx, 'state', signal, s => set.has(s.state))) { }
-    log('debug', `Found state, one of ${states.join(', ')}`);
-  }
-
-  /**
-   * Stream logs
-   */
-  static async streamLogs(ctx: ManifestContext, signal?: AbortSignal): Promise<void> {
-    if (!LogUtil.logLevel) {
-      return;
-    }
-    for await (const ev of this.fetchEvents(ctx, 'log', signal!)) {
-      LogUtil.sendLogEventToConsole(ev);
+    this.#log('debug', `Waiting for states, ${states.join(', ')}`);
+    for await (const _ of this.fetchEvents('state', { signal, until: s => set.has(s.state) })) { }
+    this.#log('debug', `Found state, one of ${states.join(', ')} `);
+    if (message) {
+      this.#log('info', message);
     }
   }
-
-  /**
-   * Wait for build
-   */
-  static async waitForBuild(ctx: ManifestContext, signal?: AbortSignal): Promise<void> {
-    await this.waitForState(ctx, ['compile-end', 'watch-start'], signal);
-    log('info', 'Successfully built');
-  }
 }

package/support/server/process-handle.ts

@@ -0,0 +1,57 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import timers from 'node:timers/promises';
+
+import type { ManifestContext } from '@travetto/manifest';
+
+export class ProcessHandle {
+
+  #file: string;
+
+  constructor(ctx: ManifestContext, name: string) {
+    this.#file = path.resolve(ctx.workspace.path, ctx.build.toolFolder, `${name}.pid`);
+  }
+
+  writePid(pid: number): Promise<void> {
+    return fs.writeFile(this.#file, JSON.stringify(pid), 'utf8');
+  }
+
+  getPid(): Promise<number | undefined> {
+    return fs.readFile(this.#file, 'utf8').then(v => +v, () => undefined);
+  }
+
+  async isRunning(): Promise<boolean> {
+    const pid = await this.getPid();
+    if (!pid) { return false; }
+    try {
+      process.kill(pid, 0); // See if process is still running
+      return false;
+    } catch { }
+    return true; // Still running
+  }
+
+  async kill(): Promise<boolean> {
+    const pid = await this.getPid();
+    if (pid && await this.isRunning()) {
+      try {
+        return process.kill(pid);
+      } catch { }
+    }
+    return false;
+  }
+
+  async ensureKilled(gracePeriod: number = 3000): Promise<boolean> {
+    const start = Date.now();
+    const pid = await this.getPid();
+    while (pid && (Date.now() - start) < gracePeriod) { // Ensure its done
+      if (!await this.isRunning()) {
+        return true;
+      }
+      await timers.setTimeout(100);
+    }
+    try {
+      pid && process.kill(pid); // Force kill
+    } catch { }
+    return pid !== undefined;
+  }
+}

package/support/server/runner.ts

@@ -1,42 +1,32 @@
-import cp from 'child_process';
-import path from 'path';
+import cp from 'node:child_process';
+import path from 'node:path';
+import { rmSync } from 'node:fs';
 
-import type { ManifestContext, ManifestRoot, DeltaEvent } from '@travetto/manifest';
+import type { ManifestContext, DeltaEvent } from '@travetto/manifest';
 
-import type { CompilerOp, CompilerProgressEvent, CompilerServerEvent } from '../types';
+import type { CompilerEvent, CompilerMode } from '../types';
 import { AsyncQueue } from '../queue';
 import { LogUtil } from '../log';
 import { CommonUtil } from '../util';
-import { CompilerClientUtil } from './client';
 
-const log = LogUtil.log.bind(null, 'compiler-exec');
+const log = LogUtil.logger('compiler-exec');
+const isEvent = (msg: unknown): msg is CompilerEvent => !!msg && typeof msg === 'object' && 'type' in msg;
 
 /**
  * Running the compiler
  */
 export class CompilerRunner {
 
-  /**
-   * Track compiler progress
-   */
-  static async trackProgress(ctx: ManifestContext, src: AsyncIterable<CompilerProgressEvent>): Promise<void> {
-    const compiler = path.resolve(ctx.workspacePath, ctx.compilerFolder);
-    const main = path.resolve(compiler, 'node_modules', '@travetto/terminal/__index__.js');
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    const { GlobalTerminal } = await import(main) as typeof import('@travetto/terminal');
-
-    await GlobalTerminal.init();
-    await GlobalTerminal.trackProgress(src,
-      x => ({ ...x, text: `Compiling [%idx/%total] -- ${x.message}` }),
-      { position: 'bottom', minDelay: 50, staticMessage: 'Compiling' }
-    );
-  }
-
   /**
    * Run compile process
    */
-  static async* runProcess(ctx: ManifestContext, manifest: ManifestRoot, changed: DeltaEvent[], op: CompilerOp, signal: AbortSignal): AsyncIterable<CompilerServerEvent> {
-    const watch = op === 'watch';
+  static async * runProcess(ctx: ManifestContext, changed: DeltaEvent[], mode: CompilerMode, signal: AbortSignal): AsyncIterable<CompilerEvent> {
+    if (signal.aborted) {
+      log('debug', 'Skipping, shutting down');
+      return;
+    }
+
+    const watch = mode === 'watch';
     if (!changed.length && !watch) {
       yield { type: 'state', payload: { state: 'compile-end' } };
       log('debug', 'Skipped');
@@ -45,63 +35,46 @@ export class CompilerRunner {
       log('debug', `Started watch=${watch} changed=${changed.slice(0, 10).map(x => `${x.module}/${x.file}`)}`);
     }
 
-    // Track progress if not in run mode
-    if (op !== 'run') {
-      this.trackProgress(ctx, CompilerClientUtil.fetchEvents(ctx, 'progress', signal, ev => !!ev.complete));
-    }
-
-    const compiler = path.resolve(ctx.workspacePath, ctx.compilerFolder);
-    const main = path.resolve(compiler, 'node_modules', '@travetto/compiler/support/entry.compiler.js');
-    const deltaFile = path.resolve(ctx.workspacePath, ctx.toolFolder, 'manifest-delta.json');
-
-    const changedFiles = changed[0]?.file === '*' ? ['*'] : changed.map(ev =>
-      path.resolve(manifest.workspacePath, manifest.modules[ev.module].sourceFolder, ev.file)
-    );
+    const main = path.resolve(ctx.workspace.path, ctx.build.compilerFolder, 'node_modules', '@travetto/compiler/support/entry.compiler.js');
+    const deltaFile = path.resolve(ctx.workspace.path, ctx.build.compilerFolder, `manifest-delta-${Date.now()}.json`);
 
-    let proc: cp.ChildProcess | undefined;
-    let kill: (() => void) | undefined;
+    const changedFiles = changed[0]?.file === '*' ? ['*'] : changed.map(ev => ev.sourceFile);
 
-    const queue = new AsyncQueue<CompilerServerEvent>(signal);
+    const queue = new AsyncQueue<CompilerEvent>();
 
     try {
       await CommonUtil.writeTextFile(deltaFile, changedFiles.join('\n'));
 
       log('info', 'Launching compiler');
-      proc = cp.spawn(process.argv0, [main, deltaFile, `${watch}`], {
+      const proc = cp.spawn(process.argv0, [main, deltaFile, `${watch}`], {
         env: {
           ...process.env,
-          TRV_MANIFEST: path.resolve(ctx.workspacePath, ctx.outputFolder, 'node_modules', ctx.mainModule),
+          TRV_MANIFEST: path.resolve(ctx.workspace.path, ctx.build.outputFolder, 'node_modules', ctx.workspace.name),
         },
-        stdio: ['pipe', 'pipe', 2, 'ipc'],
+        detached: true,
+        stdio: ['pipe', 1, 2, 'ipc'],
       })
-        .on('message', msg => {
-          if (msg && typeof msg === 'object' && 'type' in msg) {
-            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-            queue.add(msg as CompilerServerEvent);
-          }
-        })
-        .on('exit', code => {
-          if (code !== null && code > 0) {
-            log('error', 'Failed during compilation');
-          }
-          queue.close();
-        });
-
-      kill = (): void => { proc?.kill('SIGKILL'); };
+        .on('message', msg => isEvent(msg) && queue.add(msg))
+        .on('exit', () => queue.close());
+
+      const kill = (): unknown => {
+        log('debug', 'Shutting down process');
+        return (proc.connected ? proc.send('shutdown', (err) => proc.kill()) : proc.kill());
+      };
+
+      process.once('SIGINT', kill);
       signal.addEventListener('abort', kill);
-      process.on('exit', kill);
 
       yield* queue;
 
-      log('debug', `exit code: ${proc?.exitCode}`);
+      if (proc.exitCode !== 0) {
+        log('error', `Terminated during compilation, code=${proc.exitCode}, killed=${proc.killed}`);
+      }
+      process.off('SIGINT', kill);
+
       log('debug', 'Finished');
     } finally {
-      if (proc?.killed === false) {
-        proc.kill('SIGKILL');
-      }
-      if (kill) {
-        process.off('exit', kill);
-      }
+      rmSync(deltaFile, { force: true });
     }
   }
 }