@travetto/compiler 3.4.2 → 4.0.0-rc.0

package/src/watch.ts

@@ -1,11 +1,8 @@
-import { readFileSync } from 'fs';
-import { setMaxListeners } from 'events';
+import { readFileSync } from 'node:fs';
 
 import {
-  ManifestContext, ManifestModuleUtil, ManifestUtil, ManifestModuleFolderType, ManifestModuleFileType,
-  path, ManifestModule,
+  ManifestModuleUtil, ManifestUtil, ManifestModuleFolderType, ManifestModuleFileType, path, ManifestModule,
 } from '@travetto/manifest';
-import { getManifestContext } from '@travetto/manifest/bin/context';
 
 import type { CompileStateEntry } from './types';
 import { CompilerState } from './state';
@@ -20,22 +17,14 @@ type DirtyFile = { modFolder: string, mod: string, remove?: boolean, moduleFile:
  */
 export class CompilerWatcher {
 
-  /**
-   * Watch state
-   * @param state
-   * @returns
-   */
-  static watch(state: CompilerState): AsyncIterable<WatchEvent<{ entry: CompileStateEntry }>> {
-    return new CompilerWatcher(state).watchChanges();
-  }
-
   #sourceHashes = new Map<string, number>();
-  #manifestContexts = new Map<string, ManifestContext>();
   #dirtyFiles: DirtyFile[] = [];
   #state: CompilerState;
+  #signal: AbortSignal;
 
-  constructor(state: CompilerState) {
+  constructor(state: CompilerState, signal: AbortSignal) {
     this.#state = state;
+    this.#signal = signal;
   }
 
   async #rebuildManifestsIfNeeded(): Promise<void> {
@@ -43,13 +32,9 @@ export class CompilerWatcher {
       return;
     }
     const mods = [...new Set(this.#dirtyFiles.map(x => x.modFolder))];
-    const contexts = await Promise.all(mods.map(async folder => {
-      if (!this.#manifestContexts.has(folder)) {
-        const ctx = await getManifestContext(folder);
-        this.#manifestContexts.set(folder, ctx);
-      }
-      return this.#manifestContexts.get(folder)!;
-    }));
+    const contexts = await Promise.all(mods.map(folder =>
+      ManifestUtil.getModuleContext(this.#state.manifest, folder)
+    ));
 
     const files = this.#dirtyFiles.slice(0);
     this.#dirtyFiles = [];
@@ -72,7 +57,7 @@ export class CompilerWatcher {
           }
         }
       }
-      await ManifestUtil.writeManifest(ctx, newManifest);
+      await ManifestUtil.writeManifest(newManifest);
     }
     // Reindex
     this.#state.manifestIndex.init(this.#state.manifestIndex.manifestFile);
@@ -80,7 +65,7 @@ export class CompilerWatcher {
 
   #getModuleMap(): Record<string, ManifestModule> {
     return Object.fromEntries(
-      Object.values(this.#state.manifest.modules).map(x => [path.resolve(this.#state.manifest.workspacePath, x.sourceFolder), x])
+      Object.values(this.#state.manifest.modules).map(x => [path.resolve(this.#state.manifest.workspace.path, x.sourceFolder), x])
     );
   }
 
@@ -99,18 +84,20 @@ export class CompilerWatcher {
    * @returns
    */
   async * watchChanges(): AsyncIterable<WatchEvent<{ entry: CompileStateEntry }>> {
+    if (this.#signal.aborted) {
+      yield* [];
+      return;
+    }
+
     const mods = this.#getModuleMap();
-    const ctrl = new AbortController();
-    setMaxListeners(1000, ctrl.signal);
 
     const modules = [...this.#state.manifestIndex.getModuleList('all')].map(x => this.#state.manifestIndex.getModule(x)!);
 
-    const stream = fileWatchEvents(this.#state.manifest, modules, ctrl.signal);
+    const stream = fileWatchEvents(this.#state.manifest, modules, this.#signal);
     for await (const ev of stream) {
 
       if (ev.action === 'reset') {
         yield ev;
-        ctrl.abort();
         return;
       }
 
@@ -118,7 +105,7 @@ export class CompilerWatcher {
       const mod = mods[folder];
       const moduleFile = mod.sourceFolder ?
         (sourceFile.includes(mod.sourceFolder) ? sourceFile.split(`${mod.sourceFolder}/`)[1] : sourceFile) :
-        sourceFile.replace(`${this.#state.manifest.workspacePath}/`, '');
+        sourceFile.replace(`${this.#state.manifest.workspace.path}/`, '');
 
       let entry = this.#state.getBySource(sourceFile);
 

package/support/entry.trvc.ts

@@ -1,63 +1,69 @@
-import fs from 'fs/promises';
-import path from 'path';
+import fs from 'node:fs/promises';
+import path from 'node:path';
 
 import type { ManifestContext } from '@travetto/manifest';
 
+import type { CompilerEventType, CompilerOp, CompilerServerInfo } from './types';
 import { LogUtil } from './log';
+import { CommonUtil } from './util';
 import { CompilerSetup } from './setup';
 import { CompilerServer } from './server/server';
 import { CompilerRunner } from './server/runner';
-import type { CompilerOp, CompilerServerInfo } from './types';
-import { CompilerClientUtil } from './server/client';
-import { CommonUtil } from './util';
+import { CompilerClient } from './server/client';
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
-export const main = (root: ManifestContext, ctx: ManifestContext) => {
+export const main = (ctx: ManifestContext) => {
+  const client = new CompilerClient(ctx, LogUtil.scoped('client.main'));
+  const buildFolders = [ctx.build.outputFolder, ctx.build.compilerFolder];
+
   const ops = {
     /** Stop the server */
     async stop(): Promise<void> {
-      if (await fetch(`${ctx.compilerUrl}/stop`).then(v => v.ok, () => false)) {
-        console.log(`Stopped server ${ctx.workspacePath}: [${ctx.compilerUrl}]`);
+      if (await client.stop()) {
+        console.log(`Stopped server ${ctx.workspace.path}: ${client}`);
       } else {
-        console.log(`Server not running ${ctx.workspacePath}: [${ctx.compilerUrl}]`);
+        console.log(`Server not running ${ctx.workspace.path}: ${client}`);
       }
     },
 
     /** Get server info */
-    info(): Promise<CompilerServerInfo> {
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      return fetch(ctx.compilerUrl).then(v => v.json(), () => ({ state: 'Server not running' })) as Promise<CompilerServerInfo>;
-    },
+    info: (): Promise<CompilerServerInfo | undefined> => client.info(),
 
     /** Clean the server */
     async clean(): Promise<void> {
-      const folders = [ctx.outputFolder, ctx.compilerFolder];
-      if (await fetch(`${ctx.compilerUrl}/clean`).then(v => v.ok, () => false)) {
-        return console.log(`Clean triggered ${ctx.workspacePath}:`, folders);
+      if (await client.clean()) {
+        return console.log(`Clean triggered ${ctx.workspace.path}:`, buildFolders);
       } else {
-        await Promise.all(folders.map(f => fs.rm(path.resolve(ctx.workspacePath, f), { force: true, recursive: true })));
-        return console.log(`Cleaned ${ctx.workspacePath}:`, folders);
+        await Promise.all(buildFolders.map(f => fs.rm(path.resolve(ctx.workspace.path, f), { force: true, recursive: true })));
+        return console.log(`Cleaned ${ctx.workspace.path}:`, buildFolders);
       }
     },
 
+    /** Stream events */
+    events: async (type: CompilerEventType, handler: (ev: unknown) => unknown): Promise<void> => {
+      LogUtil.initLogs(ctx, 'error');
+      for await (const ev of client.fetchEvents(type)) { await handler(ev); }
+    },
+
     /** Main entry point for compilation */
     async compile(op: CompilerOp, setupOnly = false): Promise<(mod: string) => Promise<unknown>> {
       LogUtil.initLogs(ctx, op === 'run' ? 'error' : 'info');
 
-      const server = await new CompilerServer(root, op).listen();
+      const server = await new CompilerServer(ctx, op).listen();
 
       // Wait for build to be ready
       if (server) {
         await server.processEvents(async function* (signal) {
-          const { changed, manifest } = await CompilerSetup.setup(root);
+          const changed = await CompilerSetup.setup(ctx);
           if (!setupOnly) {
-            yield* CompilerRunner.runProcess(root, manifest, changed, op, signal);
-          } else {
-            yield* [];
+            yield* CompilerRunner.runProcess(ctx, changed, op, signal);
           }
         });
       } else {
-        await CompilerClientUtil.waitForBuild(root);
+        const ctrl = new AbortController();
+        LogUtil.consumeProgressEvents(() => client.fetchEvents('progress', { until: ev => !!ev.complete, signal: ctrl.signal }));
+        await client.waitForState(['compile-end', 'watch-start'], 'Successfully built');
+        ctrl.abort();
       }
       return CommonUtil.moduleLoader(ctx);
     },
@@ -69,5 +75,4 @@ export const main = (root: ManifestContext, ctx: ManifestContext) => {
     }
   };
   return ops;
-};
-
+};

package/support/log.ts

@@ -1,9 +1,11 @@
 import type { ManifestContext } from '@travetto/manifest';
-import type { CompilerLogEvent, CompilerLogLevel } from './types';
+import type { CompilerLogEvent, CompilerLogLevel, CompilerProgressEvent } from './types';
 
 export type CompilerLogger = (level: CompilerLogLevel, message: string, ...args: unknown[]) => void;
 export type WithLogger<T> = (log: CompilerLogger) => Promise<T>;
 
+type ProgressWriter = (ev: CompilerProgressEvent) => (unknown | Promise<unknown>);
+
 const LEVEL_TO_PRI: Record<CompilerLogLevel, number> = { debug: 1, info: 2, warn: 3, error: 4 };
 
 const SCOPE_MAX = 15;
@@ -14,16 +16,32 @@ export class LogUtil {
 
   static logLevel: CompilerLogLevel = 'error';
 
+  static logProgress?: ProgressWriter;
+
   /**
    * Set level for operation
    */
   static initLogs(ctx: ManifestContext, defaultLevel: CompilerLogLevel): void {
-    // Listen only if we aren't in quiet
-    if ((process.env.TRV_BUILD || !process.env.TRV_QUIET)) {
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      this.logLevel = (process.env.TRV_BUILD as 'debug') || defaultLevel;
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const build = process.env.TRV_BUILD as CompilerLogLevel | 'none';
+    if (build !== 'none' && process.env.TRV_QUIET !== 'true') {
+      this.logLevel = build || defaultLevel;
+    }
+    this.root = ctx.workspace.path;
+
+    if (this.isLevelActive('info') && process.stdout.isTTY) {
+      this.logProgress = this.#logProgressEvent;
+    }
+  }
+
+  static #logProgressEvent(ev: CompilerProgressEvent): Promise<void> | void {
+    const pct = Math.trunc(ev.idx * 100 / ev.total);
+    const text = ev.complete ? '' : `Compiling [${'#'.repeat(Math.trunc(pct / 10)).padEnd(10, ' ')}] [${ev.idx}/${ev.total}] ${ev.message}`;
+    // Move to 1st position, and clear after text
+    const done = process.stdout.write(`\x1b[1G${text}\x1b[0K`);
+    if (!done) {
+      return new Promise<void>(r => process.stdout.once('drain', r));
     }
-    this.root = ctx.workspacePath;
   }
 
   /**
@@ -36,34 +54,49 @@ export class LogUtil {
   /**
    * Log message with filtering by level
    */
-  static log(scope: string, level: CompilerLogLevel, message: string, ...args: unknown[]): void {
-    LogUtil.sendLogEventToConsole({ level, scope, message, args, time: Date.now() });
+  static log(event: CompilerLogEvent): void;
+  static log(scope: string, ...args: Parameters<CompilerLogger>): void;
+  static log(scopeOrEvent: string | CompilerLogEvent, level?: CompilerLogLevel, message?: string, ...args: unknown[]): void {
+    const ev = typeof scopeOrEvent === 'string' ? { scope: scopeOrEvent, level: level!, message, args } : scopeOrEvent;
+    if (this.isLevelActive(ev.level)) {
+      const params = [ev.message, ...ev.args ?? []].map(x => typeof x === 'string' ? x.replaceAll(this.root, '.') : x);
+      if (ev.scope) {
+        params.unshift(`[${ev.scope.padEnd(SCOPE_MAX, ' ')}]`);
+      }
+      params.unshift(new Date().toISOString(), `${ev.level.padEnd(5)}`);
+      // eslint-disable-next-line no-console
+      console[ev.level]!(...params);
+    }
   }
 
   /**
    * With logger
    */
   static withLogger<T>(scope: string, op: WithLogger<T>, basic = true): Promise<T> {
-    const log = this.log.bind(null, scope);
+    const log = this.scoped(scope);
     basic && log('debug', 'Started');
     return op(log).finally(() => basic && log('debug', 'Completed'));
   }
 
   /**
-   * Compiler log event to console
+   * With scope
    */
-  static sendLogEventToConsole(ev: CompilerLogEvent): void {
-    if (this.isLevelActive(ev.level)) {
-      const params = [ev.message, ...ev.args ?? []].map(x => typeof x === 'string' ? x.replaceAll(LogUtil.root, '.') : x);
-      if (ev.scope) {
-        params.unshift(`[${ev.scope.padEnd(SCOPE_MAX, ' ')}]`);
-      }
-      params.unshift(`${ev.level.padEnd(5)}`);
-      if (!/(0|false|off|no)$/i.test(process.env.TRV_LOG_TIME ?? '')) {
-        params.unshift(new Date().toISOString());
-      }
-      // eslint-disable-next-line no-console
-      console[ev.level]!(...params);
-    }
+  static scoped(scope: string): CompilerLogger {
+    return this.log.bind(this, scope);
+  }
+
+  /**
+   * Stream Compiler log events to console
+   */
+  static async consumeLogEvents(src: AsyncIterable<CompilerLogEvent>): Promise<void> {
+    for await (const ev of src) { this.log(ev); }
+  }
+
+  /**
+   * Write all progress events if active
+   */
+  static async consumeProgressEvents(src: () => AsyncIterable<CompilerProgressEvent>): Promise<void> {
+    if (!this.logProgress) { return; }
+    for await (const item of src()) { await this.logProgress?.(item); }
   }
 }

package/support/queue.ts

@@ -13,6 +13,9 @@ export class AsyncQueue<X> implements AsyncIterator<X>, AsyncIterable<X> {
 
   constructor(signal?: AbortSignal) {
     signal?.addEventListener('abort', () => this.close());
+    if (signal?.aborted) {
+      this.close();
+    }
   }
 
   [Symbol.asyncIterator](): AsyncIterator<X> { return this; }
@@ -25,12 +28,8 @@ export class AsyncQueue<X> implements AsyncIterator<X>, AsyncIterable<X> {
     return { value: (this.#queue.length ? this.#queue.shift() : undefined)!, done: this.#done };
   }
 
-  add(item: X, immediate = false): void {
-    if (!immediate) {
-      this.#queue.push(item);
-    } else {
-      this.#queue.unshift(item);
-    }
+  add(item: X): void {
+    this.#queue.push(item);
     this.#ready.resolve();
   }
 

package/support/server/client.ts

@@ -1,118 +1,131 @@
-import rl from 'readline/promises';
-import { Readable } from 'stream';
+import rl from 'node:readline/promises';
+import timers from 'node:timers/promises';
+import { Readable } from 'node:stream';
 
-import type { ManifestContext } from '@travetto/manifest';
-import type { CompilerServerEvent, CompilerServerEventType, CompilerServerInfo, CompilerStateType } from '../types';
+import { ManifestContext } from '@travetto/manifest';
 
-import { LogUtil } from '../log';
+import type { CompilerEvent, CompilerEventType, CompilerServerInfo, CompilerStateType } from '../types';
+import type { CompilerLogger } from '../log';
 
 declare global {
   interface RequestInit { timeout?: number }
 }
 
-const log = LogUtil.log.bind(LogUtil, 'compiler-client');
-
-function getSignal(input?: AbortSignal): AbortSignal {
-  // Ensure we capture end of process at least
-  if (!input) {
-    const ctrl = new AbortController();
-    process.on('SIGINT', () => ctrl.abort());
-    input = ctrl.signal;
-  }
-  return input;
-}
+type FetchEventsConfig<T> = {
+  signal?: AbortSignal;
+  until?: (ev: T) => boolean;
+  enforceIteration?: boolean;
+};
 
 /**
  * Compiler Client Operations
  */
-export class CompilerClientUtil {
-
-  /**
-   * Get server information, if server is running
-   */
-  static async getServerInfo(ctx: ManifestContext): Promise<CompilerServerInfo | undefined> {
-    try {
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      const res = (await fetch(`${ctx.compilerUrl}/info`).then(v => v.json())) as CompilerServerInfo;
-      return res;
-    } catch (err) {
-      return undefined;
-    }
+export class CompilerClient {
+
+  #url: string;
+  #log?: CompilerLogger;
+
+  constructor(ctx: ManifestContext, log?: CompilerLogger) {
+    this.#url = ctx.build.compilerUrl.replace('localhost', '127.0.0.1');
+    this.#log = log;
+  }
+
+  toString(): string {
+    return `[${this.constructor.name} url=${this.#url}]`;
+  }
+
+  get url(): string {
+    return this.#url;
+  }
+
+  /** Get server information, if server is running */
+  info(): Promise<CompilerServerInfo | undefined> {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    return fetch(`${this.#url}/info`).then(v => v.json(), () => undefined) as Promise<CompilerServerInfo>;
+  }
+
+  /** Clean the server */
+  clean(): Promise<boolean> {
+    return fetch(`${this.#url}/clean`).then(v => v.ok, () => false);
+  }
+
+  /** Stop server */
+  stop(): Promise<boolean> {
+    return fetch(`${this.#url}/stop`).then(v => v.ok, () => false);
   }
 
-  /**
-   * Fetch compiler events
-   */
-  static async * fetchEvents<
-    V extends CompilerServerEventType,
-    T extends (CompilerServerEvent & { type: V })['payload']
-  >(ctx: ManifestContext, type: V, signal?: AbortSignal, until?: (ev: T) => boolean): AsyncIterable<T> {
-    log('debug', `Starting watch for events of type "${type}"`);
+  /** Fetch compiler events */
+  async * fetchEvents<
+    V extends CompilerEventType,
+    T extends (CompilerEvent & { type: V })['payload']
+  >(type: V, cfg: FetchEventsConfig<T> = {}): AsyncIterable<T> {
+    let info = await this.info();
+    if (!info) {
+      return;
+    }
+
+    this.#log?.('debug', `Starting watch for events of type "${type}"`);
+
+    let signal = cfg.signal;
+
+    // Ensure we capture end of process at least
+    if (!signal) {
+      const ctrl = new AbortController();
+      process.on('SIGINT', () => ctrl.abort());
+      signal = ctrl.signal;
+    }
+
 
-    signal = getSignal(signal);
+    const { iteration } = info;
 
     for (; ;) {
       const ctrl = new AbortController();
       try {
         signal.addEventListener('abort', () => ctrl.abort());
-        const stream = await fetch(`${ctx.compilerUrl}/event/${type}`, {
+        const stream = await fetch(`${this.#url}/event/${type}`, {
           signal: ctrl.signal,
           timeout: 1000 * 60 * 60
         });
         for await (const line of rl.createInterface(Readable.fromWeb(stream.body!))) {
           if (line.trim().charAt(0) === '{') {
             const val = JSON.parse(line);
-            if (until?.(val)) {
-              setTimeout(() => ctrl.abort(), 1);
+            if (cfg.until?.(val)) {
+              await timers.setTimeout(1);
+              ctrl.abort();
             }
             yield val;
           }
         }
       } catch (err) { }
 
-      if (ctrl.signal.aborted || !(await this.getServerInfo(ctx))) { // If health check fails, or aborted
-        log('debug', `Stopping watch for events of type "${type}"`);
+      await timers.setTimeout(1);
+
+      info = await this.info();
+
+      if (ctrl.signal.aborted || !info || (cfg.enforceIteration && info.iteration !== iteration)) { // If health check fails, or aborted
+        this.#log?.('debug', `Stopping watch for events of type "${type}"`);
         return;
       } else {
-        log('debug', `Restarting watch for events of type "${type}"`);
+        this.#log?.('debug', `Restarting watch for events of type "${type}"`);
       }
     }
   }
 
-  /**
-   * Wait for one of N states to be achieved
-   */
-  static async waitForState(ctx: ManifestContext, states: CompilerStateType[], signal?: AbortSignal): Promise<void> {
+  /** Wait for one of N states to be achieved */
+  async waitForState(states: CompilerStateType[], message?: string, signal?: AbortSignal): Promise<void> {
     const set = new Set(states);
-    const existing = await this.getServerInfo(ctx);
-    log('debug', `Existing: ${JSON.stringify(existing)}`);
+    const existing = await this.info();
+    this.#log?.('debug', `Existing: ${JSON.stringify(existing)}`);
     if (existing && set.has(existing.state)) {
-      log('debug', `Waited for state, ${existing.state} in server info`);
+      this.#log?.('debug', `Waited for state, ${existing.state} in server info`);
       return;
     }
     // Loop until
-    log('debug', `Waiting for states, ${states.join(', ')}`);
-    for await (const _ of this.fetchEvents(ctx, 'state', signal, s => set.has(s.state))) { }
-    log('debug', `Found state, one of ${states.join(', ')}`);
-  }
-
-  /**
-   * Stream logs
-   */
-  static async streamLogs(ctx: ManifestContext, signal?: AbortSignal): Promise<void> {
-    if (!LogUtil.logLevel) {
-      return;
-    }
-    for await (const ev of this.fetchEvents(ctx, 'log', signal!)) {
-      LogUtil.sendLogEventToConsole(ev);
+    this.#log?.('debug', `Waiting for states, ${states.join(', ')}`);
+    for await (const _ of this.fetchEvents('state', { signal, until: s => set.has(s.state) })) { }
+    this.#log?.('debug', `Found state, one of ${states.join(', ')} `);
+    if (message) {
+      this.#log?.('info', message);
     }
   }
-
-  /**
-   * Wait for build
-   */
-  static async waitForBuild(ctx: ManifestContext, signal?: AbortSignal): Promise<void> {
-    await this.waitForState(ctx, ['compile-end', 'watch-start'], signal);
-    log('info', 'Successfully built');
-  }
 }