@travetto/compiler 4.0.0-rc.0 → 4.0.0-rc.1

package/support/server/process-handle.ts

@@ -0,0 +1,57 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import timers from 'node:timers/promises';
+
+import type { ManifestContext } from '@travetto/manifest';
+
+export class ProcessHandle {
+
+  #file: string;
+
+  constructor(ctx: ManifestContext, name: string) {
+    this.#file = path.resolve(ctx.workspace.path, ctx.build.toolFolder, `${name}.pid`);
+  }
+
+  writePid(pid: number): Promise<void> {
+    return fs.writeFile(this.#file, JSON.stringify(pid), 'utf8');
+  }
+
+  getPid(): Promise<number | undefined> {
+    return fs.readFile(this.#file, 'utf8').then(v => +v, () => undefined);
+  }
+
+  async isRunning(): Promise<boolean> {
+    const pid = await this.getPid();
+    if (!pid) { return false; }
+    try {
+      process.kill(pid, 0); // See if process is still running
+      return false;
+    } catch { }
+    return true; // Still running
+  }
+
+  async kill(): Promise<boolean> {
+    const pid = await this.getPid();
+    if (pid && await this.isRunning()) {
+      try {
+        return process.kill(pid);
+      } catch { }
+    }
+    return false;
+  }
+
+  async ensureKilled(gracePeriod: number = 3000): Promise<boolean> {
+    const start = Date.now();
+    const pid = await this.getPid();
+    while (pid && (Date.now() - start) < gracePeriod) { // Ensure its done
+      if (!await this.isRunning()) {
+        return true;
+      }
+      await timers.setTimeout(100);
+    }
+    try {
+      pid && process.kill(pid); // Force kill
+    } catch { }
+    return pid !== undefined;
+  }
+}

package/support/server/runner.ts

@@ -4,12 +4,12 @@ import { rmSync } from 'node:fs';
 
 import type { ManifestContext, DeltaEvent } from '@travetto/manifest';
 
-import type { CompilerOp, CompilerEvent } from '../types';
+import type { CompilerEvent, CompilerMode } from '../types';
 import { AsyncQueue } from '../queue';
 import { LogUtil } from '../log';
 import { CommonUtil } from '../util';
 
-const log = LogUtil.scoped('compiler-exec');
+const log = LogUtil.logger('compiler-exec');
 const isEvent = (msg: unknown): msg is CompilerEvent => !!msg && typeof msg === 'object' && 'type' in msg;
 
 /**
@@ -20,8 +20,13 @@ export class CompilerRunner {
   /**
    * Run compile process
    */
-  static async * runProcess(ctx: ManifestContext, changed: DeltaEvent[], op: CompilerOp, signal: AbortSignal): AsyncIterable<CompilerEvent> {
-    const watch = op === 'watch';
+  static async * runProcess(ctx: ManifestContext, changed: DeltaEvent[], mode: CompilerMode, signal: AbortSignal): AsyncIterable<CompilerEvent> {
+    if (signal.aborted) {
+      log('debug', 'Skipping, shutting down');
+      return;
+    }
+
+    const watch = mode === 'watch';
     if (!changed.length && !watch) {
       yield { type: 'state', payload: { state: 'compile-end' } };
       log('debug', 'Skipped');
@@ -36,7 +41,6 @@ export class CompilerRunner {
     const changedFiles = changed[0]?.file === '*' ? ['*'] : changed.map(ev => ev.sourceFile);
 
     const queue = new AsyncQueue<CompilerEvent>();
-    let kill: (() => void) | undefined;
 
     try {
       await CommonUtil.writeTextFile(deltaFile, changedFiles.join('\n'));
@@ -53,22 +57,23 @@ export class CompilerRunner {
         .on('message', msg => isEvent(msg) && queue.add(msg))
         .on('exit', () => queue.close());
 
-      kill = (): unknown => proc.kill('SIGINT');
+      const kill = (): unknown => {
+        log('debug', 'Shutting down process');
+        return (proc.connected ? proc.send('shutdown', (err) => proc.kill()) : proc.kill());
+      };
 
       process.once('SIGINT', kill);
       signal.addEventListener('abort', kill);
 
       yield* queue;
 
-      if (!proc.killed && proc.exitCode !== 0) {
-        log('error', `Failed during compilation, code=${proc.exitCode}, killed=${proc.killed}`);
+      if (proc.exitCode !== 0) {
+        log('error', `Terminated during compilation, code=${proc.exitCode}, killed=${proc.killed}`);
       }
+      process.off('SIGINT', kill);
 
       log('debug', 'Finished');
     } finally {
-      if (kill) {
-        process.off('SIGINT', kill);
-      }
       rmSync(deltaFile, { force: true });
     }
   }

package/support/server/server.ts

@@ -1,36 +1,23 @@
 import http from 'node:http';
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { setMaxListeners } from 'node:events';
 
 import type { ManifestContext } from '@travetto/manifest';
 
-import type { CompilerMode, CompilerOp, CompilerProgressEvent, CompilerEvent, CompilerEventType, CompilerServerInfo } from '../types';
+import type { CompilerMode, CompilerProgressEvent, CompilerEvent, CompilerEventType, CompilerServerInfo } from '../types';
 import { LogUtil } from '../log';
 import { CompilerClient } from './client';
 import { CommonUtil } from '../util';
+import { ProcessHandle } from './process-handle';
 
-const log = LogUtil.scoped('compiler-server');
+const log = LogUtil.logger('compiler-server');
 
 /**
  * Compiler Server Class
  */
 export class CompilerServer {
 
-  static readJSONRequest<T>(req: http.IncomingMessage): Promise<T> {
-    return new Promise<T>((res, rej) => {
-      const body: Buffer[] = [];
-      req.on('data', (chunk) => body.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk));
-      req.on('end', () => {
-        try {
-          res(JSON.parse(Buffer.concat(body).toString('utf8')));
-        } catch (err) {
-          rej(err);
-        }
-      });
-      req.on('error', rej);
-    });
-  }
-
   #ctx: ManifestContext;
   #server: http.Server;
   #listeners: { res: http.ServerResponse, type: CompilerEventType }[] = [];
@@ -39,15 +26,18 @@ export class CompilerServer {
   info: CompilerServerInfo;
   #client: CompilerClient;
   #url: string;
+  #handle: Record<'compiler' | 'server', ProcessHandle>;
 
-  constructor(ctx: ManifestContext, op: CompilerOp) {
+  constructor(ctx: ManifestContext, mode: CompilerMode) {
     this.#ctx = ctx;
-    this.#client = new CompilerClient(ctx, LogUtil.scoped('client.server'));
+    this.#client = new CompilerClient(ctx, LogUtil.logger('client.server'));
     this.#url = this.#client.url;
+    this.#handle = { server: new ProcessHandle(ctx, 'server'), compiler: new ProcessHandle(ctx, 'compiler') };
+
     this.info = {
       state: 'startup',
       iteration: Date.now(),
-      mode: op === 'run' ? 'build' : op,
+      mode,
       serverPid: process.pid,
       compilerPid: -1,
       path: ctx.workspace.path,
@@ -60,7 +50,7 @@ export class CompilerServer {
       keepAliveTimeout: 1000 * 60 * 60,
     }, (req, res) => this.#onRequest(req, res));
 
-    process.once('exit', () => this.#shutdown.abort());
+    setMaxListeners(1000, this.signal);
   }
 
   get mode(): CompilerMode {
@@ -80,9 +70,11 @@ export class CompilerServer {
             const info = await this.#client.info();
             resolve((info && info.mode === 'build' && this.mode === 'watch') ? 'retry' : 'running');
           } else {
+            log('warn', 'Failed in running server', err);
             reject(err);
           }
-        });
+        })
+        .on('close', () => log('debug', 'Server close event'));
 
       const url = new URL(this.#url);
       setTimeout(() => this.#server.listen(+url.port, url.hostname), 1); // Run async
@@ -96,6 +88,8 @@ export class CompilerServer {
       // Let the server finish
       await this.#client.waitForState(['close'], 'Server closed', this.signal);
       return this.#tryListen(attempt + 1);
+    } else if (output === 'ok') {
+      await this.#handle.server.writePid(this.info.serverPid);
     }
 
     return output;
@@ -105,6 +99,10 @@ export class CompilerServer {
     res.writeHead(200);
     // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
     this.#listeners.push({ res, type: type as 'change' });
+    if (type === 'state') { // Send on initial connect
+      res.write(JSON.stringify({ state: this.info.state }));
+    }
+    res.write('\n'); // Send at least one byte on listen
     await new Promise(resolve => res.on('close', resolve));
     this.#listeners.splice(this.#listeners.findIndex(x => x.res === res), 1);
     res.end();
@@ -123,7 +121,9 @@ export class CompilerServer {
     log('info', 'Server disconnect requested');
     this.info.iteration = Date.now();
     await new Promise(r => setTimeout(r, 20));
-    this.#server.closeAllConnections(); // Force reconnects
+    for (const el of this.#listeners) {
+      el.res.destroy();
+    }
   }
 
   async #clean(): Promise<{ clean: boolean }> {
@@ -144,10 +144,14 @@ export class CompilerServer {
 
     let out: unknown;
     switch (action) {
-      case 'send-event': await this.#emitEvent(await CompilerServer.readJSONRequest(req)); out = { received: true }; break;
       case 'event': return await this.#addListener(subAction, res);
-      case 'stop': out = await this.close(true); break;
       case 'clean': out = await this.#clean(); break;
+      case 'stop': {
+        // Must send immediately
+        res.end(JSON.stringify({ closing: true }));
+        await this.close();
+        break;
+      }
       case 'info':
       default: out = this.info ?? {}; break;
     }
@@ -158,10 +162,6 @@ export class CompilerServer {
    * Process events
    */
   async processEvents(src: (signal: AbortSignal) => AsyncIterable<CompilerEvent>): Promise<void> {
-
-    LogUtil.log('compiler', 'debug', 'Started, streaming logs');
-    LogUtil.consumeLogEvents(this.#client.fetchEvents('log', { signal: this.signal }));
-
     for await (const ev of CommonUtil.restartableEvents(src, this.signal, this.isResetEvent)) {
       if (ev.type === 'progress') {
         await LogUtil.logProgress?.(ev.payload);
@@ -171,10 +171,13 @@ export class CompilerServer {
 
       if (ev.type === 'state') {
         this.info.state = ev.payload.state;
+        await this.#handle.compiler.writePid(this.info.compilerPid);
         if (ev.payload.state === 'init' && ev.payload.extra && 'pid' in ev.payload.extra && typeof ev.payload.extra.pid === 'number') {
           this.info.compilerPid = ev.payload.extra.pid;
         }
         log('info', `State changed: ${this.info.state}`);
+      } else if (ev.type === 'log') {
+        LogUtil.logEvent(ev.payload);
       }
       if (this.isResetEvent(ev)) {
         await this.#disconnectActive();
@@ -182,31 +185,45 @@ export class CompilerServer {
     }
 
     // Terminate, after letting all remaining events emit
-    await this.close(this.signal.aborted);
+    await this.close();
+
+    log('debug', 'Finished processing events');
   }
 
   /**
    * Close server
    */
-  async close(force: boolean): Promise<unknown> {
+  async close(): Promise<void> {
     log('info', 'Closing down server');
-    await new Promise(r => {
 
-      if (force) {
-        const cancel: CompilerProgressEvent = { complete: true, idx: 0, total: 0, message: 'Complete', operation: 'compile' };
-        LogUtil.logProgress?.(cancel);
-        this.#emitEvent({ type: 'progress', payload: cancel });
-      }
+    // If we are in a place where progress exists
+    if (this.info.state === 'compile-start') {
+      const cancel: CompilerProgressEvent = { complete: true, idx: 0, total: 0, message: 'Complete', operation: 'compile' };
+      LogUtil.logProgress?.(cancel);
+      this.#emitEvent({ type: 'progress', payload: cancel });
+    }
 
-      this.#server.close(r);
-      this.#emitEvent({ type: 'state', payload: { state: 'close' } });
-      this.#server.unref();
-      setImmediate(() => {
-        this.#server.closeAllConnections();
-        this.#shutdown.abort();
+    try {
+      await new Promise((resolve, reject) => {
+        setTimeout(reject, 2000).unref(); // 2s max wait
+        this.#server.close(resolve);
+        this.#emitEvent({ type: 'state', payload: { state: 'close' } });
+        setImmediate(() => {
+          this.#server.closeAllConnections();
+          this.#shutdown.abort();
+        });
       });
-    });
-    return { closing: true };
+    } catch { // Timeout or other error
+      // Force shutdown
+      this.#server.closeAllConnections();
+      if (this.info.compilerPid) { // Ensure its killed
+        try {
+          process.kill(this.info.compilerPid);
+        } catch { }
+      }
+    }
+
+    log('info', 'Closed down server');
   }
 
   /**

package/support/setup.ts

@@ -214,7 +214,7 @@ export class CompilerSetup {
       // Update all manifests when in mono repo
       if (delta.length && ctx.workspace.mono) {
         const names: string[] = [];
-        const mods = Object.values(manifest.modules).filter(x => x.local && x.name !== ctx.workspace.name);
+        const mods = Object.values(manifest.modules).filter(x => x.workspace && x.name !== ctx.workspace.name);
         for (const mod of mods) {
           const modCtx = ManifestUtil.getModuleContext(ctx, mod.sourceFolder);
           const modManifest = await ManifestUtil.buildManifest(modCtx);

package/support/types.ts

@@ -1,8 +1,7 @@
 export type CompilerMode = 'build' | 'watch';
-export type CompilerOp = CompilerMode | 'run';
 
 export type CompilerStateType = 'startup' | 'init' | 'compile-start' | 'compile-end' | 'watch-start' | 'watch-end' | 'reset' | 'close';
-export type CompilerChangeEvent = { file: string, action: 'create' | 'update' | 'delete', folder: string, output: string, module: string, time: number };
+export type CompilerChangeEvent = { file: string, action: 'create' | 'update' | 'delete', output: string, module: string, time: number };
 export type CompilerLogLevel = 'info' | 'debug' | 'warn' | 'error';
 export type CompilerLogEvent = { level: CompilerLogLevel, message: string, time: number, args?: unknown[], scope?: string };
 export type CompilerProgressEvent = { idx: number, total: number, message: string, operation: 'compile', complete?: boolean };

package/support/util.ts

@@ -58,6 +58,7 @@ export class CommonUtil {
    * Restartable Event Stream
    */
   static async * restartableEvents<T>(src: (signal: AbortSignal) => AsyncIterable<T>, parent: AbortSignal, shouldRestart: (item: T) => boolean): AsyncIterable<T> {
+    const log = LogUtil.logger('event-stream');
     outer: while (!parent.aborted) {
       const controller = new AbortController();
       setMaxListeners(1000, controller.signal);
@@ -66,18 +67,19 @@ export class CommonUtil {
 
       const comp = src(controller.signal);
 
-      LogUtil.log('event-stream', 'debug', 'Started event stream');
+      log('debug', 'Started event stream');
 
       // Wait for all events, close at the end
       for await (const ev of comp) {
         yield ev;
         if (shouldRestart(ev)) {
+          log('debug', 'Restarting stream');
           controller.abort(); // Ensure terminated of process
           continue outer;
         }
       }
 
-      LogUtil.log('event-stream', 'debug', 'Finished event stream');
+      log('debug', 'Finished event stream');
 
       // Natural exit, we done
       if (!controller.signal.aborted) { // Shutdown source if still running

package/src/internal/watch-core.ts

@@ -1,107 +0,0 @@
-import fs from 'node:fs/promises';
-
-import { IndexedModule, ManifestContext, ManifestModuleUtil, path } from '@travetto/manifest';
-
-import { AsyncQueue } from '../../support/queue';
-
-export type WatchEvent<T = {}> =
-  ({ action: 'create' | 'update' | 'delete', file: string, folder: string } & T) |
-  { action: 'reset', file: string };
-
-const CREATE_THRESHOLD = 50;
-const VALID_TYPES = new Set(['ts', 'typings', 'js', 'package-json']);
-type ToWatch = { file: string, actions: string[] };
-
-/** Watch file for reset */
-function watchForReset(q: AsyncQueue<WatchEvent>, root: string, files: ToWatch[], signal: AbortSignal): void {
-  const watchers: Record<string, { folder: string, files: Map<string, (ToWatch & { name: string, actionSet: Set<string> })> }> = {};
-  // Group by base path
-  for (const el of files) {
-    const full = path.resolve(root, el.file);
-    const folder = path.dirname(full);
-    const tgt = { ...el, name: path.basename(el.file), actionSet: new Set(el.actions) };
-    const watcher = (watchers[folder] ??= { folder, files: new Map() });
-    watcher.files.set(tgt.name, tgt);
-  }
-
-  // Fire them all off
-  Object.values(watchers).map(async (watcher) => {
-    try {
-      for await (const ev of fs.watch(watcher.folder, { persistent: true, encoding: 'utf8', signal })) {
-        const toWatch = watcher.files.get(ev.filename!);
-        if (toWatch) {
-          const stat = await fs.stat(path.resolve(root, ev.filename!)).catch(() => undefined);
-          const action = !stat ? 'delete' : ((Date.now() - stat.ctimeMs) < CREATE_THRESHOLD) ? 'create' : 'update';
-          if (toWatch.actionSet.has(action)) {
-            q.add({ action: 'reset', file: ev.filename! });
-          }
-        }
-      }
-    } catch (err) {
-      // Ignore
-    }
-  });
-}
-
-/** Watch recursive files for a given folder */
-async function watchFolder(ctx: ManifestContext, q: AsyncQueue<WatchEvent>, src: string, target: string, signal: AbortSignal): Promise<void> {
-  const lib = await import('@parcel/watcher');
-  const ignore = [
-    'node_modules', '**/.trv',
-    ...((!ctx.workspace.mono || src === ctx.workspace.path) ? [ctx.build.compilerFolder, ctx.build.outputFolder] : []),
-    ...(await fs.readdir(src)).filter(x => x.startsWith('.'))
-  ];
-
-  const cleanup = await lib.subscribe(src, async (err, events) => {
-    if (err) {
-      console.error('Watch Error', err);
-    }
-    for (const ev of events) {
-      const finalEv = { action: ev.type, file: path.toPosix(ev.path), folder: target };
-      if (ev.type !== 'delete') {
-        const stats = await fs.stat(finalEv.file);
-        if ((Date.now() - stats.ctimeMs) < CREATE_THRESHOLD) {
-          ev.type = 'create'; // Force create on newly stated files
-        }
-      }
-
-      if (ev.type === 'delete' && finalEv.file === src) {
-        return q.close();
-      }
-
-      const matches = !finalEv.file.includes('/.') && VALID_TYPES.has(ManifestModuleUtil.getFileType(finalEv.file));
-      if (matches) {
-        q.add(finalEv);
-      }
-    }
-  }, { ignore });
-  signal.addEventListener('abort', () => cleanup.unsubscribe());
-}
-
-/** Watch files */
-export async function* fileWatchEvents(manifest: ManifestContext, modules: IndexedModule[], signal: AbortSignal): AsyncIterable<WatchEvent> {
-  const q = new AsyncQueue<WatchEvent>(signal);
-
-  for (const m of modules.filter(x => !manifest.workspace.mono || x.sourcePath !== manifest.workspace.path)) {
-    await watchFolder(manifest, q, m.sourcePath, m.sourcePath, signal);
-  }
-
-  // Add monorepo folders
-  if (manifest.workspace.mono) {
-    const mono = modules.find(x => x.sourcePath === manifest.workspace.path)!;
-    for (const folder of Object.keys(mono.files)) {
-      if (!folder.startsWith('$')) {
-        await watchFolder(manifest, q, path.resolve(mono.sourcePath, folder), mono.sourcePath, signal);
-      }
-    }
-  }
-
-  watchForReset(q, manifest.workspace.path, [
-    { file: manifest.build.outputFolder, actions: ['delete'] },
-    { file: manifest.build.compilerFolder, actions: ['delete'] },
-    { file: 'package-lock.json', actions: ['delete', 'update', 'create'] },
-    { file: 'package.json', actions: ['delete', 'update', 'create'] }
-  ], signal);
-
-  yield* q;
-}