@travetto/compiler 3.4.4 → 4.0.0-rc.1

package/README.md

@@ -30,7 +30,8 @@ The compiler cli supports the following operations:
    *  `build` - Ensure the project is built and upto date
    *  `clean` - Clean out the output and compiler caches
    *  `info` - Retrieve the compiler information, if running
-   *  `manifest [output] [prod]` - Generate the project manifest
+   *  `event <log|progress|state>` - Watch events in realtime as newline delimited JSON
+   *  `manifest --prod [output]` - Generate the project manifest
 In addition to the normal output, the compiler supports an environment variable `TRV_BUILD` that supports the following values: `debug`, `info`, `warn` or `none`.  This provides different level of logging during the build process which is helpful to diagnose any odd behaviors.  When invoking an unknown command (e.g. `<other>` from above), the default level is `warn`.  Otherwise the default logging level is `info`.
 
 **Terminal: Sample trv output with debug logging**
@@ -38,34 +39,35 @@ In addition to the normal output, the compiler supports an environment variable
 $ TRV_BUILD=debug trvc build
 
 2029-03-14T04:00:00.618Z info  [compiler-server] Starting server http://127.0.0.1:25539
-2029-03-14T04:00:00.837Z debug [compiler-client] Starting watch for events of type "log"
+2029-03-14T04:00:00.837Z debug [client.main    ] Start Server
 2029-03-14T04:00:01.510Z debug [event-stream   ] Started event stream
 2029-03-14T04:00:02.450Z debug [precompile     ] Started
-2029-03-14T04:00:02.762Z debug [compiler-server] Receive request { action: 'event', subAction: 'log' }
-2029-03-14T04:00:02.947Z debug [precompile     ] Skipped @travetto/terminal
-2029-03-14T04:00:03.093Z debug [precompile     ] Skipped @travetto/manifest
-2029-03-14T04:00:04.003Z debug [precompile     ] Skipped @travetto/transformer
-2029-03-14T04:00:04.495Z debug [precompile     ] Skipped @travetto/compiler
-2029-03-14T04:00:05.066Z debug [precompile     ] Completed
-2029-03-14T04:00:05.307Z debug [manifest       ] Started
-2029-03-14T04:00:05.952Z debug [manifest       ] Completed
-2029-03-14T04:00:06.859Z debug [transformers   ] Started
-2029-03-14T04:00:07.720Z debug [transformers   ] Skipped @travetto/base
-2029-03-14T04:00:08.179Z debug [transformers   ] Skipped @travetto/cli
-2029-03-14T04:00:08.588Z debug [transformers   ] Skipped @travetto/manifest
-2029-03-14T04:00:09.493Z debug [transformers   ] Skipped @travetto/registry
-2029-03-14T04:00:10.395Z debug [transformers   ] Skipped @travetto/schema
-2029-03-14T04:00:10.407Z debug [transformers   ] Completed
-2029-03-14T04:00:10.799Z debug [delta          ] Started
-2029-03-14T04:00:11.013Z debug [delta          ] Completed
-2029-03-14T04:00:11.827Z debug [manifest       ] Started
-2029-03-14T04:00:11.894Z debug [manifest       ] Wrote manifest @travetto-doc/compiler
-2029-03-14T04:00:12.133Z debug [manifest       ] Completed
-2029-03-14T04:00:13.123Z info  [compiler-server] State changed: compile-end
-2029-03-14T04:00:14.014Z debug [compiler-exec  ] Skipped
-2029-03-14T04:00:14.924Z debug [event-stream   ] Finished event stream
-2029-03-14T04:00:15.690Z info  [compiler-server] Closing down server
-2029-03-14T04:00:15.865Z debug [compiler-client] Stopping watch for events of type "log"
+2029-03-14T04:00:02.762Z debug [precompile     ] Skipped @travetto/manifest
+2029-03-14T04:00:02.947Z debug [precompile     ] Skipped @travetto/transformer
+2029-03-14T04:00:03.093Z debug [precompile     ] Skipped @travetto/compiler
+2029-03-14T04:00:04.003Z debug [precompile     ] Completed
+2029-03-14T04:00:04.495Z debug [manifest       ] Started
+2029-03-14T04:00:05.066Z debug [manifest       ] Completed
+2029-03-14T04:00:05.307Z debug [transformers   ] Started
+2029-03-14T04:00:05.952Z debug [transformers   ] Skipped @travetto/base
+2029-03-14T04:00:06.859Z debug [transformers   ] Skipped @travetto/cli
+2029-03-14T04:00:07.720Z debug [transformers   ] Skipped @travetto/manifest
+2029-03-14T04:00:08.179Z debug [transformers   ] Skipped @travetto/registry
+2029-03-14T04:00:08.588Z debug [transformers   ] Skipped @travetto/schema
+2029-03-14T04:00:09.493Z debug [transformers   ] Completed
+2029-03-14T04:00:10.395Z debug [delta          ] Started
+2029-03-14T04:00:10.407Z debug [delta          ] Completed
+2029-03-14T04:00:10.799Z debug [manifest       ] Started
+2029-03-14T04:00:11.013Z debug [manifest       ] Wrote manifest @travetto-doc/compiler
+2029-03-14T04:00:11.827Z debug [manifest       ] Completed
+2029-03-14T04:00:11.894Z info  [compiler-server] State changed: compile-end
+2029-03-14T04:00:12.133Z debug [compiler-exec  ] Skipped
+2029-03-14T04:00:13.123Z debug [event-stream   ] Finished event stream
+2029-03-14T04:00:14.014Z info  [compiler-server] Closing down server
+2029-03-14T04:00:14.924Z debug [compiler-server] Server close event
+2029-03-14T04:00:15.690Z info  [compiler-server] Closed down server
+2029-03-14T04:00:15.865Z debug [compiler-server] Finished processing events
+2029-03-14T04:00:16.757Z debug [client.main    ] End Server
 ```
 
 **Terminal: Sample trv output with default log level**

package/bin/common.js

@@ -1,55 +1,60 @@
 // @ts-check
-
-import fs from 'fs/promises';
-import path from 'path';
-import { createRequire } from 'module';
+import { statSync, readFileSync, writeFileSync, mkdirSync, readdirSync, existsSync, rmSync } from 'node:fs';
+import path from 'node:path';
 
 import { getManifestContext } from '@travetto/manifest/bin/context.js';
 
-const COMPILER_FILES = [...['entry.trvc', 'log', 'queue', 'server/client', 'server/runner', 'server/server', 'setup', 'util'].map(x => `support/${x}.ts`), 'package.json'];
+/** @typedef {import('@travetto/manifest').ManifestContext} Ctx */
 
-/** @return {Promise<ReturnType<import('@travetto/compiler/support/entry.trvc').main>>} */
-export const getEntry = async () => {
-  const ctx = await getManifestContext();
-  const tsconfigFile = path.resolve(ctx.workspacePath, 'tsconfig.json');
-  if (!(await fs.stat(tsconfigFile).catch(() => undefined))) {
-    await fs.writeFile(tsconfigFile, JSON.stringify({ extends: '@travetto/compiler/tsconfig.trv.json' }), 'utf8');
-  }
-  const compMod = path.dirname(createRequire(path.resolve(ctx.workspacePath, 'node_modules')).resolve('@travetto/compiler/package.json'));
-  const files = [];
-
-  for (const file of COMPILER_FILES) {
-    const target = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/compiler', file).replace(/[.]tsx?$/, '.js');
-    const src = path.resolve(compMod, file);
-
-    const targetTime = await fs.stat(target).then(s => Math.max(s.mtimeMs, s.ctimeMs)).catch(() => 0);
-    const srcTime = await fs.stat(src).then(s => Math.max(s.mtimeMs, s.ctimeMs));
-    // If stale
-    if (srcTime > targetTime) {
-      const ts = (await import('typescript')).default;
-      const module = ctx.moduleType === 'module' ? ts.ModuleKind.ESNext : ts.ModuleKind.CommonJS;
-      await fs.mkdir(path.dirname(target), { recursive: true });
-      const text = await fs.readFile(src, 'utf8');
-      if (/[.]tsx?$/.test(file)) {
-        const content = ts.transpile(
-          text,
-          { target: ts.ScriptTarget.ES2022, module, esModuleInterop: true, allowSyntheticDefaultImports: true }
-        )
-          .replace(/^((?:im|ex)port .*from '[.][^']+)(')/mg, (_, a, b) => `${a}.js${b}`)
-          .replace(/^(import [^\n]*from '[^.][^\n/]+[/][^\n/]+[/][^\n']+)(')/mg, (_, a, b) => `${a}.js${b}`);
-        await fs.writeFile(target, content, 'utf8');
-      } else {
-        const pkg = JSON.parse(text);
-        pkg.type = ctx.moduleType;
-        await fs.writeFile(target, JSON.stringify(pkg, null, 2), 'utf8');
-      }
-      // Compile
+const TS_EXT = /[.]tsx?$/;
+
+const getAge = (/** @type {{mtimeMs:number, ctimeMs:number}} */ st) => Math.max(st.mtimeMs, st.ctimeMs);
+
+const getTarget = (/** @type {Ctx} */ ctx, file = '') => ({
+  dest: path.resolve(ctx.workspace.path, ctx.build.compilerFolder, 'node_modules', '@travetto/compiler', file).replace(TS_EXT, '.js'),
+  src: path.resolve(ctx.workspace.path, ctx.build.compilerModuleFolder, file),
+  async writeIfStale(/** @type {(text:string)=>(string|Promise<string>)}*/ transform) {
+    if (!existsSync(this.dest) || getAge(statSync(this.dest)) < getAge(statSync(this.src))) {
+      const text = readFileSync(this.src, 'utf8');
+      mkdirSync(path.dirname(this.dest), { recursive: true });
+      writeFileSync(this.dest, await transform(text), 'utf8');
     }
-    files.push(target);
   }
+});
+
+const getTranspiler = async (/** @type {Ctx} */ ctx) => {
+  const ts = (await import('typescript')).default;
+  const module = ctx.workspace.type === 'module' ? ts.ModuleKind.ESNext : ts.ModuleKind.CommonJS;
+  return (content = '') => ts.transpile(content, { target: ts.ScriptTarget.ES2022, module, esModuleInterop: true, allowSyntheticDefaultImports: true });
+};
+
+/** @returns {Promise<import('@travetto/compiler/support/entry.trvc')>} */
+async function imp(f = '') { try { return require(f); } catch (err) { return import(f); } }
 
-  const rootCtx = await (ctx.monoRepo ? getManifestContext(ctx.workspacePath) : ctx);
+export async function getEntry() {
+  process.setSourceMapsEnabled(true); // Ensure source map during compilation/development
+  process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --enable-source-maps`; // Ensure it passes to children
 
-  try { return require(files[0]).main(rootCtx, ctx); }
-  catch { return import(files[0]).then(v => v.main(rootCtx, ctx)); }
-};
+  const ctx = getManifestContext();
+  const target = getTarget.bind(null, ctx);
+
+  // Setup Tsconfig
+  const tsconfig = path.resolve(ctx.workspace.path, 'tsconfig.json');
+  existsSync(tsconfig) || writeFileSync(tsconfig, JSON.stringify({ extends: '@travetto/compiler/tsconfig.trv.json' }), 'utf8');
+
+  // Compile support folder
+  await target('package.json').writeIfStale(text => JSON.stringify(Object.assign(JSON.parse(text), { type: ctx.workspace.type }), null, 2));
+
+  let transpile;
+  for (const file of readdirSync(target('support').src, { recursive: true, encoding: 'utf8' })) {
+    if (TS_EXT.test(file)) { await target(`support/${file}`).writeIfStale(async (text) => (transpile ??= await getTranspiler(ctx))(text)); }
+  }
+
+  // Load
+  try {
+    return await imp(target('support/entry.trvc.ts').dest).then(v => v.main(ctx));
+  } catch (err) {
+    rmSync(target('.').dest, { recursive: true, force: true });
+    throw err;
+  }
+}

package/bin/trvc.js

@@ -7,30 +7,37 @@ const help = () => [
   'npx trvc [command]',
   '',
   'Available Commands:',
-  ' * start|watch               - Run the compiler in watch mode',
-  ' * stop                      - Stop the compiler if running',
-  ' * restart                   - Restart the compiler in watch mode',
-  ' * build                     - Ensure the project is built and upto date',
-  ' * clean                     - Clean out the output and compiler caches',
-  ' * info                      - Retrieve the compiler information, if running',
-  ' * manifest [output] [prod]  - Generate the project manifest',
+  ' * start|watch                - Run the compiler in watch mode',
+  ' * stop                       - Stop the compiler if running',
+  ' * restart                    - Restart the compiler in watch mode',
+  ' * build                      - Ensure the project is built and upto date',
+  ' * clean                      - Clean out the output and compiler caches',
+  ' * info                       - Retrieve the compiler information, if running',
+  ' * event <log|progress|state> - Watch events in realtime as newline delimited JSON',
+  ' * manifest --prod [output]   - Generate the project manifest',
 ].join('\n');
 
 getEntry().then(async (ops) => {
-  const [op, ...args] = process.argv.slice(2);
-  const baseArgs = args.filter(x => !x.startsWith('-'));
+  const [op, ...all] = process.argv.slice(2);
+  const args = all.filter(x => !x.startsWith('-'));
+  const flags = all.filter(x => x.startsWith('-'));
 
   switch (op) {
     case undefined:
     case 'help': return console.log(`\n${help()}\n`);
-    case 'restart': return ops.stop().then(() => ops.compile('watch'));
+    case 'restart': return ops.restart();
     case 'stop': return ops.stop();
     case 'info': return ops.info().then(v => console.log(JSON.stringify(v, null, 2)));
+    case 'event': return ops.events(args[0], v => {
+      if (!process.stdout.write(`${JSON.stringify(v)}\n`)) {
+        return new Promise(r => process.stdout.once('drain', r));
+      }
+    });
     case 'clean': return ops.clean();
-    case 'manifest': return ops.manifest(baseArgs[0], /prod/i.test(baseArgs[1] ?? ''));
-    case 'start': return ops.compile('watch');
-    case 'watch':
-    case 'build': return ops.compile(op);
+    case 'manifest': return ops.manifest(args[0], flags.some(x => x === '--prod'));
+    case 'start':
+    case 'watch': return ops.watch();
+    case 'build': return ops.build();
     default: console.error(`Unknown trvc operation: ${op}\n`); return console.error(help());
   }
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/compiler",
-  "version": "3.4.4",
+  "version": "4.0.0-rc.1",
   "description": "The compiler infrastructure for the Travetto framework",
   "keywords": [
     "compiler",
@@ -30,14 +30,13 @@
     "directory": "module/compiler"
   },
   "dependencies": {
-    "@parcel/watcher": "^2.3.0",
-    "@travetto/manifest": "^3.4.0",
-    "@travetto/terminal": "^3.4.0",
-    "@travetto/transformer": "^3.4.3",
-    "@types/node": "^20.9.2"
+    "@parcel/watcher": "^2.4.0",
+    "@travetto/manifest": "^4.0.0-rc.1",
+    "@travetto/transformer": "^4.0.0-rc.1",
+    "@types/node": "^20.11.16"
   },
   "peerDependencies": {
-    "@travetto/cli": "^3.4.6"
+    "@travetto/cli": "^4.0.0-rc.1"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/compiler.ts

@@ -1,8 +1,9 @@
-import { install } from 'source-map-support';
 import ts from 'typescript';
-import fs from 'fs/promises';
+import timers from 'node:timers/promises';
+import fs from 'node:fs/promises';
+import { setMaxListeners } from 'node:events';
 
-import { ManifestModuleUtil, RootIndex } from '@travetto/manifest';
+import { ManifestModuleUtil, RuntimeIndex } from '@travetto/manifest';
 
 import { CompilerUtil } from './util';
 import { CompilerState } from './state';
@@ -21,22 +22,65 @@ export class Compiler {
    */
   static async main(): Promise<void> {
     const [dirty, watch] = process.argv.slice(2);
-    const state = await CompilerState.get(RootIndex);
+    const state = await CompilerState.get(RuntimeIndex);
+    Log.debug('Running compiler with dirty file', dirty);
     const dirtyFiles = ManifestModuleUtil.getFileType(dirty) === 'ts' ? [dirty] : (await fs.readFile(dirty, 'utf8')).split(/\n/).filter(x => !!x);
     await new Compiler(state, dirtyFiles, watch === 'true').run();
-    process.exit();
   }
 
   #state: CompilerState;
   #dirtyFiles: string[];
   #watch?: boolean;
+  #ctrl: AbortController;
+  #signal: AbortSignal;
+  #shuttingDown = false;
 
   constructor(state: CompilerState, dirtyFiles: string[], watch?: boolean) {
     this.#state = state;
     this.#dirtyFiles = dirtyFiles[0] === '*' ?
       this.#state.getAllFiles() :
-      dirtyFiles.map(f => this.#state.getBySource(f)!.input);
+      dirtyFiles.map(f => this.#state.getBySource(f)!.inputFile);
     this.#watch = watch;
+
+    this.#ctrl = new AbortController();
+    this.#signal = this.#ctrl.signal;
+    setMaxListeners(1000, this.#signal);
+    process
+      .once('disconnect', () => this.#shutdown('manual'))
+      .on('message', ev => (ev === 'shutdown') && this.#shutdown('manual'));
+  }
+
+  #shutdown(mode: 'error' | 'manual' | 'complete' | 'reset', err?: Error): void {
+    if (this.#shuttingDown) {
+      return;
+    }
+
+    this.#shuttingDown = true;
+    switch (mode) {
+      case 'manual': {
+        Log.error('Shutting down manually');
+        process.exitCode = 2;
+        break;
+      }
+      case 'error': {
+        process.exitCode = 1;
+        if (err) {
+          Log.error('Shutting down due to failure', err.message);
+        }
+        break;
+      }
+      case 'reset': {
+        Log.info('Triggering reset due to change in core files');
+        EventUtil.sendEvent('state', { state: 'reset' });
+        process.exitCode = 0;
+        break;
+      }
+    }
+    // No longer listen to disconnect
+    process.removeAllListeners('disconnect');
+    process.removeAllListeners('message');
+    this.#ctrl.abort();
+    setTimeout(() => process.exit(), 1000).unref(); // Allow upto 1s to shutdown gracefully
   }
 
   /**
@@ -50,6 +94,7 @@ export class Compiler {
         if (needsNewProgram) {
           program = this.#state.createProgram(program);
         }
+        await timers.setImmediate();
         const result = this.#state.writeInputFile(program, inputFile);
         if (result?.diagnostics?.length) {
           return result.diagnostics;
@@ -72,6 +117,7 @@ export class Compiler {
   async * emit(files: string[], emitter: CompileEmitter): AsyncIterable<CompileEmitEvent> {
     let i = 0;
     let lastSent = Date.now();
+
     for (const file of files) {
       const err = await emitter(file);
       const imp = file.replace(/.*node_modules\//, '');
@@ -80,8 +126,14 @@ export class Compiler {
         lastSent = Date.now();
         EventUtil.sendEvent('progress', { total: files.length, idx: i, message: imp, operation: 'compile' });
       }
+      if (this.#signal.aborted) {
+        break;
+      }
     }
     EventUtil.sendEvent('progress', { total: files.length, idx: files.length, message: 'Complete', operation: 'compile', complete: true });
+
+    await timers.setTimeout(1);
+
     Log.debug(`Compiled ${i} files`);
   }
 
@@ -89,16 +141,10 @@ export class Compiler {
    * Run the compiler
    */
   async run(): Promise<void> {
-    install();
-
     Log.debug('Compilation started');
 
     EventUtil.sendEvent('state', { state: 'init', extra: { pid: process.pid } });
 
-    if (process.send) {
-      process.on('disconnect', () => process.exit(0));
-    }
-
     const emitter = await this.getCompiler();
     let failed = false;
 
@@ -114,9 +160,12 @@ export class Compiler {
           EventUtil.sendEvent('log', { level: 'error', message: compileError.toString(), time: Date.now() });
         }
       }
-      if (failed) {
+      if (this.#signal.aborted) {
+        Log.debug('Compilation aborted');
+      } else if (failed) {
         Log.debug('Compilation failed');
-        process.exit(1);
+        process.exitCode = 1;
+        return;
       } else {
         Log.debug('Compilation succeeded');
       }
@@ -128,43 +177,47 @@ export class Compiler {
 
     EventUtil.sendEvent('state', { state: 'compile-end' });
 
-    if (this.#watch) {
+    if (this.#watch && !this.#signal.aborted) {
       Log.info('Watch is ready');
 
       EventUtil.sendEvent('state', { state: 'watch-start' });
-
-      for await (const ev of CompilerWatcher.watch(this.#state)) {
-        if (ev.action === 'reset') {
-          Log.info(`Triggering reset due to change in ${ev.file}`);
-          EventUtil.sendEvent('state', { state: 'reset' });
-          return;
-        }
-        const { action, entry } = ev;
-        if (action !== 'delete') {
-          const err = await emitter(entry.input, true);
-          if (err) {
-            Log.info('Compilation Error', CompilerUtil.buildTranspileError(entry.input, err));
+      try {
+        for await (const ev of new CompilerWatcher(this.#state, this.#signal).watchChanges()) {
+          if (ev.action !== 'delete') {
+            const err = await emitter(ev.entry.inputFile, true);
+            if (err) {
+              Log.info('Compilation Error', CompilerUtil.buildTranspileError(ev.entry.inputFile, err));
+            } else {
+              Log.info(`Compiled ${ev.entry.sourceFile} on ${ev.action}`);
+            }
           } else {
-            Log.info(`Compiled ${entry.source}`);
+            if (ev.entry.outputFile) {
+              // Remove output
+              Log.info(`Removed ${ev.entry.sourceFile}, ${ev.entry.outputFile}`);
+              await fs.rm(ev.entry.outputFile, { force: true }); // Ensure output is deleted
+            }
           }
-        } else {
-          // Remove output
-          Log.info(`Removed ${entry.source}, ${entry.output}`);
-          await fs.rm(entry.output!, { force: true }); // Ensure output is deleted
+
+          // Send change events
+          EventUtil.sendEvent('change', {
+            action: ev.action,
+            time: Date.now(),
+            file: ev.file,
+            output: ev.entry.outputFile!,
+            module: ev.entry.module.name
+          });
         }
+        EventUtil.sendEvent('state', { state: 'watch-end' });
 
-        // Send change events
-        EventUtil.sendEvent('change', {
-          action: ev.action,
-          time: Date.now(),
-          file: ev.file,
-          folder: ev.folder,
-          output: ev.entry.output!,
-          module: ev.entry.module.name
-        });
+      } catch (err) {
+        if (err instanceof Error) {
+          this.#shutdown(err.message === 'RESET' ? 'reset' : 'error', err);
+        }
       }
-
-      EventUtil.sendEvent('state', { state: 'watch-end' });
     }
+
+    Log.debug('Compiler process shutdown');
+
+    this.#shutdown('complete');
   }
 }

package/src/event.ts

@@ -1,7 +1,7 @@
-import type { CompilerServerEvent, CompilerServerEventType } from '../support/types';
+import type { CompilerEvent, CompilerEventType } from '../support/types';
 
 export class EventUtil {
-  static sendEvent<K extends CompilerServerEventType, T extends CompilerServerEvent & { type: K }>(type: K, payload: T['payload']): void {
-    process.send?.({ type, payload });
+  static sendEvent<K extends CompilerEventType, T extends CompilerEvent & { type: K }>(type: K, payload: T['payload']): void {
+    process.connected && process.send!({ type, payload }, undefined, { swallowErrors: true });
   }
 }

package/src/log.ts

@@ -3,7 +3,7 @@ import { EventUtil } from './event';
 
 function log(level: CompilerLogLevel, message: string, ...args: unknown[]): void {
   EventUtil.sendEvent('log', { level, message, args, time: Date.now(), scope: 'compiler-exec' });
-  if (!process.send) {
+  if (!process.connected) {
     // eslint-disable-next-line no-console
     console[level](message, ...args);
   }