@travetto/compiler 3.0.0-rc.3 → 3.0.0-rc.31

package/support/lock-pinger.ts

@@ -0,0 +1,25 @@
+import { workerData, parentPort } from 'worker_threads';
+import { utimesSync } from 'fs';
+
+const data: { files: string[], interval: number } = workerData;
+const files = data.files;
+
+const interval = setInterval(() => {
+  const now = Date.now() / 1000;
+  for (const file of files) {
+    try {
+      utimesSync(file, now, now);
+    } catch { }
+  }
+}, data.interval);
+
+
+parentPort?.on('message', val => {
+  if (val === 'stop') {
+    files.splice(0, files.length);
+    clearInterval(interval);
+  } else if (val && typeof val === 'object' && 'files' in val && Array.isArray(val.files)) {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    files.splice(0, files.length, ...val.files as string[]);
+  }
+});

package/support/lock.ts

@@ -0,0 +1,234 @@
+import { Stats, watchFile, unwatchFile, rmSync, mkdirSync, writeFileSync, existsSync } from 'fs';
+import fs from 'fs/promises';
+import { Worker } from 'worker_threads';
+import path from 'path';
+
+import type { ManifestContext } from '@travetto/manifest';
+
+import { CompilerLogger, LogUtil } from './log';
+
+type LockStatus = 'complete' | 'stale';
+type LockDetails = {
+  pid: number | undefined;
+  file: string;
+};
+
+export type LockType = 'build' | 'watch';
+export type LockCompileAction = 'skip' | 'build';
+type LockAction = LockCompileAction | 'retry';
+
+const STALE_THRESHOLD = 1000;
+
+/**
+ * Manager for all lock activity
+ */
+export class LockManager {
+
+  /**
+   * Get the lock file name
+   */
+  static #getFileName(ctx: ManifestContext, type: LockType): string {
+    return path.resolve(ctx.workspacePath, ctx.toolFolder, `${type}.lock`);
+  }
+
+  /**
+   * Determine if the given stats are stale for modification time
+   */
+  static #isStale(stat?: Stats): boolean {
+    return !!stat && stat.mtimeMs < (Date.now() - STALE_THRESHOLD * 1.1);
+  }
+
+  /**
+   * Get the lock file details
+   */
+  static async #getDetails(ctx: ManifestContext, type: LockType): Promise<LockDetails> {
+    const file = this.#getFileName(ctx, type);
+    const stat = await fs.stat(file).catch(() => undefined);
+    const stale = this.#isStale(stat);
+    let pid: number | undefined;
+    if (stat) {
+      const { pid: filePid } = JSON.parse(await fs.readFile(file, 'utf8'));
+      if (stale) {
+        LogUtil.log('lock', [], 'debug', `${type} file is stale: ${stat.mtimeMs} vs ${Date.now()}`);
+      } else {
+        pid = filePid;
+      }
+    }
+    return { pid, file };
+  }
+
+  /**
+   * Acquire the lock file, and register a cleanup on exit
+   */
+  static #acquireFile(ctx: ManifestContext, type: LockType): void {
+    const file = this.#getFileName(ctx, type);
+    mkdirSync(path.dirname(file), { recursive: true });
+    LogUtil.log('lock', [], 'debug', `Acquiring ${type}`);
+    writeFileSync(file, JSON.stringify({ pid: process.pid }), 'utf8');
+  }
+
+  /**
+   * Release the lock file (i.e. deleting)
+   */
+  static #releaseFile(ctx: ManifestContext, type: LockType): void {
+    const file = this.#getFileName(ctx, type);
+    if (existsSync(file)) {
+      rmSync(file, { force: true });
+      LogUtil.log('lock', [], 'debug', `Releasing ${type}`);
+    }
+  }
+
+  /**
+   * Wait until a lock file is released, or it becomes stale
+   */
+  static async #waitForRelease(ctx: ManifestContext, type: LockType): Promise<LockStatus> {
+    const file = this.#getFileName(ctx, type);
+    let remove: (() => void) | undefined = undefined;
+
+    const prom = new Promise<LockStatus>(resolve => {
+      let timer: NodeJS.Timeout | undefined = undefined;
+      const handler = async (): Promise<void> => {
+        if (timer) {
+          clearTimeout(timer);
+        }
+        const stats = await fs.stat(file).catch(() => undefined);
+        if (!stats) {
+          resolve('complete');
+        } else if (this.#isStale(stats)) {
+          resolve('stale');
+        } else {
+          timer = setTimeout(handler, STALE_THRESHOLD * 1.1);
+        }
+      };
+
+      watchFile(file, handler);
+      handler();
+
+      remove = (): void => {
+        clearInterval(timer);
+        unwatchFile(file, handler);
+      };
+    });
+
+    return prom.finally(remove);
+  }
+
+  /**
+   * Read the watch lock file and determine its result, communicating with the user as necessary
+   */
+  static async #getWatchAction(ctx: ManifestContext, log: CompilerLogger, lockType: LockType | undefined, buildState: LockDetails): Promise<LockAction> {
+    if (lockType === 'watch') {
+      log('info', 'Already running');
+      return 'skip';
+    } else {
+      if (buildState.pid) {
+        log('warn', 'Already running, waiting for build to finish');
+        switch (await this.#waitForRelease(ctx, 'build')) {
+          case 'complete': {
+            log('info', 'Completed build');
+            return 'skip';
+          }
+          case 'stale': {
+            log('info', 'Became stale, retrying');
+            return 'retry';
+          }
+        }
+      } else {
+        log('info', 'Already running, and has built');
+        return 'skip';
+      }
+    }
+  }
+
+  /**
+   * Read the build lock file and determine its result, communicating with the user as necessary
+   */
+  static async #getBuildAction(ctx: ManifestContext, log: CompilerLogger, lockType: LockType | undefined): Promise<LockAction> {
+    if (lockType === 'watch') {
+      log('warn', 'Build already running, waiting to begin watch');
+      const res = await this.#waitForRelease(ctx, 'build');
+      log('info', `Finished with status of ${res}, retrying`);
+      return 'retry';
+    } else {
+      log('warn', 'Already running, waiting for completion');
+      switch (await this.#waitForRelease(ctx, lockType ?? 'build')) {
+        case 'complete': {
+          log('info', 'Completed');
+          return 'skip';
+        }
+        case 'stale': {
+          log('info', 'Became stale, retrying');
+          return 'retry';
+        }
+      }
+    }
+  }
+
+  /**
+   * Run code with support for lock acquire and release
+   */
+  static async withLocks(ctx: ManifestContext, fn: (acquire: (type: LockType) => void, release: (type: LockType) => void) => Promise<unknown>): Promise<void> {
+    const activeLockTypes = new Set<LockType>();
+
+    const pinger = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', '@travetto/compiler/support/lock-pinger.js');
+    const worker = new Worker(pinger, {
+      workerData: {
+        interval: STALE_THRESHOLD,
+        files: []
+      }
+    });
+
+    const notify = (): void => worker.postMessage({ files: [...activeLockTypes].map(t => this.#getFileName(ctx, t)) });
+
+    const stop = (): void => {
+      worker.postMessage('stop');
+      for (const type of activeLockTypes) {
+        this.#releaseFile(ctx, type);
+      }
+      worker.terminate().then(() => { });
+    };
+
+    process.on('SIGINT', stop);
+    process.on('exit', stop);
+
+    try {
+      await new Promise(r => worker.on('online', r));
+      await fn(
+        type => {
+          if (!activeLockTypes.has(type)) {
+            activeLockTypes.add(type);
+            this.#acquireFile(ctx, type);
+            notify();
+          }
+        },
+        type => {
+          if (activeLockTypes.has(type)) {
+            activeLockTypes.delete(type);
+            this.#releaseFile(ctx, type);
+            notify();
+          }
+        }
+      );
+    } finally {
+      stop();
+    }
+  }
+
+  /**
+   * Reads the lock file states (build + watch) to determine what action should be taken for the compiler
+   */
+  static async getCompileAction(ctx: ManifestContext, lockType: LockType | undefined): Promise<LockCompileAction> {
+    let result: LockAction;
+    do {
+      result = 'build';
+      const buildState = await this.#getDetails(ctx, 'build');
+      const watchState = await this.#getDetails(ctx, 'watch');
+      if (watchState.pid) { // Existing watch operation
+        result = await LogUtil.withLogger('lock', log => this.#getWatchAction(ctx, log, lockType, buildState), true, ['watch', `pid=${watchState.pid}`]);
+      } else if (buildState.pid) { // Existing build operation
+        result = await LogUtil.withLogger('lock', log => this.#getBuildAction(ctx, log, lockType), true, ['build', `pid=${buildState.pid}`]);
+      }
+    } while (result === 'retry');
+    return result;
+  }
+}

package/support/log.ts

@@ -0,0 +1,51 @@
+export type CompilerLogEvent = [level: 'info' | 'debug' | 'warn', message: string];
+export type CompilerLogger = (...args: CompilerLogEvent) => void;
+export type WithLogger<T> = (log: CompilerLogger) => Promise<T>;
+
+const SCOPE_MAX = 15;
+
+export class LogUtil {
+
+  static levels: {
+    debug: boolean;
+    info: boolean;
+    warn: boolean;
+  };
+
+  static set level(value: string) {
+    this.levels = {
+      warn: /^(debug|info|warn)$/.test(value),
+      info: /^(debug|info)$/.test(value),
+      debug: /^debug$/.test(value),
+    };
+  }
+
+  /**
+   * Is object a log event
+   */
+  static isLogEvent = (o: unknown): o is CompilerLogEvent => o !== null && o !== undefined && Array.isArray(o);
+
+  /**
+   * Log message with filtering by level
+   */
+  static log(scope: string, args: string[], ...[level, msg]: CompilerLogEvent): void {
+    const message = msg.replaceAll(process.cwd(), '.');
+    if (LogUtil.levels[level]) {
+      const params = [`[${scope.padEnd(SCOPE_MAX, ' ')}]`, ...args, message];
+      if (!/(0|false|off|no)$/i.test(process.env.TRV_LOG_TIME ?? '')) {
+        params.unshift(new Date().toISOString());
+      }
+      // eslint-disable-next-line no-console
+      console[level]!(...params);
+    }
+  }
+
+  /**
+   * With logger
+   */
+  static withLogger<T>(scope: string, op: WithLogger<T>, basic = true, args: string[] = []): Promise<T> {
+    const log = this.log.bind(null, scope, args);
+    basic && log('debug', 'Started');
+    return op(log).finally(() => basic && log('debug', 'Completed'));
+  }
+}

package/support/transpile.ts

@@ -0,0 +1,210 @@
+import path from 'path';
+import fs from 'fs/promises';
+import os from 'os';
+import cp from 'child_process';
+import { createRequire } from 'module';
+
+import { DeltaEvent, ManifestContext, ManifestRoot, Package } from '@travetto/manifest';
+
+import { LogUtil } from './log';
+
+type ModFile = { input: string, output: string, stale: boolean };
+export type CompileResult = 'restart' | 'complete' | 'skipped';
+
+const OPT_CACHE: Record<string, import('typescript').CompilerOptions> = {};
+const SRC_REQ = createRequire(path.resolve('node_modules'));
+const RECENT_STAT = (stat: { ctimeMs: number, mtimeMs: number }): number => Math.max(stat.ctimeMs, stat.mtimeMs);
+
+/**
+ * Transpile utilities for launching
+ */
+export class TranspileUtil {
+  /**
+   * Write text file, and ensure folder exists
+   */
+  static writeTextFile = (file: string, content: string): Promise<void> =>
+    fs.mkdir(path.dirname(file), { recursive: true }).then(() => fs.writeFile(file, content, 'utf8'));
+
+  /**
+   * Returns the compiler options
+   */
+  static async getCompilerOptions(ctx: ManifestContext): Promise<{}> {
+    if (!(ctx.workspacePath in OPT_CACHE)) {
+      let tsconfig = path.resolve(ctx.workspacePath, 'tsconfig.json');
+
+      if (!await fs.stat(tsconfig).then(_ => true, _ => false)) {
+        tsconfig = SRC_REQ.resolve('@travetto/compiler/tsconfig.trv.json');
+      }
+
+      const ts = (await import('typescript')).default;
+
+      const { options } = ts.parseJsonSourceFileConfigFileContent(
+        ts.readJsonConfigFile(tsconfig, ts.sys.readFile), ts.sys, ctx.workspacePath
+      );
+
+      OPT_CACHE[ctx.workspacePath] = {
+        ...options,
+        allowJs: true,
+        sourceMap: false,
+        inlineSourceMap: true,
+        resolveJsonModule: true,
+        sourceRoot: ctx.workspacePath,
+        rootDir: ctx.workspacePath,
+        outDir: path.resolve(ctx.workspacePath),
+        module: ctx.moduleType === 'commonjs' ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext,
+      };
+    }
+    return OPT_CACHE[ctx.workspacePath];
+  }
+
+  /**
+   * Output a file, support for ts, js, and package.json
+   */
+  static async transpileFile(ctx: ManifestContext, inputFile: string, outputFile: string): Promise<void> {
+    if (inputFile.endsWith('.ts') || inputFile.endsWith('.js')) {
+      const compilerOut = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules');
+
+      const text = (await fs.readFile(inputFile, 'utf8'))
+        .replace(/from '([.][^']+)'/g, (_, i) => `from '${i.replace(/[.]js$/, '')}.js'`)
+        .replace(/from '(@travetto\/(.*?))'/g, (_, i, s) => `from '${path.resolve(compilerOut, `${i}${s.includes('/') ? '.js' : '/__index__.js'}`)}'`);
+
+      const ts = (await import('typescript')).default;
+      const content = ts.transpile(text, await this.getCompilerOptions(ctx), inputFile);
+      await this.writeTextFile(outputFile, content);
+    } else if (inputFile.endsWith('package.json')) {
+      const pkg: Package = JSON.parse(await fs.readFile(inputFile, 'utf8'));
+      const main = pkg.main?.replace(/[.]ts$/, '.js');
+      const files = pkg.files?.map(x => x.replace('.ts', '.js'));
+
+      const content = JSON.stringify({ ...pkg, main, type: ctx.moduleType, files }, null, 2);
+      await this.writeTextFile(outputFile, content);
+    }
+  }
+
+  /**
+   * Scan directory to find all project sources for comparison
+   */
+  static async getModuleSources(ctx: ManifestContext, module: string, seed: string[]): Promise<ModFile[]> {
+    const inputFolder = (ctx.mainModule === module) ?
+      process.cwd() :
+      path.dirname(SRC_REQ.resolve(`${module}/package.json`));
+
+    const folders = seed.filter(x => !/[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+    const files = seed.filter(x => /[.]/.test(x)).map(x => path.resolve(inputFolder, x));
+
+    while (folders.length) {
+      const sub = folders.pop();
+      if (!sub) {
+        continue;
+      }
+
+      for (const file of await fs.readdir(sub).catch(() => [])) {
+        if (file.startsWith('.')) {
+          continue;
+        }
+        const resolvedInput = path.resolve(sub, file);
+        const stat = await fs.stat(resolvedInput);
+
+        if (stat.isDirectory()) {
+          folders.push(resolvedInput);
+        } else if (file.endsWith('.d.ts')) {
+          // Do nothing
+        } else if (file.endsWith('.ts') || file.endsWith('.js')) {
+          files.push(resolvedInput);
+        }
+      }
+    }
+
+    const outputFolder = path.resolve(ctx.workspacePath, ctx.compilerFolder, 'node_modules', module);
+    const out: ModFile[] = [];
+    for (const input of files) {
+      const output = input.replace(inputFolder, outputFolder).replace(/[.]ts$/, '.js');
+      const inputTs = await fs.stat(input).then(RECENT_STAT, () => 0);
+      if (inputTs) {
+        const outputTs = await fs.stat(output).then(RECENT_STAT, () => 0);
+        await fs.mkdir(path.dirname(output), { recursive: true, });
+        out.push({ input, output, stale: inputTs > outputTs });
+      }
+    }
+
+    return out;
+  }
+
+  /**
+   * Recompile folder if stale
+   */
+  static async compileIfStale(ctx: ManifestContext, scope: string, mod: string, seed: string[]): Promise<string[]> {
+    const files = await this.getModuleSources(ctx, mod, seed);
+    const changes = files.filter(x => x.stale).map(x => x.input);
+    const out: string[] = [];
+
+    try {
+      await LogUtil.withLogger(scope, async log => {
+        if (files.some(f => f.stale)) {
+          log('debug', 'Starting');
+          for (const file of files.filter(x => x.stale)) {
+            await this.transpileFile(ctx, file.input, file.output);
+          }
+          if (changes.length) {
+            out.push(...changes.map(x => `${mod}/${x}`));
+            log('debug', `Source changed: ${changes.join(', ')}`);
+          }
+          log('debug', 'Completed');
+        } else {
+          log('debug', 'Skipped');
+        }
+      }, false, [mod]);
+    } catch (err) {
+      console.error(err);
+    }
+    return out;
+  }
+
+  /**
+   * Run compiler
+   */
+  static async runCompiler(ctx: ManifestContext, manifest: ManifestRoot, changed: DeltaEvent[], watch: boolean, onMessage: (msg: unknown) => void): Promise<CompileResult> {
+    const compiler = path.resolve(ctx.workspacePath, ctx.compilerFolder);
+    const main = path.resolve(compiler, 'node_modules', '@travetto/compiler/support/compiler-entry.js');
+    const deltaFile = path.resolve(os.tmpdir(), `manifest-delta.${process.pid}.${process.ppid}.${Date.now()}.json`);
+
+    const changedFiles = changed[0]?.file === '*' ? ['*'] : changed.map(ev =>
+      path.resolve(manifest.workspacePath, manifest.modules[ev.module].sourceFolder, ev.file)
+    );
+
+    let proc: cp.ChildProcess | undefined;
+    let kill: (() => void) | undefined;
+
+    try {
+      await this.writeTextFile(deltaFile, changedFiles.join('\n'));
+
+      return await LogUtil.withLogger('compiler-exec', log => new Promise<CompileResult>((res, rej) => {
+        proc = cp.spawn(process.argv0, [main, deltaFile, `${watch}`], {
+          env: {
+            ...process.env,
+            TRV_MANIFEST: path.resolve(ctx.workspacePath, ctx.outputFolder, 'node_modules', ctx.mainModule),
+          },
+          stdio: [0, 1, 2, 'ipc'],
+        })
+          .on('message', msg => {
+            if (LogUtil.isLogEvent(msg)) {
+              log(...msg);
+            } else if (msg === 'restart') {
+              res(msg);
+            } else {
+              onMessage(msg);
+            }
+          })
+          .on('exit', code => (code !== null && code > 0) ? rej(new Error('Failed during compilation')) : res('complete'));
+        kill = (): void => { proc?.kill('SIGKILL'); };
+        process.on('exit', kill);
+      }));
+    } finally {
+      if (proc?.killed === false) { proc.kill('SIGKILL'); }
+      if (kill) {
+        process.removeListener('exit', kill);
+      }
+      await fs.rm(deltaFile, { force: true });
+    }
+  }
+}

package/tsconfig.trv.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "module": "CommonJS",
+    "target": "esnext",
+    "moduleResolution": "node",
+    "lib": [
+      "es2022"
+    ],
+    "strict": true,
+    "esModuleInterop": true,
+    "strictPropertyInitialization": false,
+    "experimentalDecorators": true,
+    "noEmitOnError": false,
+    "noErrorTruncation": true,
+    "resolveJsonModule": true,
+    "sourceMap": true,
+    "inlineSourceMap": false,
+    "removeComments": true,
+    "importHelpers": true,
+    "noEmitHelpers": true,
+    "outDir": "build/"
+  },
+}

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2020 ArcSine Technologies
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/index.ts

@@ -1,3 +0,0 @@
-export * from './src/host';
-export * from './src/compiler';
-export * from './src/transformer';