@travetto/runtime 7.0.0-rc.2 → 7.0.0-rc.4

package/README.md

@@ -19,6 +19,7 @@ Runtime is the foundation of all [Travetto](https://travetto.dev) applications.
    *  Standard Error Support
    *  Console Management
    *  Resource Access
+   *  JSON Utilities
    *  Common Utilities
    *  Time Utilities
    *  Process Execution
@@ -36,10 +37,10 @@ class $Runtime {
   constructor(idx: ManifestIndex, resourceOverrides?: Record<string, string>);
   /** Get env name, with support for the default env */
   get env(): string | undefined;
-  /** Are we in development mode */
+  /** Are we in production mode */
   get production(): boolean;
-  /** Is the app in dynamic mode? */
-  get dynamic(): boolean;
+  /** Get environment type mode */
+  get envType(): 'production' | 'development' | 'test';
   /** Get debug value */
   get debug(): false | string;
   /** Manifest main */
@@ -95,10 +96,6 @@ interface EnvData {
      * Special role to run as, used to access additional files from the manifest during runtime.  
      */
     TRV_ROLE: Role;
-    /** 
-     * Whether or not to run the program in dynamic mode, allowing for real-time updates  
-     */
-    TRV_DYNAMIC: boolean;
     /** 
      * The folders to use for resource lookup
      */
@@ -258,9 +255,18 @@ $ DEBUG=express:*,@travetto/web npx trv run web
 ## Resource Access
 The primary access patterns for resources, is to directly request a file, and to resolve that file either via file-system look up or leveraging the [Manifest](https://github.com/travetto/travetto/tree/main/module/manifest#readme "Support for project indexing, manifesting, along with file watching")'s data for what resources were found at manifesting time.
 
-The [FileLoader](https://github.com/travetto/travetto/tree/main/module/runtime/src/file-loader.ts#L11) allows for accessing information about the resources, and subsequently reading the file as text/binary or to access the resource as a `Readable` stream.  If a file is not found, it will throw an [AppError](https://github.com/travetto/travetto/tree/main/module/runtime/src/error.ts#L26) with a category of 'notfound'.  
+The [FileLoader](https://github.com/travetto/travetto/tree/main/module/runtime/src/file-loader.ts#L12) allows for accessing information about the resources, and subsequently reading the file as text/binary or to access the resource as a `Readable` stream.  If a file is not found, it will throw an [AppError](https://github.com/travetto/travetto/tree/main/module/runtime/src/error.ts#L26) with a category of 'notfound'.  
+
+The [FileLoader](https://github.com/travetto/travetto/tree/main/module/runtime/src/file-loader.ts#L12) also supports tying itself to [Env](https://github.com/travetto/travetto/tree/main/module/runtime/src/env.ts#L114)'s `TRV_RESOURCES` information on where to attempt to find a requested resource.
+
+## JSON Utilities
+The framework provides utilities for working with JSON data.  This module provides methods for reading and writing JSON files, as well as serializing and deserializing JSON data.  It also provides support for working with Base64 encoded data for web safe transfer.  The primary goal is ease of use, but also a centralized location for performance and security improvements over time.
 
-The [FileLoader](https://github.com/travetto/travetto/tree/main/module/runtime/src/file-loader.ts#L11) also supports tying itself to [Env](https://github.com/travetto/travetto/tree/main/module/runtime/src/env.ts#L114)'s `TRV_RESOURCES` information on where to attempt to find a requested resource.
+   *  `parseSafe(input: string | Buffer)` parses JSON safely from a string or Buffer.
+   *  `stringifyBase64(value: any)` encodes a JSON value as a base64 encoded string.
+   *  `parseBase64(input: string)` decodes a JSON value from a base64 encoded string.
+   *  `readFile(file: string)` reads a JSON file asynchronously.
+   *  `readFileSync(file: string, onMissing?: any)` reads a JSON file synchronously.
 
 ## Common Utilities
 Common utilities used throughout the framework. Currently [Util](https://github.com/travetto/travetto/tree/main/module/runtime/src/util.ts#L12) includes:
@@ -323,7 +329,7 @@ export class TimeUtil {
 ```
 
 ## Process Execution
-[ExecUtil](https://github.com/travetto/travetto/tree/main/module/runtime/src/exec.ts#L43) exposes `getResult` as a means to wrap [child_process](https://nodejs.org/api/child_process.html)'s process object.  This wrapper allows for a promise-based resolution of the subprocess with the ability to capture the stderr/stdout.
+[ExecUtil](https://github.com/travetto/travetto/tree/main/module/runtime/src/exec.ts#L40) exposes `getResult` as a means to wrap [child_process](https://nodejs.org/api/child_process.html)'s process object.  This wrapper allows for a promise-based resolution of the subprocess with the ability to capture the stderr/stdout.
 
 A simple example would be:
 

package/__index__.ts

@@ -9,6 +9,7 @@ export * from './src/exec.ts';
 export * from './src/env.ts';
 export * from './src/file-loader.ts';
 export * from './src/function.ts';
+export * from './src/json.ts';
 export * from './src/manifest-index.ts';
 export * from './src/queue.ts';
 export * from './src/resources.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/runtime",
-  "version": "7.0.0-rc.2",
+  "version": "7.0.0-rc.4",
   "description": "Runtime for travetto applications.",
   "keywords": [
     "console-manager",
@@ -25,12 +25,12 @@
     "directory": "module/runtime"
   },
   "dependencies": {
-    "@travetto/manifest": "^7.0.0-rc.1",
+    "@travetto/manifest": "^7.0.0-rc.2",
     "@types/debug": "^4.1.12",
     "debug": "^4.4.3"
   },
   "peerDependencies": {
-    "@travetto/transformer": "^7.0.0-rc.2"
+    "@travetto/transformer": "^7.0.0-rc.3"
   },
   "peerDependenciesMeta": {
     "@travetto/transformer": {

package/src/context.ts

@@ -7,6 +7,7 @@ import { Env } from './env.ts';
 import { RuntimeIndex } from './manifest-index.ts';
 import { describeFunction } from './function.ts';
 import { castTo } from './types.ts';
+import { JSONUtil } from './json.ts';
 
 /** Constrained version of {@type ManifestContext} */
 class $Runtime {
@@ -31,14 +32,18 @@ class $Runtime {
     return Env.TRV_ENV.value || (!this.production ? this.#idx.manifest.workspace.defaultEnv : undefined);
   }
 
-  /** Are we in development mode */
+  /** Are we in production mode */
   get production(): boolean {
     return process.env.NODE_ENV === 'production';
   }
 
-  /** Is the app in dynamic mode? */
-  get dynamic(): boolean {
-    return Env.TRV_DYNAMIC.isTrue;
+  /** Get environment type mode */
+  get envType(): 'production' | 'development' | 'test' {
+    switch (process.env.NODE_ENV) {
+      case 'production': return 'production';
+      case 'test': return 'test';
+      default: return 'development';
+    }
   }
 
   /** Get debug value */
@@ -118,7 +123,7 @@ class $Runtime {
       throw new Error(`Unable to find ${imp}, not in the manifest`);
     } else if (imp.endsWith('.json')) {
       imp = this.#idx.getFromImport(imp)?.sourceFile ?? imp;
-      return fs.readFile(imp, 'utf8').then(JSON.parse);
+      return fs.readFile(imp, 'utf8').then(JSONUtil.parseSafe<T>);
     }
 
     if (!ManifestModuleUtil.SOURCE_EXT_REGEX.test(imp)) {

package/src/exec.ts

@@ -1,14 +1,23 @@
-import { ChildProcess } from 'node:child_process';
-import { Readable } from 'node:stream';
+import { type ChildProcess, spawn, type SpawnOptions } from 'node:child_process';
+import type { Readable } from 'node:stream';
 import { createInterface } from 'node:readline/promises';
 
-import { castTo } from './types.ts';
-
-const MINUTE = (1000 * 60);
+import { castTo, type Any } from './types.ts';
 
 const ResultSymbol = Symbol();
 
-interface ExecutionBaseResult {
+/**
+ * Result of an execution
+ */
+export interface ExecutionResult<T extends string | Buffer = string | Buffer> {
+  /**
+   * Stdout
+   */
+  stdout: T;
+  /**
+   * Stderr
+   */
+  stderr: T;
   /**
    * Exit code
    */
@@ -23,77 +32,19 @@ interface ExecutionBaseResult {
   valid: boolean;
 }
 
-/**
- * Result of an execution
- */
-export interface ExecutionResult<T extends string | Buffer = string | Buffer> extends ExecutionBaseResult {
-  /**
-   * Stdout
-   */
-  stdout: T;
-  /**
-   * Stderr
-   */
-  stderr: T;
-}
+type ExecutionBaseResult = Omit<ExecutionResult, 'stdout' | 'stderr'>;
 
 /**
  * Standard utilities for managing executions
  */
 export class ExecUtil {
 
-  static RESTART_EXIT_CODE = 200;
-
   /**
-   * Run with automatic restart support
-   * @param run The factory to produce the next running process
-   * @param maxRetriesPerMinute The number of times to allow a retry within a minute
+   * Spawn wrapper that ensures performant invocation of trv commands
    */
-  static async withRestart(
-    run: () => ChildProcess,
-    config?: { maxRetriesPerMinute?: number, relayInterrupt?: boolean }
-  ): Promise<ExecutionResult> {
-    const maxRetries = config?.maxRetriesPerMinute ?? 5;
-    const relayInterrupt = config?.relayInterrupt ?? false;
-
-    const restarts: number[] = [];
-
-    if (!relayInterrupt) {
-      process.removeAllListeners('SIGINT'); // Remove any existing listeners
-      process.on('SIGINT', () => { }); // Prevents SIGINT from killing parent process, the child will handle
-    }
-
-    for (; ;) {
-      const subProcess = run();
-      const interrupt = (): void => { subProcess.kill('SIGINT'); };
-      const toMessage = (value: unknown): void => { subProcess.send?.(value!); };
-
-      // Proxy kill requests
-      process.on('message', toMessage);
-      if (relayInterrupt) {
-        process.on('SIGINT', interrupt);
-      }
-      subProcess.on('message', value => process.send?.(value));
-
-      const result = await this.getResult(subProcess, { catch: true });
-      if (result.code !== this.RESTART_EXIT_CODE) {
-        return result;
-      } else {
-        if (relayInterrupt) {
-          process.off('SIGINT', interrupt);
-        }
-        process.off('message', toMessage);
-        restarts.unshift(Date.now());
-        if (restarts.length === maxRetries) {
-          if ((restarts[0] - restarts[maxRetries - 1]) <= MINUTE) {
-            console.error(`Bailing, due to ${maxRetries} restarts in under a minute`);
-            return result;
-          }
-          restarts.pop(); // Keep list short
-        }
-        console.error('Restarting...', { pid: process.pid });
-      }
-    }
+  static spawnTrv(cmd: string, args: string[], options: SpawnOptions): ChildProcess {
+    const entry = (globalThis as Any).__entry_point__ ?? process.argv.at(1);
+    return spawn(process.argv0, [entry, cmd, ...args], options);
   }
 
   /**

package/src/file-loader.ts

@@ -4,6 +4,7 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 
 import { AppError } from './error.ts';
+import { JSONUtil } from './json.ts';
 
 /**
  * File loader that will search for files across the provided search paths
@@ -64,4 +65,12 @@ export class FileLoader {
   async readFile(relativePath: string): Promise<File> {
     return new File([await this.read(relativePath, true)], path.basename(relativePath));
   }
+
+  /**
+   * Read relative file as JSON
+   */
+  async readJSON<T>(relativePath: string): Promise<T> {
+    const location = await this.resolve(relativePath);
+    return JSONUtil.readFile<T>(location);
+  }
 }

package/src/json.ts

@@ -0,0 +1,74 @@
+import fs from 'node:fs/promises';
+import { existsSync, readFileSync } from 'node:fs';
+
+import type { Any } from './types';
+
+/**
+ * JSON Utility functions
+ */
+export class JSONUtil {
+
+  /**
+   * Parse JSON safely
+   */
+  static parseSafe<T>(input: string | Buffer, reviver?: (this: unknown, key: string, value: Any) => unknown): T {
+    if (typeof input !== 'string') {
+      input = input.toString('utf8');
+    }
+    // TODO: Ensure we aren't vulnerable to prototype pollution
+    return JSON.parse(input, reviver);
+  }
+
+  /**
+   * Encode JSON value as base64 encoded string
+   */
+  static stringifyBase64<T>(value: T | undefined): string | undefined {
+    if (value === undefined) {
+      return;
+    }
+    const text = JSON.stringify(value);
+    return Buffer.from(text, 'utf8').toString('base64');
+  }
+
+  /**
+   * Decode JSON value from base64 encoded string
+   */
+  static parseBase64<T>(input: string): T;
+  static parseBase64<T>(input?: string | undefined): T | undefined;
+  static parseBase64<T>(input?: string | undefined): T | undefined {
+    if (!input) {
+      return undefined;
+    }
+
+    let decoded = Buffer.from(input, 'base64').toString('utf8');
+
+    // Read from encoded if it happens
+    if (decoded.startsWith('%')) {
+      decoded = decodeURIComponent(decoded);
+    }
+
+    return this.parseSafe(decoded);
+  }
+
+  /**
+   * Read JSON file asynchronously
+   * @param file
+   * @returns
+   */
+  static async readFile<T>(file: string): Promise<T> {
+    const content = await fs.readFile(file, 'utf8');
+    return this.parseSafe(content);
+  }
+
+  /**
+   * Read JSON file synchronously
+   * @param file
+   */
+  static readFileSync<T>(file: string, onMissing?: T): T {
+    if (!existsSync(file) && onMissing !== undefined) {
+      return onMissing;
+    }
+    const content = readFileSync(file, 'utf8');
+    return this.parseSafe(content);
+  }
+}

package/src/trv.d.ts

@@ -21,10 +21,6 @@ declare module "@travetto/runtime" {
      * Special role to run as, used to access additional files from the manifest during runtime.  
      */
     TRV_ROLE: Role;
-    /** 
-     * Whether or not to run the program in dynamic mode, allowing for real-time updates  
-     */
-    TRV_DYNAMIC: boolean;
     /** 
      * The folders to use for resource lookup
      */

package/src/types.ts

@@ -37,6 +37,7 @@ export const TypedObject: {
   keys<T = unknown, K extends keyof T = keyof T & string>(value: T): K[];
   fromEntries<K extends string | symbol, V>(items: ([K, V] | readonly [K, V])[]): Record<K, V>;
   entries<K extends Record<symbol | string, unknown>>(record: K): [keyof K, K[keyof K]][];
+  assign<T extends {}, U extends T>(target: T, ...sources: U[]): U;
 } & ObjectConstructor = Object;
 
 export const safeAssign = <T extends {}, U extends {}>(target: T, ...sources: U[]): T & U =>
@@ -47,6 +48,7 @@ export function castTo<T>(input: unknown): T {
   return input as T;
 }
 
+export const isClass = (input: unknown): input is Class => typeof input === 'function' && 'Ⲑid' in input;
 export const castKey = <T>(input: string | number | symbol): keyof T => castTo(input);
 export const asFull = <T>(input: Partial<T>): T => castTo(input);
 export const asConstructable = <Z = unknown>(input: Class | unknown): { constructor: Class<Z> } => castTo(input);
@@ -61,6 +63,11 @@ export const hasFunction = <T>(key: keyof T) => (value: unknown): value is T =>
 
 export const hasToJSON = hasFunction<{ toJSON(): object }>('toJSON');
 
+/**
+ * A type representing unknown type
+ */
+export class UnknownType { }
+
 export function toConcrete<T extends unknown>(): Class<T> {
   return arguments[0];
 }

package/src/util.ts

@@ -1,7 +1,7 @@
 import crypto from 'node:crypto';
 import timers from 'node:timers/promises';
 
-import { castTo, hasToJSON } from './types.ts';
+import { castTo } from './types.ts';
 import { AppError } from './error.ts';
 
 type MapFn<T, U> = (value: T, i: number) => U | Promise<U>;
@@ -120,82 +120,6 @@ export class Util {
     }
   }
 
-  /**
-   * Encode JSON value as base64 encoded string
-   */
-  static encodeSafeJSON<T>(value: T | undefined): string | undefined {
-    if (value === undefined) {
-      return;
-    }
-    const text = JSON.stringify(value);
-    return Buffer.from(text, 'utf8').toString('base64');
-  }
-
-  /**
-   * Decode JSON value from base64 encoded string
-   */
-  static decodeSafeJSON<T>(input: string): T;
-  static decodeSafeJSON<T>(input?: string | undefined): T | undefined;
-  static decodeSafeJSON<T>(input?: string | undefined): T | undefined {
-    if (!input) {
-      return undefined;
-    }
-
-    let decoded = Buffer.from(input, 'base64').toString('utf8');
-
-    // Read from encoded if it happens
-    if (decoded.startsWith('%')) {
-      decoded = decodeURIComponent(decoded);
-    }
-
-    return JSON.parse(decoded, undefined);
-  }
-
-  /**
-   * Serialize to JSON
-   */
-  static serializeToJSON<T>(out: T): string {
-    return JSON.stringify(out, function (key, value) {
-      const objectValue = this[key];
-      if (objectValue && objectValue instanceof Error) {
-        return {
-          $: true,
-          ...hasToJSON(objectValue) ? objectValue.toJSON() : objectValue,
-          name: objectValue.name,
-          message: objectValue.message,
-          stack: objectValue.stack,
-        };
-      } else if (typeof value === 'bigint') {
-        return `${value.toString()}$n`;
-      } else {
-        return value;
-      }
-    });
-  }
-
-  /**
-   * Deserialize from JSON
-   */
-  static deserializeFromJson<T = unknown>(input: string): T {
-    return JSON.parse(input, function (key, value) {
-      if (value && typeof value === 'object' && '$' in value) {
-        const error = AppError.fromJSON(value) ?? new Error();
-        if (!(error instanceof AppError)) {
-          const { $: _, ...rest } = value;
-          Object.assign(error, rest);
-        }
-        error.message = value.message;
-        error.stack = value.stack;
-        error.name = value.name;
-        return error;
-      } else if (typeof value === 'string' && /^\d+[$]n$/.test(value)) {
-        return BigInt(value.split('$')[0]);
-      } else {
-        return value;
-      }
-    });
-  }
-
   /**
    * Retry an operation, with a custom conflict handler
    * @param operation The operation to retry

package/src/watch.ts

@@ -1,11 +1,23 @@
+import { ManifestModuleUtil, type ChangeEventType, type ManifestModuleFileType } from '@travetto/manifest';
+
 import { RuntimeIndex } from './manifest-index.ts';
-import { ExecUtil } from './exec.ts';
 import { ShutdownManager } from './shutdown.ts';
 import { Util } from './util.ts';
 
-export type WatchEvent = { file: string, action: 'create' | 'update' | 'delete', output: string, module: string, time: number };
+type WatchEvent = { file: string, action: ChangeEventType, output: string, module: string, import: string, time: number };
+
+type WatchCompilerOptions = {
+  /**
+   * Restart the watch loop on compiler exit
+   */
+  restartOnCompilerExit?: boolean;
+  /**
+   * Run on restart
+   */
+  onRestart?: () => void;
+};
 
-export async function* watchCompiler(config?: { restartOnExit?: boolean, signal?: AbortSignal }): AsyncIterable<WatchEvent> {
+export async function* watchCompiler(config?: WatchCompilerOptions): AsyncIterable<WatchEvent> {
   // Load at runtime
   const { CompilerClient } = await import('@travetto/compiler/support/server/client.ts');
 
@@ -19,21 +31,56 @@ export async function* watchCompiler(config?: { restartOnExit?: boolean, signal?
   const controller = new AbortController();
   const remove = ShutdownManager.onGracefulShutdown(async () => controller.abort());
 
-  await client.waitForState(['compile-end', 'watch-start'], undefined, controller.signal);
+  const maxIterations = 10;
+  const maxWindow = 10 * 1000;
+  const iterations: number[] = [];
+  let iterationsExhausted = false;
+
+  while (
+    !controller.signal.aborted &&
+    !iterationsExhausted &&
+    (config?.restartOnCompilerExit || iterations.length === 0)
+  ) {
+    if (iterations.length) { // Wait on next iteration
+      await Util.nonBlockingTimeout(10);
+    }
+
+    await client.waitForState(['compile-end', 'watch-start'], undefined, controller.signal);
 
-  if (!await client.isWatching()) { // If we get here, without a watch
-    while (!await client.isWatching()) { // Wait until watch starts
-      await Util.nonBlockingTimeout(1000 * 60);
+    if (!await client.isWatching()) { // If we get here, without a watch
+      await Util.nonBlockingTimeout(maxWindow / (maxIterations * 2));
+    } else {
+      if (iterations.length) {
+        config?.onRestart?.();
+      }
+      yield* client.fetchEvents('change', { signal: controller.signal, enforceIteration: true });
+    }
+
+    iterations.push(Date.now());
+    if (iterations.length >= maxIterations) {
+      iterationsExhausted = (Date.now() - iterations[0]) > maxWindow;
+      iterations.shift();
     }
-  } else {
-    yield* client.fetchEvents('change', { signal: controller.signal, enforceIteration: true });
   }
 
   remove();
+}
+
+export function listenForSourceChanges(onChange: () => void, debounceDelay = 10): void {
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+
+  const validFileTypes = new Set<ManifestModuleFileType>(['ts', 'js', 'package-json', 'typings']);
 
-  if (config?.restartOnExit) {
-    // We are done, request restart
-    await ShutdownManager.gracefulShutdown('@travetto/runtime:watchCompiler');
-    process.exit(ExecUtil.RESTART_EXIT_CODE);
+  function send(): void {
+    clearTimeout(timeout);
+    timeout = setTimeout(onChange, debounceDelay);
   }
+
+  (async function (): Promise<void> {
+    for await (const item of watchCompiler({ restartOnCompilerExit: true, onRestart: send })) {
+      if (validFileTypes.has(ManifestModuleUtil.getFileType(item.file)) && RuntimeIndex.findModuleForArbitraryFile(item.file)) {
+        send();
+      }
+    }
+  })();
 }

package/support/transformer/metadata.ts

@@ -1,4 +1,4 @@
-import * as ts from 'typescript';
+import ts from 'typescript';
 
 import { FunctionMetadataTag } from '@travetto/runtime';
 import { CoreUtil, Import, SystemUtil, TransformerState } from '@travetto/transformer';