@travetto/manifest 3.0.0-rc.10

package/src/package.ts

@@ -0,0 +1,220 @@
+import { readFileSync } from 'fs';
+import fs from 'fs/promises';
+import { createRequire } from 'module';
+import { execSync } from 'child_process';
+
+import { ManifestContext, Package, PackageDigest, PackageRel, PackageVisitor, PackageVisitReq, PackageWorkspaceEntry } from './types';
+import { path } from './path';
+
+export class PackageUtil {
+
+  static #req = createRequire(path.resolve('node_modules'));
+  static #framework: Package;
+  static #cache: Record<string, Package> = {};
+  static #workspaces: Record<string, PackageWorkspaceEntry[]> = {};
+
+  /**
+   * Clear out cached package file reads
+   */
+  static clearCache(): void {
+    this.#cache = {};
+    this.#workspaces = {};
+  }
+
+  static resolveImport = (library: string): string => this.#req.resolve(library);
+
+  /**
+   * Resolve version path, if file: url
+   */
+  static resolveVersionPath(rootPath: string, ver: string): string | undefined {
+    if (ver.startsWith('file:')) {
+      return path.resolve(rootPath, ver.replace('file:', ''));
+    } else {
+      return;
+    }
+  }
+
+  /**
+   * Find package.json folder for a given dependency
+   */
+  static resolvePackagePath(name: string): string {
+    try {
+      return path.dirname(this.resolveImport(`${name}/package.json`));
+    } catch {
+      try {
+        const resolved = this.resolveImport(name);
+        return path.join(resolved.split(name)[0], name);
+      } catch { }
+    }
+    throw new Error(`Unable to resolve: ${name}`);
+  }
+
+  /**
+   * Build a package visit req
+   */
+  static packageReq<T>(sourcePath: string, rel: PackageRel): PackageVisitReq<T> {
+    return { pkg: this.readPackage(sourcePath), sourcePath, rel };
+  }
+
+  /**
+   * Extract all dependencies from a package
+   */
+  static getAllDependencies<T = unknown>(modulePath: string, rootPath: string): PackageVisitReq<T>[] {
+    const pkg = this.readPackage(modulePath);
+    const children: Record<string, PackageVisitReq<T>> = {};
+    const local = modulePath === rootPath && !modulePath.includes('node_modules');
+    for (const [deps, rel] of [
+      [pkg.dependencies, 'prod'],
+      [pkg.peerDependencies, 'peer'],
+      [pkg.optionalDependencies, 'opt'],
+      ...(local ? [[pkg.devDependencies, 'dev'] as const] : []),
+    ] as const) {
+      for (const [name, version] of Object.entries(deps ?? {})) {
+        try {
+          const depPath = this.resolveVersionPath(modulePath, version) ?? this.resolvePackagePath(name);
+          children[`${name}#${version}`] = this.packageReq<T>(depPath, rel);
+        } catch (err) {
+          if (rel === 'opt' || (rel === 'peer' && !!pkg.peerDependenciesMeta?.[name].optional)) {
+            continue;
+          }
+          throw err;
+        }
+      }
+    }
+    return Object.values(children).sort((a, b) => a.pkg.name.localeCompare(b.pkg.name));
+  }
+
+  /**
+   * Read a package.json from a given folder
+   */
+  static readPackage(modulePath: string, forceRead = false): Package {
+    if (forceRead) {
+      delete this.#cache[modulePath];
+    }
+    return this.#cache[modulePath] ??= JSON.parse(readFileSync(
+      modulePath.endsWith('.json') ? modulePath : path.resolve(modulePath, 'package.json'),
+      'utf8'
+    ));
+  }
+
+  /**
+   * import a package.json from a given module name
+   */
+  static importPackage(moduleName: string): Package {
+    return this.readPackage(this.resolvePackagePath(moduleName));
+  }
+
+  /**
+   * Write package
+   */
+  static async writePackageIfChanged(modulePath: string, pkg: Package): Promise<void> {
+    const final = JSON.stringify(pkg, null, 2);
+    const target = path.resolve(modulePath, 'package.json');
+    const current = (await fs.readFile(target, 'utf8').catch(() => '')).trim();
+    if (final !== current) {
+      await fs.writeFile(target, `${final}\n`, 'utf8');
+    }
+  }
+
+  /**
+   * Visit packages with ability to track duplicates
+   */
+  static async visitPackages<T>(
+    rootOrPath: PackageVisitReq<T> | string,
+    visitor: PackageVisitor<T>
+  ): Promise<Set<T>> {
+
+    const root = typeof rootOrPath === 'string' ?
+      this.packageReq<T>(rootOrPath, 'root') :
+      rootOrPath;
+
+    const seen = new Map<string, T>();
+    const queue: PackageVisitReq<T>[] = [...await visitor.init?.(root) ?? [], root];
+    const out = new Set<T>();
+
+    while (queue.length) {
+      const req = queue.pop();
+
+      if (!req || (visitor.valid && !visitor.valid(req))) {
+        continue;
+      }
+
+      const key = req.sourcePath;
+      if (seen.has(key)) {
+        await visitor.visit?.(req, seen.get(key)!);
+      } else {
+        const dep = await visitor.create(req);
+        out.add(dep);
+        await visitor.visit?.(req, dep);
+        seen.set(key, dep);
+        const children = this.getAllDependencies<T>(req.sourcePath, root.sourcePath);
+        queue.push(...children.map(x => ({ ...x, parent: dep })));
+      }
+    }
+    return (await visitor.complete?.(out)) ?? out;
+  }
+
+  /**
+   * Get version of manifest package
+   */
+  static getFrameworkVersion(): string {
+    return (this.#framework ??= this.importPackage('@travetto/manifest')).version;
+  }
+
+  /**
+   * Produce simple digest of
+   */
+  static digest(pkg: Package): PackageDigest {
+    const { main, name, author, license, version } = pkg;
+    return { name, main, author, license, version, framework: this.getFrameworkVersion() };
+  }
+
+  /**
+   * Find workspace values from rootPath
+   */
+  static async resolveWorkspaces(ctx: ManifestContext, rootPath: string): Promise<PackageWorkspaceEntry[]> {
+    if (!this.#workspaces[rootPath]) {
+      await fs.mkdir(path.resolve(ctx.workspacePath, ctx.outputFolder), { recursive: true });
+      const cache = path.resolve(ctx.workspacePath, ctx.outputFolder, 'workspaces.json');
+      try {
+        return JSON.parse(await fs.readFile(cache, 'utf8'));
+      } catch {
+        const text = execSync('npm query .workspace', { cwd: rootPath, encoding: 'utf8', env: { PATH: process.env.PATH, NODE_PATH: process.env.NODE_PATH } });
+        const res: { location: string, name: string }[] = JSON.parse(text);
+        const out = this.#workspaces[rootPath] = res.map(d => ({ sourcePath: d.location, name: d.name }));
+        await fs.writeFile(cache, JSON.stringify(out), 'utf8');
+      }
+    }
+    return this.#workspaces[rootPath];
+  }
+
+  /**
+   * Sync versions across a series of folders
+   */
+  static async syncVersions(folders: string[], versionMapping: Record<string, string> = {}): Promise<void> {
+    const packages = folders.map(folder => {
+      const pkg = this.readPackage(folder, true);
+      versionMapping[pkg.name] = `^${pkg.version}`;
+      return { folder, pkg };
+    });
+
+    for (const { pkg } of packages) {
+      for (const group of [
+        pkg.dependencies ?? {},
+        pkg.devDependencies ?? {},
+        pkg.optionalDependencies ?? {},
+        pkg.peerDependencies ?? {}
+      ]) {
+        for (const [mod, ver] of Object.entries(versionMapping)) {
+          if (mod in group && !/^[*]|(file:.*)$/.test(group[mod])) {
+            group[mod] = ver;
+          }
+        }
+      }
+    }
+
+    for (const { folder, pkg } of packages) {
+      await this.writePackageIfChanged(folder, pkg);
+    }
+  }
+}

package/src/path.ts

@@ -0,0 +1,17 @@
+import { extname, dirname, resolve, basename, delimiter, join, sep } from 'path';
+
+const posix = (file: string): string => file.replaceAll('\\', '/');
+
+const cwd = (): string => posix(process.cwd());
+
+export const path = {
+  cwd,
+  toPosix: posix,
+  delimiter,
+  basename: (file: string): string => posix(basename(file)),
+  extname: (file: string): string => posix(extname(file)),
+  dirname: (file: string): string => posix(dirname(file)),
+  resolve: (...args: string[]): string => posix(resolve(cwd(), ...args.map(f => posix(f)))),
+  join: (root: string, ...args: string[]): string => posix(join(posix(root), ...args.map(f => posix(f)))),
+  toNative: (file: string): string => file.replaceAll('/', sep)
+};

package/src/root-index.ts

@@ -0,0 +1,133 @@
+import { path } from './path';
+import { IndexedModule, ManifestIndex } from './manifest-index';
+import { FunctionMetadata, Package, PackageDigest } from './types';
+import { PackageUtil } from './package';
+
+const METADATA = Symbol.for('@travetto/manifest:metadata');
+type Metadated = { [METADATA]: FunctionMetadata };
+
+/**
+ * Extended manifest index geared for application execution
+ */
+class $RootIndex extends ManifestIndex {
+
+  #config: Package | undefined;
+  #metadata = new Map<string, FunctionMetadata>();
+
+  /**
+   * **WARNING**: This is a destructive operation, and should only be called before loading any code
+   * @private
+   */
+  reinitForModule(module: string): void {
+    this.init(`${this.outputRoot}/node_modules/${module}`);
+    this.#config = undefined;
+  }
+
+  /**
+   * Determines if the manifest root is the root for a monorepo
+   */
+  isMonoRepoRoot(): boolean {
+    return !!this.manifest.monoRepo && !this.manifest.mainFolder;
+  }
+
+  /**
+   * Asynchronously load all source files from manifest
+   */
+  async loadSource(): Promise<void> {
+    for (const { import: imp } of this.findSrc()) {
+      await import(imp);
+    }
+  }
+
+  /**
+   * Get internal id from file name and optionally, class name
+   */
+  getId(filename: string, clsName?: string): string {
+    filename = path.toPosix(filename);
+    const id = this.getEntry(filename)?.id ?? filename;
+    return clsName ? `${id}￮${clsName}` : id;
+  }
+
+  /**
+   * Get main module for manifest
+   */
+  get mainModule(): IndexedModule {
+    return this.getModule(this.mainPackage.name)!;
+  }
+
+  /**
+   * Get main package for manifest
+   */
+  get mainPackage(): Package {
+    if (!this.#config) {
+      const { outputPath } = this.getModule(this.manifest.mainModule)!;
+      this.#config = {
+        ...{
+          name: 'untitled',
+          description: 'A Travetto application',
+          version: '0.0.0',
+        },
+        ...PackageUtil.readPackage(outputPath)
+      };
+    }
+    return this.#config;
+  }
+
+  mainDigest(): PackageDigest {
+    return PackageUtil.digest(this.mainPackage);
+  }
+
+  /**
+  * Get source file from import location
+  * @param outputFile
+  */
+  getSourceFile(importFile: string): string {
+    return this.getFromImport(importFile)?.sourceFile ?? importFile;
+  }
+
+  /**
+   * Initialize the meta data for a function/class
+   * @param cls Class
+   * @param `file` Filename
+   * @param `hash` Hash of class contents
+   * @param `methods` Methods and their hashes
+   * @param `abstract` Is the class abstract
+   */
+  registerFunction(cls: Function, fileOrImport: string, hash: number, methods?: Record<string, { hash: number }>, abstract?: boolean, synthetic?: boolean): boolean {
+    const source = this.getSourceFile(fileOrImport);
+    const id = this.getId(source, cls.name);
+    Object.defineProperty(cls, 'Ⲑid', { value: id });
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    (cls as unknown as Metadated)[METADATA] = { id, source, hash, methods, abstract, synthetic };
+    this.#metadata.set(id, { id, source, hash, methods, abstract, synthetic });
+    return true;
+  }
+
+  /**
+   * Retrieve function metadata by function, or function id
+   */
+  getFunctionMetadataFromClass(cls: Function | undefined): FunctionMetadata | undefined {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    return (cls as unknown as Metadated)?.[METADATA];
+  }
+
+  /**
+   * Retrieve function metadata by function, or function id
+   */
+  getFunctionMetadata(clsId: string | Function): FunctionMetadata | undefined {
+    const id = clsId === undefined ? '' : typeof clsId === 'string' ? clsId : clsId.Ⲑid;
+    return this.#metadata.get(id);
+  }
+}
+
+let index: $RootIndex | undefined;
+
+try {
+  index = new $RootIndex(process.env.TRV_MANIFEST!);
+} catch (err) {
+  if (/prod/i.test(process.env.NODE_ENV ?? '')) {
+    throw err;
+  }
+}
+
+export const RootIndex: $RootIndex = index!;

package/src/types.ts

@@ -0,0 +1,111 @@
+export type ManifestModuleFileType = 'typings' | 'ts' | 'js' | 'json' | 'package-json' | 'unknown' | 'fixture' | 'md';
+export type ManifestModuleFolderType =
+  '$root' | '$index' | '$package' |
+  'src' | 'bin' | 'support' | 'resources' | 'test' | 'doc' |
+  'test/fixtures' | 'support/fixtures' | 'support/resources' |
+  '$other' | '$transformer';
+
+export type ManifestProfile = 'compile' | 'test' | 'doc' | 'build' | 'std';
+export type PackageRel = 'dev' | 'prod' | 'peer' | 'opt' | 'root' | 'global';
+
+export type ManifestModuleFile = [string, ManifestModuleFileType, number] | [string, ManifestModuleFileType, number, ManifestProfile];
+export type ManifestModuleCore = {
+  name: string;
+  main?: boolean;
+  local?: boolean;
+  version: string;
+  sourceFolder: string;
+  outputFolder: string;
+  profiles: ManifestProfile[];
+  parents: string[];
+  internal?: boolean;
+};
+
+export type ManifestModule = ManifestModuleCore & {
+  files: Partial<Record<ManifestModuleFolderType, ManifestModuleFile[]>>;
+};
+
+export type ManifestContext = {
+  mainModule: string;
+  mainFolder: string;
+  workspacePath: string;
+  outputFolder: string;
+  toolFolder: string;
+  compilerFolder: string;
+  monoRepo?: boolean;
+  moduleType: 'module' | 'commonjs';
+};
+
+export type ManifestRoot = ManifestContext & {
+  generated: number;
+  modules: Record<string, ManifestModule>;
+};
+
+export type Package = {
+  name: string;
+  type?: 'module' | 'commonjs';
+  version: string;
+  description?: string;
+  license?: string;
+  repository?: {
+    url: string;
+    directory?: string;
+  };
+  author?: {
+    email?: string;
+    name?: string;
+  };
+  main: string;
+  homepage?: string;
+  files?: string[];
+  bin?: Record<string, string>;
+  scripts?: Record<string, string>;
+  engines?: Record<string, string>;
+  keywords?: string[];
+
+  dependencies?: Record<string, string>;
+  devDependencies?: Record<string, string>;
+  peerDependencies?: Record<string, string>;
+  peerDependenciesMeta?: Record<string, { optional?: boolean }>;
+  optionalDependencies?: Record<string, string>;
+  travetto?: {
+    isolated?: boolean;
+    displayName?: string;
+    profiles?: ManifestProfile[];
+    globalModules?: string[];
+    mainSource?: string[];
+    docOutput?: string[];
+    docRoot?: string;
+    docBaseUrl?: string;
+    docOutputs?: string[];
+    outputFolder?: string;
+  };
+  workspaces?: string[];
+  private?: boolean;
+  publishConfig?: { access?: 'restricted' | 'public' };
+};
+
+export type PackageDigestField = 'name' | 'main' | 'author' | 'license' | 'version';
+export type PackageDigest = Pick<Package, PackageDigestField> & { framework: string };
+
+type OrProm<T> = T | Promise<T>;
+
+export type PackageVisitReq<T> = { pkg: Package, rel: PackageRel, sourcePath: string, parent?: T };
+export type PackageVisitor<T> = {
+  init?(req: PackageVisitReq<T>): OrProm<undefined | void | PackageVisitReq<T>[]>;
+  valid?(req: PackageVisitReq<T>): boolean;
+  create(req: PackageVisitReq<T>): OrProm<T>;
+  visit?(req: PackageVisitReq<T>, item: T): OrProm<void>;
+  complete?(values: Set<T>): OrProm<Set<T> | undefined>;
+};
+
+export type PackageWorkspaceEntry = { name: string, sourcePath: string };
+
+export type FunctionMetadata = {
+  id: string;
+  source: string;
+  hash?: number;
+  methods?: Record<string, { hash: number }>;
+  synthetic?: boolean;
+  abstract?: boolean;
+};

package/src/typings.d.ts

@@ -0,0 +1,3 @@
+declare interface Function {
+  Ⲑid: string;
+}

package/src/util.ts

@@ -0,0 +1,115 @@
+import { readFileSync } from 'fs';
+import fs from 'fs/promises';
+import os from 'os';
+
+import { path } from './path';
+import { ManifestContext, ManifestRoot } from './types';
+import { ManifestModuleUtil } from './module';
+
+const MANIFEST_FILE = 'manifest.json';
+
+/**
+ * Manifest utils
+ */
+export class ManifestUtil {
+  /**
+   * Write file and copy over when ready
+   */
+  static async #writeJsonWithBuffer(ctx: ManifestContext, filename: string, obj: object): Promise<string> {
+    const tempName = path.resolve(ctx.workspacePath, ctx.mainFolder, filename).replace(/[\/\\: ]/g, '_');
+    const file = path.resolve(ctx.workspacePath, ctx.outputFolder, 'node_modules', ctx.mainModule, filename);
+    await fs.mkdir(path.dirname(file), { recursive: true });
+    const temp = path.resolve(os.tmpdir(), `${tempName}.${Date.now()}`);
+    await fs.writeFile(temp, JSON.stringify(obj), 'utf8');
+    await fs.copyFile(temp, file);
+    fs.unlink(temp);
+    return file;
+  }
+
+  /**
+   * Build a manifest context
+   * @param folder
+   */
+  static async buildContext(folder?: string): Promise<ManifestContext> {
+    const { getManifestContext } = await import('../bin/context.js');
+    return getManifestContext(folder);
+  }
+
+  /**
+   * Produce manifest in memory
+   */
+  static async buildManifest(ctx: ManifestContext): Promise<ManifestRoot> {
+    return {
+      modules: await ManifestModuleUtil.produceModules(ctx),
+      generated: Date.now(),
+      ...ctx
+    };
+  }
+
+  /**
+   * Produce a production manifest from a given manifest
+   */
+  static createProductionManifest(manifest: ManifestRoot): ManifestRoot {
+    return {
+      ...manifest,
+      // If in prod mode, only include std modules
+      modules: Object.fromEntries(
+        Object.values(manifest.modules)
+          .filter(x => x.profiles.includes('std'))
+          .map(m => [m.name, m])
+      ),
+      // Mark output folder/workspace path as portable
+      outputFolder: '',
+      workspacePath: '',
+    };
+  }
+
+  /**
+   * Read manifest, synchronously
+   *
+   * @param file
+   * @returns
+   */
+  static readManifestSync(file: string): { manifest: ManifestRoot, file: string } {
+    file = path.resolve(file);
+    if (!file.endsWith('.json')) {
+      file = path.resolve(file, MANIFEST_FILE);
+    }
+    const manifest: ManifestRoot = JSON.parse(readFileSync(file, 'utf8'));
+    if (!manifest.outputFolder) {
+      manifest.outputFolder = path.cwd();
+      manifest.workspacePath = path.cwd();
+    }
+    return { manifest, file };
+  }
+
+  /**
+   * Write manifest for a given context, return location
+   */
+  static writeManifest(ctx: ManifestContext, manifest: ManifestRoot): Promise<string> {
+    return this.#writeJsonWithBuffer(ctx, MANIFEST_FILE, manifest);
+  }
+
+  /**
+   * Write a manifest to a specific file, if no file extension provided, the file is assumed to be a folder
+   */
+  static async writeManifestToFile(location: string, manifest: ManifestRoot): Promise<string> {
+    if (!location.endsWith('.json')) {
+      location = path.resolve(location, MANIFEST_FILE);
+    }
+
+    await fs.mkdir(path.dirname(location), { recursive: true });
+    await fs.writeFile(location, JSON.stringify(manifest), 'utf8');
+
+    return location;
+  }
+
+  /**
+   * Rewrite manifest for a given folder
+   */
+  static async rewriteManifest(source: string): Promise<void> {
+    const subCtx = await this.buildContext(source);
+    const subManifest = await this.buildManifest(subCtx);
+    await this.writeManifest(subCtx, subManifest);
+  }
+}

package/src/watch.ts

@@ -0,0 +1,56 @@
+import fs from 'fs/promises';
+import { path } from './path';
+
+export type WatchEvent = { action: 'create' | 'update' | 'delete', file: string };
+
+export type WatchEventListener = (ev: WatchEvent, folder: string) => void;
+type EventFilter = (ev: WatchEvent) => boolean;
+type WatchConfig = { filter?: EventFilter, ignore?: string[], createMissing?: boolean };
+
+async function getWatcher(): Promise<typeof import('@parcel/watcher')> {
+  try {
+    return await import('@parcel/watcher');
+  } catch (err) {
+    console.error('@parcel/watcher must be installed to use watching functionality');
+    throw err;
+  }
+}
+
+/**
+ * Leverages @parcel/watcher to watch a series of folders
+ * @param folders
+ * @param onEvent
+ * @private
+ */
+export async function watchFolders(folders: string[], onEvent: WatchEventListener, config: WatchConfig = {}): Promise<() => Promise<void>> {
+  const lib = await getWatcher();
+  const createMissing = config.createMissing ?? false;
+  const validFolders = new Set(folders);
+
+  const subs = await Promise.all(folders.map(async folder => {
+    if (await fs.stat(folder).then(() => true, () => createMissing)) {
+      await fs.mkdir(folder, { recursive: true });
+      const ignore = (await fs.readdir(folder)).filter(x => x.startsWith('.') && x.length > 2);
+      return lib.subscribe(folder, (err, events) => {
+        for (const ev of events) {
+          if (ev.type === 'delete' && validFolders.has(path.toPosix(ev.path))) {
+            return process.exit(0); // Exit when watched folder is removed
+          }
+          const finalEv = { action: ev.type, file: ev.path };
+          if (!config.filter || config.filter(finalEv)) {
+            onEvent(finalEv, folder);
+          }
+        }
+      }, { ignore: [...ignore, ...config.ignore ?? []] });
+    }
+  }));
+
+  // Allow for multiple calls
+  let finalProm: Promise<void> | undefined;
+  const remove = (): Promise<void> => finalProm ??= Promise.all(subs.map(x => x?.unsubscribe())).then(() => { });
+
+  // Cleanup on exit
+  process.on('exit', remove);
+
+  return remove;
+}