@travetto/manifest 3.0.0-rc.3

package/src/manifest.ts

@@ -0,0 +1,110 @@
+import fs from 'fs/promises';
+import os from 'os';
+
+import { path } from './path';
+import { ManifestContext, ManifestRoot, ManifestState } from './types';
+
+import { ManifestModuleUtil } from './module';
+import { ManifestDeltaUtil } from './delta';
+
+/**
+ * Manifest utils
+ */
+export class ManifestUtil {
+
+  /**
+   * Produce manifest in memory
+   */
+  static async buildManifest(ctx: ManifestContext): Promise<ManifestRoot> {
+    return {
+      modules: await ManifestModuleUtil.produceModules(ctx),
+      generated: Date.now(),
+      ...ctx
+    };
+  }
+
+  /**
+   * Generate manifest for a given context, and persist
+   */
+  static async createAndWriteManifest(ctx: ManifestContext): Promise<void> {
+    const manifest = await this.buildManifest(ctx);
+    await this.writeManifest(ctx, manifest);
+  }
+
+  /**
+   * Read manifest from a folder
+   */
+  static async readManifest(ctx: ManifestContext): Promise<ManifestRoot> {
+    const file = path.resolve(ctx.workspacePath, ctx.outputFolder, ctx.manifestFile);
+    if (await fs.stat(file).catch(() => false)) {
+      try {
+        return JSON.parse(await fs.readFile(file, 'utf8'));
+      } catch (err) {
+        await fs.unlink(file).catch(() => { });
+        const final = new Error(`Corrupted manifest ${ctx.manifestFile}: file has been removed, retry`);
+        if (err instanceof Error) {
+          final.stack = [final.message, ...(err.stack ?? '').split('\n').slice(1)].join('\n');
+        }
+        throw final;
+      }
+    } else {
+      return {
+        modules: {},
+        generated: Date.now(),
+        ...ctx,
+      };
+    }
+  }
+
+  /**
+   * Load state from disk
+   */
+  static async readState(file: string): Promise<ManifestState> {
+    return JSON.parse(await fs.readFile(file, 'utf8'));
+  }
+
+  /**
+   * Persist state to disk in a temp file, return said temp file
+   */
+  static async writeState(state: ManifestState, file?: string): Promise<string> {
+    const manifestTemp = file ?? path.resolve(os.tmpdir(), `manifest-state.${Date.now()}${Math.random()}.json`);
+    await fs.writeFile(manifestTemp, JSON.stringify(state), 'utf8');
+    return manifestTemp;
+  }
+
+  /**
+   * Generate the manifest and delta as a single output
+   */
+  static async produceState(ctx: ManifestContext): Promise<ManifestState> {
+    const manifest = await this.buildManifest(ctx);
+    const oldManifest = await this.readManifest(ctx);
+    const delta = await ManifestDeltaUtil.produceDelta(
+      path.resolve(manifest.workspacePath, ctx.outputFolder),
+      manifest,
+      oldManifest
+    );
+    return { manifest, delta };
+  }
+
+  /**
+   * Resolves a module file, from a context and manifest
+   */
+  static resolveFile(ctx: ManifestContext, manifest: ManifestRoot, module: string, file: string): string {
+    return path.resolve(
+      ctx.workspacePath,
+      ctx.compilerFolder,
+      manifest.modules[module].output,
+      file
+    );
+  }
+
+  /**
+   * Write manifest for a given context
+   */
+  static async writeManifest(ctx: ManifestContext, manifest: ManifestRoot): Promise<void> {
+    // Write manifest in the scenario we are in mono-repo state where everything pre-existed
+    const file = path.resolve(ctx.workspacePath, ctx.outputFolder, ctx.manifestFile);
+    await fs.mkdir(path.dirname(file), { recursive: true });
+    await fs.writeFile(file, JSON.stringify(manifest));
+  }
+}

package/src/module.ts

@@ -0,0 +1,192 @@
+import fs from 'fs/promises';
+
+import { path } from './path';
+import {
+  ManifestContext,
+  ManifestModule, ManifestModuleFile, ManifestModuleFileType,
+  ManifestModuleFolderType, ManifestProfile
+} from './types';
+import { ModuleDep, ModuleDependencyVisitor } from './dependencies';
+import { PackageUtil } from './package';
+
+const EXT_MAPPING: Record<string, ManifestModuleFileType> = {
+  '.js': 'js',
+  '.mjs': 'js',
+  '.cjs': 'js',
+  '.json': 'json',
+  '.ts': 'ts',
+  '.md': 'md'
+};
+
+const INDEX_FILES = new Set([
+  'index.ts',
+  'index.js',
+  '__index__.ts',
+  '__index__.js',
+  '__index.ts',
+  '__index.js'
+]);
+
+export class ManifestModuleUtil {
+
+  static #getNewest(stat: { mtimeMs: number, ctimeMs: number }): number {
+    return Math.max(stat.mtimeMs, stat.ctimeMs);
+  }
+
+  /**
+   * Simple file scanning
+   */
+  static async #scanFolder(folder: string, topFolders = new Set<string>(), topFiles = new Set<string>()): Promise<string[]> {
+    const out: string[] = [];
+    if (!fs.stat(folder).catch(() => false)) {
+      return out;
+    }
+    const stack: [string, number][] = [[folder, 0]];
+    while (stack.length) {
+      const popped = stack.pop();
+      if (!popped) {
+        continue;
+      }
+
+      const [top, depth] = popped;
+
+      // Don't navigate into sub-folders with package.json's
+      if (top !== folder && await fs.stat(`${top}/package.json`).catch(() => false)) {
+        continue;
+      }
+
+      for (const sub of await fs.readdir(top)) {
+        const stat = await fs.stat(`${top}/${sub}`);
+        if (stat.isFile()) {
+          if (!sub.startsWith('.') && (depth > 0 || !topFiles.size || topFiles.has(sub))) {
+            out.push(`${top}/${sub}`);
+          }
+        } else {
+          if (!sub.includes('node_modules') && !sub.startsWith('.') && (depth > 0 || !topFolders.size || topFolders.has(sub))) {
+            stack.push([`${top}/${sub}`, depth + 1]);
+          }
+        }
+      }
+    }
+    return out;
+  }
+
+  /**
+   * Get file type for a file name
+   */
+  static getFileType(moduleFile: string): ManifestModuleFileType {
+    if (moduleFile === 'package.json') {
+      return 'package-json';
+    } else if (
+      moduleFile.startsWith('support/fixtures/') ||
+      moduleFile.startsWith('test/fixtures/') ||
+      moduleFile.startsWith('support/resources/')
+    ) {
+      return 'fixture';
+    } else if (moduleFile.endsWith('.d.ts')) {
+      return 'typings';
+    } else {
+      const ext = path.extname(moduleFile);
+      return EXT_MAPPING[ext] ?? 'unknown';
+    }
+  }
+
+  /**
+   * Get file type for a file name
+   */
+  static getFileProfile(moduleFile: string): ManifestProfile | undefined {
+    if (moduleFile.startsWith('support/transform')) {
+      return 'compile';
+    } else if (moduleFile.startsWith('support/test/') || moduleFile.startsWith('test/')) {
+      return 'test';
+    } else if (moduleFile.startsWith('doc/') || moduleFile === 'DOC.ts') {
+      return 'doc';
+    } else {
+      return;
+    }
+  }
+
+  /**
+   * Get folder key
+   */
+  static getFolderKey(moduleFile: string): ManifestModuleFolderType {
+    const folderLocation = moduleFile.indexOf('/');
+    if (folderLocation > 0) {
+      if (moduleFile.startsWith('test/fixtures/')) {
+        return 'test/fixtures';
+      } else if (moduleFile.startsWith('support/fixtures/')) {
+        return 'support/fixtures';
+      } else if (moduleFile.startsWith('support/resources/')) {
+        return 'support/resources';
+      }
+      const key = moduleFile.substring(0, folderLocation);
+      switch (key) {
+        case 'src':
+        case 'bin':
+        case 'test':
+        case 'doc':
+        case 'resources':
+        case 'support': return key;
+        default: return '$other';
+      }
+    } else if (moduleFile === 'DOC.ts') {
+      return 'doc';
+    } else if (INDEX_FILES.has(moduleFile)) {
+      return '$index';
+    } else if (moduleFile === 'package.json') {
+      return '$package';
+    } else {
+      return '$root';
+    }
+  }
+
+  /**
+   * Convert file (by ext) to a known file type and also retrieve its latest timestamp
+   */
+  static async transformFile(moduleFile: string, full: string): Promise<ManifestModuleFile> {
+    const res: ManifestModuleFile = [moduleFile, this.getFileType(moduleFile), this.#getNewest(await fs.stat(full))];
+    const profile = this.getFileProfile(moduleFile);
+    return profile ? [...res, profile] : res;
+  }
+
+  /**
+   * Visit a module and describe files, and metadata
+   */
+  static async describeModule(dep: ModuleDep): Promise<ManifestModule> {
+    const { main, mainSource, local, name, version, sourcePath, profileSet, parentSet, internal } = dep;
+    const files: ManifestModule['files'] = {};
+    const folderSet = new Set<ManifestModuleFolderType>(mainSource ? [] : ['src', 'bin', 'support']);
+    const fileSet = new Set(mainSource ? [] : [...INDEX_FILES, 'package.json']);
+
+    for (const file of await this.#scanFolder(sourcePath, folderSet, fileSet)) {
+      // Group by top folder
+      const moduleFile = file.replace(`${sourcePath}/`, '');
+      const entry = await this.transformFile(moduleFile, file);
+      const key = this.getFolderKey(moduleFile);
+      (files[key] ??= []).push(entry);
+    }
+
+    // Refine non-main source
+    if (!mainSource) {
+      files.$root = files.$root?.filter(([file, type]) => type !== 'ts');
+    }
+
+    const profiles = [...profileSet].sort();
+    const parents = [...parentSet].sort();
+    const output = `node_modules/${name}`;
+
+    return { main, name, version, local, internal, source: sourcePath, output, files, profiles, parents, };
+  }
+
+  /**
+   * Produce all modules for a given manifest folder, adding in some given modules when developing framework
+   */
+  static async produceModules(ctx: ManifestContext): Promise<Record<string, ManifestModule>> {
+    const visitor = new ModuleDependencyVisitor(ctx);
+    const declared = await PackageUtil.visitPackages(ctx.mainPath, visitor);
+    const sorted = [...declared].sort((a, b) => a.name.localeCompare(b.name));
+
+    const modules = await Promise.all(sorted.map(x => this.describeModule(x)));
+    return Object.fromEntries(modules.map(m => [m.name, m]));
+  }
+}

package/src/package.ts

@@ -0,0 +1,201 @@
+import { readFileSync } from 'fs';
+import fs from 'fs/promises';
+import { createRequire } from 'module';
+import { execSync } from 'child_process';
+
+import { Package, PackageDigest, PackageRel, PackageVisitor, PackageVisitReq, PackageWorkspaceEntry } from './types';
+import { path } from './path';
+
+export class PackageUtil {
+
+  static #req = createRequire(`${path.cwd()}/node_modules`);
+  static #framework: Package;
+  static #cache: Record<string, Package> = {};
+  static #workspaces: Record<string, PackageWorkspaceEntry[]> = {};
+
+  static resolveImport = (library: string): string => this.#req.resolve(library);
+
+  /**
+   * Resolve version path, if file: url
+   */
+  static resolveVersionPath(rootPath: string, ver: string): string | undefined {
+    if (ver.startsWith('file:')) {
+      return path.resolve(rootPath, ver.replace('file:', ''));
+    } else {
+      return;
+    }
+  }
+
+  /**
+   * Find package.json folder for a given dependency
+   */
+  static resolvePackagePath(name: string): string {
+    try {
+      return path.dirname(this.resolveImport(`${name}/package.json`));
+    } catch {
+      try {
+        const resolved = this.resolveImport(name);
+        return path.join(resolved.split(name)[0], name);
+      } catch { }
+    }
+    throw new Error(`Unable to resolve: ${name}`);
+  }
+
+  /**
+   * Build a package visit req
+   */
+  static packageReq<T>(sourcePath: string, rel: PackageRel): PackageVisitReq<T> {
+    return { pkg: this.readPackage(sourcePath), sourcePath, rel };
+  }
+
+  /**
+   * Extract all dependencies from a package
+   */
+  static getAllDependencies<T = unknown>(modulePath: string, rootPath: string): PackageVisitReq<T>[] {
+    const pkg = this.readPackage(modulePath);
+    const children: Record<string, PackageVisitReq<T>> = {};
+    for (const [deps, rel] of [
+      [pkg.dependencies, 'prod'],
+      [pkg.peerDependencies, 'peer'],
+      [pkg.optionalDependencies, 'opt'],
+      ...(modulePath === rootPath ? [[pkg.devDependencies, 'dev'] as const] : []),
+    ] as const) {
+      for (const [name, version] of Object.entries(deps ?? {})) {
+        try {
+          const depPath = this.resolveVersionPath(modulePath, version) ?? this.resolvePackagePath(name);
+          children[`${name}#${version}`] = this.packageReq<T>(depPath, rel);
+        } catch (err) {
+          if (rel === 'opt' || (rel === 'peer' && !!pkg.peerDependenciesMeta?.[name].optional)) {
+            continue;
+          }
+          throw err;
+        }
+      }
+    }
+    return Object.values(children).sort((a, b) => a.pkg.name.localeCompare(b.pkg.name));
+  }
+
+  /**
+   * Read a package.json from a given folder
+   */
+  static readPackage(modulePath: string): Package {
+    return this.#cache[modulePath] ??= JSON.parse(readFileSync(
+      modulePath.endsWith('.json') ? modulePath : path.resolve(modulePath, 'package.json'),
+      'utf8'
+    ));
+  }
+
+  /**
+   * import a package.json from a given module name
+   */
+  static importPackage(moduleName: string): Package {
+    return this.readPackage(this.resolvePackagePath(moduleName));
+  }
+
+  /**
+   * Write package
+   */
+  static async writePackageIfChanged(modulePath: string, pkg: Package): Promise<void> {
+    const final = JSON.stringify(pkg, null, 2);
+    const target = path.resolve(modulePath, 'package.json');
+    const current = (await fs.readFile(target, 'utf8').catch(() => '')).trim();
+    if (final !== current) {
+      await fs.writeFile(target, `${final}\n`, 'utf8');
+    }
+  }
+
+  /**
+   * Visit packages with ability to track duplicates
+   */
+  static async visitPackages<T>(
+    rootOrPath: PackageVisitReq<T> | string,
+    visitor: PackageVisitor<T>
+  ): Promise<Set<T>> {
+
+    const root = typeof rootOrPath === 'string' ?
+      this.packageReq<T>(rootOrPath, 'root') :
+      rootOrPath;
+
+    const seen = new Map<string, T>();
+    const queue: PackageVisitReq<T>[] = [...await visitor.init?.(root) ?? [], root];
+    const out = new Set<T>();
+
+    while (queue.length) {
+      const req = queue.pop();
+
+      if (!req || (visitor.valid && !visitor.valid(req))) {
+        continue;
+      }
+
+      const key = req.sourcePath;
+      if (seen.has(key)) {
+        await visitor.visit?.(req, seen.get(key)!);
+      } else {
+        const dep = await visitor.create(req);
+        out.add(dep);
+        await visitor.visit?.(req, dep);
+        seen.set(key, dep);
+        const children = this.getAllDependencies<T>(req.sourcePath, root.sourcePath);
+        queue.push(...children.map(x => ({ ...x, parent: dep })));
+      }
+    }
+    return (await visitor.complete?.(out)) ?? out;
+  }
+
+  /**
+   * Get version of manifest package
+   */
+  static getFrameworkVersion(): string {
+    return (this.#framework ??= this.importPackage('@travetto/manifest')).version;
+  }
+
+  /**
+   * Produce simple digest of
+   */
+  static digest(pkg: Package): PackageDigest {
+    const { main, name, author, license, version } = pkg;
+    return { name, main, author, license, version, framework: this.getFrameworkVersion() };
+  }
+
+  /**
+   * Find workspace values from rootPath
+   */
+  static resolveWorkspaces(rootPath: string): PackageWorkspaceEntry[] {
+    if (!this.#workspaces[rootPath]) {
+      const text = execSync('npm query .workspace', { cwd: rootPath, encoding: 'utf8', env: {} });
+      const res: { location: string, name: string }[] = JSON.parse(text);
+      this.#workspaces[rootPath] = res.map(d => ({ sourcePath: d.location, name: d.name }));
+    }
+    return this.#workspaces[rootPath];
+  }
+
+  /**
+   * Sync versions across a series of folders
+   */
+  static async syncVersions(folders: string[], versionMapping: Record<string, string> = {}): Promise<void> {
+    const packages = folders.map(folder => {
+      const pkg = this.readPackage(folder);
+      versionMapping[pkg.name] = `^${pkg.version}`;
+      return { folder, pkg };
+    });
+
+    for (const { pkg } of packages) {
+      for (const group of [
+        pkg.dependencies ?? {},
+        pkg.devDependencies ?? {},
+        pkg.optionalDependencies ?? {},
+        pkg.peerDependencies ?? {}
+      ]) {
+        for (const [mod, ver] of Object.entries(versionMapping)) {
+          if (mod in group && !/^[*]|(file:.*)$/.test(group[mod])) {
+            group[mod] = ver;
+          }
+        }
+      }
+    }
+
+    for (const { folder, pkg } of packages) {
+      await this.writePackageIfChanged(folder, pkg);
+    }
+  }
+}

package/src/path.ts

@@ -0,0 +1,17 @@
+import { extname, dirname, resolve, basename, delimiter, join, sep } from 'path';
+
+const posix = (file: string): string => file.replaceAll('\\', '/');
+
+const cwd = (): string => posix(process.cwd());
+
+export const path = {
+  cwd,
+  toPosix: posix,
+  delimiter,
+  basename: (file: string): string => posix(basename(file)),
+  extname: (file: string): string => posix(extname(file)),
+  dirname: (file: string): string => posix(dirname(file)),
+  resolve: (...args: string[]): string => posix(resolve(cwd(), ...args.map(f => posix(f)))),
+  join: (root: string, ...args: string[]): string => posix(join(posix(root), ...args.map(f => posix(f)))),
+  toNative: (file: string): string => file.replaceAll('/', sep)
+};

package/src/root-index.ts

@@ -0,0 +1,169 @@
+import { fileURLToPath } from 'url';
+import { createRequire } from 'module';
+
+import { path } from './path';
+import { IndexedModule, ManifestIndex } from './manifest-index';
+import { FunctionMetadata, Package, PackageDigest } from './types';
+import { PackageUtil } from './package';
+
+const METADATA = Symbol.for('@travetto/manifest:metadata');
+type Metadated = { [METADATA]: FunctionMetadata };
+
+/**
+ * Extended manifest index geared for application execution
+ */
+class $RootIndex extends ManifestIndex {
+  /**
+   * Load all source modules
+   */
+  static resolveManifestJSON(root: string, file?: string): string {
+    file = file || path.resolve(root, 'manifest.json');
+
+    // IF not a file
+    if (!file.endsWith('.json')) {
+      try {
+        // Try to resolve
+        const req = createRequire(path.resolve(root, 'node_modules'));
+        file = req.resolve(`${file}/manifest.json`);
+      } catch {
+        // Fallback to assumed node_modules pattern
+        file = `${root}/node_modules/${file}/manifest.json`;
+      }
+    }
+    return file;
+  }
+
+  #config: Package | undefined;
+  #srcCache = new Map();
+  #metadata = new Map<string, FunctionMetadata>();
+
+  constructor(output: string = process.env.TRV_OUTPUT ?? process.cwd()) {
+    super(output, $RootIndex.resolveManifestJSON(output, process.env.TRV_MANIFEST));
+  }
+
+  /**
+   * **WARNING**: This is a destructive operation, and should only be called before loading any code
+   * @private
+   */
+  reinitForModule(module: string): void {
+    this.init(this.root, $RootIndex.resolveManifestJSON(this.root, module));
+    this.#config = undefined;
+  }
+
+  /**
+   * Determines if the manifest root is the root for a monorepo
+   */
+  isMonoRepoRoot(): boolean {
+    return !!this.manifest.monoRepo && this.manifest.workspacePath === this.manifest.mainPath;
+  }
+
+  /**
+   * Asynchronously load all source files from manifest
+   */
+  async loadSource(): Promise<void> {
+    for (const { output } of this.findSrc()) {
+      await import(output);
+    }
+  }
+
+  /**
+   * Get internal id from file name and optionally, class name
+   */
+  getId(filename: string, clsName?: string): string {
+    filename = path.toPosix(filename);
+    const id = this.getEntry(filename)?.id ?? filename;
+    return clsName ? `${id}￮${clsName}` : id;
+  }
+
+  /**
+   * Get main module for manifest
+   */
+  get mainModule(): IndexedModule {
+    return this.getModule(this.mainPackage.name)!;
+  }
+
+  /**
+   * Get main package for manifest
+   */
+  get mainPackage(): Package {
+    if (!this.#config) {
+      const { output: mainFolder } = this.getModule(this.manifest.mainModule)!;
+      this.#config = {
+        ...{
+          name: 'untitled',
+          description: 'A Travetto application',
+          version: '0.0.0',
+        },
+        ...PackageUtil.readPackage(mainFolder)
+      };
+    }
+    return this.#config;
+  }
+
+  mainDigest(): PackageDigest {
+    return PackageUtil.digest(this.mainPackage);
+  }
+
+  /**
+  * Get source file from output location
+  * @param outputFile
+  */
+  getSourceFile(file: string): string {
+    if (!this.#srcCache.has(file)) {
+      if (file.startsWith('file:')) {
+        this.#srcCache.set(file, path.toPosix(fileURLToPath(file)));
+      } else {
+        this.#srcCache.set(file, path.toPosix(file));
+      }
+    }
+
+    const outputFile = this.#srcCache.get(file)!;
+    return this.getEntry(outputFile)?.source ?? outputFile;
+  }
+
+  /**
+   * Initialize the meta data for a function/class
+   * @param cls Class
+   * @param `file` Filename
+   * @param `hash` Hash of class contents
+   * @param `methods` Methods and their hashes
+   * @param `abstract` Is the class abstract
+   */
+  registerFunction(cls: Function, file: string, hash: number, methods?: Record<string, { hash: number }>, abstract?: boolean, synthetic?: boolean): boolean {
+    const source = this.getSourceFile(file);
+    const id = this.getId(source, cls.name);
+    Object.defineProperty(cls, 'Ⲑid', { value: id });
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    (cls as unknown as Metadated)[METADATA] = { id, source, hash, methods, abstract, synthetic };
+    this.#metadata.set(id, { id, source, hash, methods, abstract, synthetic });
+    return true;
+  }
+
+  /**
+   * Retrieve function metadata by function, or function id
+   */
+  getFunctionMetadataFromClass(cls: Function | undefined): FunctionMetadata | undefined {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    return (cls as unknown as Metadated)?.[METADATA];
+  }
+
+  /**
+   * Retrieve function metadata by function, or function id
+   */
+  getFunctionMetadata(clsId: string | Function): FunctionMetadata | undefined {
+    const id = clsId === undefined ? '' : typeof clsId === 'string' ? clsId : clsId.Ⲑid;
+    return this.#metadata.get(id);
+  }
+}
+
+let index: $RootIndex | undefined;
+
+try {
+  index = new $RootIndex();
+} catch (err) {
+  if (process.env.TRV_THROW_ROOT_INDEX_ERR) {
+    throw err;
+  }
+}
+
+export const RootIndex: $RootIndex = index!;