@travetto/manifest 3.4.0-rc.0 → 3.4.0-rc.3

package/README.md

@@ -99,7 +99,7 @@ By default, all paths within the framework are assumed to be in a POSIX style, a
   "outputFolder": ".trv/output",
   "toolFolder": ".trv/tool",
   "compilerFolder": ".trv/compiler",
-  "compilerUrl": "http://127.0.0.1:22154",
+  "compilerUrl": "http://127.0.0.1:26803",
   "frameworkVersion": "x.x.x",
   "mainModule": "@travetto/manifest",
   "mainFolder": "module/manifest",
@@ -114,6 +114,9 @@ By default, all paths within the framework are assumed to be in a POSIX style, a
       "internal": false,
       "sourceFolder": "module/manifest",
       "outputFolder": "node_modules/@travetto/manifest",
+      "roles": [ "std" ],
+      "parents": [],
+      "prod": true,
       "files": {
         "$root": [
           [ "DOC.html", "unknown", 1868155200000 ],
@@ -158,9 +161,7 @@ By default, all paths within the framework are assumed to be in a POSIX style, a
           [ "bin/context.d.ts", "typings", 1868155200000 ],
           [ "bin/context.js", "js", 1868155200000 ]
         ]
-      },
-      "profiles": [ "std" ],
-      "parents": []
+      }
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/manifest",
-  "version": "3.4.0-rc.0",
+  "version": "3.4.0-rc.3",
   "description": "Support for project indexing, manifesting, along with file watching",
   "keywords": [
     "path",
@@ -24,7 +24,7 @@
   ],
   "main": "__index__.ts",
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=20.0.0"
   },
   "repository": {
     "url": "https://github.com/travetto/travetto.git",

package/src/dependencies.ts

@@ -1,8 +1,9 @@
 import { PackageUtil } from './package';
 import { path } from './path';
-import { ManifestContext, ManifestProfile, PackageRel, PackageVisitor, PackageVisitReq } from './types';
+import { ManifestContext, ManifestModuleRole, PackageVisitor, PackageVisitReq, Package } from './types';
 
 export type ModuleDep = {
+  pkg: Package;
   version: string;
   name: string;
   main?: boolean;
@@ -10,9 +11,11 @@ export type ModuleDep = {
   local?: boolean;
   internal?: boolean;
   sourcePath: string;
-  childSet: Map<string, Set<PackageRel>>;
+  childSet: Set<string>;
   parentSet: Set<string>;
-  profileSet: Set<ManifestProfile>;
+  roleSet: Set<ManifestModuleRole>;
+  prod: boolean;
+  topLevel?: boolean;
 };
 
 /**
@@ -60,19 +63,13 @@ export class ModuleDependencyVisitor implements PackageVisitor<ModuleDep> {
       ...workspaceModules.map(x => x.name)
     ]);
 
-    const globals = [
-      ...(workspacePkg.travetto?.globalModules ?? []),
-      ...(pkg.travetto?.globalModules ?? [])
-    ]
-      .map(f => PackageUtil.resolvePackagePath(f));
+    const globals = (workspacePkg.travetto?.globalModules ?? [])
+      .map(name => PackageUtil.packageReq<ModuleDep>(PackageUtil.resolvePackagePath(name), name in (workspacePkg.dependencies ?? {}), true));
 
     const workspaceModuleDeps = workspaceModules
-      .map(entry => path.resolve(req.sourcePath, entry.sourcePath));
+      .map(entry => PackageUtil.packageReq<ModuleDep>(path.resolve(req.sourcePath, entry.sourcePath), false, true));
 
-    return [
-      ...globals,
-      ...workspaceModuleDeps
-    ].map(s => PackageUtil.packageReq(s, 'global'));
+    return [...globals, ...workspaceModuleDeps];
   }
 
   /**
@@ -80,8 +77,7 @@ export class ModuleDependencyVisitor implements PackageVisitor<ModuleDep> {
    */
   valid(req: PackageVisitReq<ModuleDep>): boolean {
     return req.sourcePath === this.#mainSourcePath || (
-      req.rel !== 'peer' &&
-      (!!req.pkg.travetto || req.pkg.private === true || !req.sourcePath.includes('node_modules') || req.rel === 'global')
+      (!!req.pkg.travetto || req.pkg.private === true || !req.sourcePath.includes('node_modules'))
     );
   }
 
@@ -89,18 +85,16 @@ export class ModuleDependencyVisitor implements PackageVisitor<ModuleDep> {
    * Create dependency from request
    */
   create(req: PackageVisitReq<ModuleDep>): ModuleDep {
-    const { pkg: { name, version, travetto: { profiles = [] } = {}, ...pkg }, sourcePath } = req;
-    const profileSet = new Set<ManifestProfile>([
-      ...profiles ?? []
-    ]);
+    const { pkg, sourcePath } = req;
+    const { name, version } = pkg;
     const main = name === this.ctx.mainModule;
     const mainSource = main || this.#mainPatterns.some(x => x.test(name));
     const internal = pkg.private === true;
     const local = internal || mainSource || !sourcePath.includes('node_modules');
 
     const dep = {
-      name, version, sourcePath, main, mainSource, local, internal,
-      parentSet: new Set([]), childSet: new Map(), profileSet
+      name, version, sourcePath, main, mainSource, local, internal, pkg: req.pkg,
+      parentSet: new Set([]), childSet: new Set([]), roleSet: new Set([]), prod: req.prod, topLevel: req.topLevel
     };
 
     return dep;
@@ -113,28 +107,30 @@ export class ModuleDependencyVisitor implements PackageVisitor<ModuleDep> {
     const { parent } = req;
     if (parent && dep.name !== this.ctx.mainModule) {
       dep.parentSet.add(parent.name);
-      const set = parent.childSet.get(dep.name) ?? new Set();
-      parent.childSet.set(dep.name, set);
-      set.add(req.rel);
+      parent.childSet.add(dep.name);
     }
   }
 
   /**
-   * Propagate profile/relationship information through graph
+   * Propagate prod, role information through graph
    */
   complete(deps: Set<ModuleDep>): Set<ModuleDep> {
-    const mapping = new Map<string, { parent: Set<string>, child: Map<string, Set<PackageRel>>, el: ModuleDep }>();
+    const mapping = new Map<string, { parent: Set<string>, child: Set<string>, el: ModuleDep }>();
     for (const el of deps) {
-      mapping.set(el.name, { parent: new Set(el.parentSet), child: new Map(el.childSet), el });
+      mapping.set(el.name, { parent: new Set(el.parentSet), child: new Set(el.childSet), el });
     }
 
     const main = mapping.get(this.ctx.mainModule)!;
 
     // Visit all direct dependencies and mark
-    for (const [name, relSet] of main.child) {
-      const childDep = mapping.get(name)!.el;
-      if (!relSet.has('dev')) {
-        childDep.profileSet.add('std');
+    for (const { el } of mapping.values()) {
+      if (el.topLevel) {
+        el.roleSet = new Set(el.pkg.travetto?.roles ?? []);
+        if (!el.roleSet.size) {
+          el.roleSet.add('std');
+        }
+      } else if (!main.child.has(el.name)) { // Not a direct descendent
+        el.prod = false;
       }
     }
 
@@ -148,9 +144,10 @@ export class ModuleDependencyVisitor implements PackageVisitor<ModuleDep> {
         for (const c of child.keys()) {
           const { el: cDep, parent } = mapping.get(c)!;
           parent.delete(el.name); // Remove from child
-          for (const prof of el.profileSet) {
-            cDep.profileSet.add(prof);
+          for (const role of el.roleSet) {
+            cDep.roleSet.add(role);
           }
+          cDep.prod ||= el.prod; // Allow prod to trickle down as needed
         }
       }
       // Remove from mapping
@@ -159,8 +156,10 @@ export class ModuleDependencyVisitor implements PackageVisitor<ModuleDep> {
       }
     }
 
-    // Color the main folder as std
-    main.el.profileSet.add('std');
+    // Color parent as final step
+    main.el.prod = true;
+    main.el.roleSet.add('std');
+
     return deps;
   }
 }

package/src/manifest-index.ts

@@ -3,18 +3,16 @@ import { path } from './path';
 
 import {
   ManifestModule, ManifestModuleCore, ManifestModuleFile,
-  ManifestModuleFileType, ManifestModuleFolderType, ManifestProfile, ManifestRoot
+  ManifestModuleFileType, ManifestModuleFolderType, ManifestModuleRole, ManifestRoot
 } from './types';
 
 import { ManifestUtil } from './util';
 
-type ScanTest = ((full: string) => boolean) | { test: (full: string) => boolean };
 export type FindConfig = {
-  folders?: ManifestModuleFolderType[];
-  filter?: ScanTest;
-  includeIndex?: boolean;
-  profiles?: string[];
-  checkProfile?: boolean;
+  folder?: (folder: ManifestModuleFolderType) => boolean;
+  module?: (module: IndexedModule) => boolean;
+  file?: (file: IndexedFile) => boolean;
+  sourceOnly?: boolean;
 };
 
 export type IndexedFile = {
@@ -24,7 +22,7 @@ export type IndexedFile = {
   sourceFile: string;
   outputFile: string;
   relativeFile: string;
-  profile: ManifestProfile;
+  role: ManifestModuleRole;
   type: ManifestModuleFileType;
 };
 
@@ -34,6 +32,12 @@ export type IndexedModule = ManifestModuleCore & {
   files: Record<ManifestModuleFolderType, IndexedFile[]>;
 };
 
+const TypedObject: {
+  keys<T = unknown, K extends keyof T = keyof T>(o: T): K[];
+  fromEntries<K extends string | symbol, V>(items: ([K, V] | readonly [K, V])[]): Record<K, V>;
+  entries<K extends Record<symbol | string, unknown>>(record: K): [keyof K, K[keyof K]][];
+} & ObjectConstructor = Object;
+
 /**
  * Manifest index
  */
@@ -78,7 +82,7 @@ export class ManifestIndex {
   }
 
   #moduleFiles(m: ManifestModule, files: ManifestModuleFile[]): IndexedFile[] {
-    return files.map(([f, type, ts, profile = 'std']) => {
+    return files.map(([f, type, ts, role = 'std']) => {
       const isSource = type === 'ts' || type === 'js';
       const sourceFile = path.resolve(this.#manifest.workspacePath, m.sourceFolder, f);
       const js = isSource ? ManifestModuleUtil.sourceToOutputExt(f) : f;
@@ -89,7 +93,7 @@ export class ManifestIndex {
         id = ManifestModuleUtil.sourceToBlankExt(id);
       }
 
-      return { id, type, sourceFile, outputFile, import: modImport, profile, relativeFile: f, module: m.name };
+      return { id, type, sourceFile, outputFile, import: modImport, role, relativeFile: f, module: m.name };
     });
   }
 
@@ -143,58 +147,27 @@ export class ManifestIndex {
 
   /**
    * Find files from the index
-   * @param folder The sub-folder to check into
-   * @param filter The filter to determine if this is a valid support file
+   * @param config The configuration for controlling the find process
    */
   find(config: FindConfig): IndexedFile[] {
-    const { filter: f, folders } = config;
-    const filter = f ? 'test' in f ? f.test.bind(f) : f : f;
-
-    let idx = this.#modules;
-
-    const checkProfile = config.checkProfile ?? true;
-
-    const activeProfiles = new Set(['std', ...(config.profiles ?? process.env.TRV_PROFILES?.split(/\s*,\s*/g) ?? [])]);
-
-    if (checkProfile) {
-      idx = idx.filter(m => m.profiles.length === 0 || m.profiles.some(p => activeProfiles.has(p)));
-    }
-
-    let searchSpace = folders ?
-      idx.flatMap(m => [...folders.flatMap(fo => m.files[fo] ?? []), ...(config.includeIndex ? (m.files.$index ?? []) : [])]) :
-      idx.flatMap(m => [...Object.values(m.files)].flat());
-
-    if (checkProfile) {
-      searchSpace = searchSpace.filter(fi => activeProfiles.has(fi.profile));
+    const searchSpace: IndexedFile[] = [];
+    for (const m of this.#modules) {
+      if (config.module?.(m) ?? true) {
+        for (const [folder, files] of TypedObject.entries(m.files)) {
+          if (config.folder?.(folder) ?? true) {
+            for (const file of files) {
+              if (
+                (config.file?.(file) ?? true) &&
+                (config.sourceOnly === false || file.type === 'ts')
+              ) {
+                searchSpace.push(file);
+              }
+            }
+          }
+        }
+      }
     }
-
-    return searchSpace
-      .filter(({ type }) => type === 'ts')
-      .filter(({ sourceFile: source }) => filter?.(source) ?? true);
-  }
-
-  /**
-   * Find files from the index
-   * @param filter The filter to determine if this is a valid support file
-   */
-  findSupport(config: Omit<FindConfig, 'folder'>): IndexedFile[] {
-    return this.find({ ...config, folders: ['support'] });
-  }
-
-  /**
-   * Find files from the index
-   * @param filter The filter to determine if this is a valid support file
-   */
-  findSrc(config: Omit<FindConfig, 'folder'> = {}): IndexedFile[] {
-    return this.find({ ...config, includeIndex: true, folders: ['src'] });
-  }
-
-  /**
-   * Find files from the index
-   * @param filter The filter to determine if this is a valid support file
-   */
-  findTest(config: Omit<FindConfig, 'folder'>): IndexedFile[] {
-    return this.find({ ...config, folders: ['test'] });
+    return searchSpace;
   }
 
   /**

package/src/module.ts

@@ -4,7 +4,8 @@ import { path } from './path';
 import {
   ManifestContext,
   ManifestModule, ManifestModuleFile, ManifestModuleFileType,
-  ManifestModuleFolderType, ManifestProfile
+  ManifestModuleFolderType,
+  ManifestModuleRole
 } from './types';
 import { ModuleDep, ModuleDependencyVisitor } from './dependencies';
 import { PackageUtil } from './package';
@@ -114,7 +115,7 @@ export class ManifestModuleUtil {
   /**
    * Get file type for a file name
    */
-  static getFileProfile(moduleFile: string): ManifestProfile | undefined {
+  static getFileRole(moduleFile: string): ManifestModuleRole | undefined {
     if (moduleFile.startsWith('support/transform')) {
       return 'compile';
     } else if (moduleFile.startsWith('support/test/') || moduleFile.startsWith('test/')) {
@@ -167,15 +168,15 @@ export class ManifestModuleUtil {
    */
   static async transformFile(moduleFile: string, full: string): Promise<ManifestModuleFile> {
     const res: ManifestModuleFile = [moduleFile, this.getFileType(moduleFile), this.#getNewest(await fs.stat(full))];
-    const profile = this.getFileProfile(moduleFile);
-    return profile ? [...res, profile] : res;
+    const role = this.getFileRole(moduleFile);
+    return role ? [...res, role] : res;
   }
 
   /**
    * Visit a module and describe files, and metadata
    */
   static async describeModule(ctx: ManifestContext, dep: ModuleDep): Promise<ManifestModule> {
-    const { main, mainSource, local, name, version, sourcePath, profileSet, parentSet, internal } = dep;
+    const { main, mainSource, local, name, version, sourcePath, roleSet, prod, parentSet, internal } = dep;
 
     const files: ManifestModule['files'] = {};
 
@@ -192,12 +193,12 @@ export class ManifestModuleUtil {
       files.$root = files.$root?.filter(([file, type]) => type !== 'ts');
     }
 
-    const profiles = [...profileSet].sort();
+    const roles = [...roleSet ?? []].sort();
     const parents = [...parentSet].sort();
     const outputFolder = `node_modules/${name}`;
     const sourceFolder = sourcePath === ctx.workspacePath ? '' : sourcePath.replace(`${ctx.workspacePath}/`, '');
 
-    const res = { main, name, version, local, internal, sourceFolder, outputFolder, files, profiles, parents, };
+    const res = { main, name, version, local, internal, sourceFolder, outputFolder, roles, parents, prod, files };
     return res;
   }
 

package/src/package.ts

@@ -1,7 +1,7 @@
 import { createRequire } from 'module';
 import { execSync } from 'child_process';
 
-import { ManifestContext, Package, PackageRel, PackageVisitor, PackageVisitReq, PackageWorkspaceEntry } from './types';
+import { ManifestContext, Package, PackageVisitor, PackageVisitReq, PackageWorkspaceEntry } from './types';
 import { path } from './path';
 import { ManifestFileUtil } from './file';
 
@@ -59,33 +59,23 @@ export class PackageUtil {
   /**
    * Build a package visit req
    */
-  static packageReq<T>(sourcePath: string, rel: PackageRel): PackageVisitReq<T> {
-    return { pkg: this.readPackage(sourcePath), sourcePath, rel };
+  static packageReq<T>(sourcePath: string, prod: boolean, topLevel?: boolean): PackageVisitReq<T> {
+    return { pkg: this.readPackage(sourcePath), sourcePath, prod, topLevel };
   }
 
   /**
    * Extract all dependencies from a package
    */
-  static getAllDependencies<T = unknown>(modulePath: string, rootPath: string): PackageVisitReq<T>[] {
+  static getAllDependencies<T = unknown>(modulePath: string, local: boolean): PackageVisitReq<T>[] {
     const pkg = this.readPackage(modulePath);
     const children: Record<string, PackageVisitReq<T>> = {};
-    const local = modulePath === rootPath && !modulePath.includes('node_modules');
-    for (const [deps, rel] of [
-      [pkg.dependencies, 'prod'],
-      [pkg.peerDependencies, 'peer'],
-      [pkg.optionalDependencies, 'opt'],
-      ...(local ? [[pkg.devDependencies, 'dev'] as const] : []),
+    for (const [deps, prod] of [
+      [pkg.dependencies, true],
+      ...(local ? [[pkg.devDependencies, false] as const] : []),
     ] as const) {
       for (const [name, version] of Object.entries(deps ?? {})) {
-        try {
-          const depPath = this.resolveVersionPath(modulePath, version) ?? this.resolvePackagePath(name);
-          children[`${name}#${version}`] = this.packageReq<T>(depPath, rel);
-        } catch (err) {
-          if (rel === 'opt' || (rel === 'peer' && !!pkg.peerDependenciesMeta?.[name].optional)) {
-            continue;
-          }
-          throw err;
-        }
+        const depPath = this.resolveVersionPath(modulePath, version) ?? this.resolvePackagePath(name);
+        children[`${name}#${version}`] = this.packageReq<T>(depPath, prod, false);
       }
     }
     return Object.values(children).sort((a, b) => a.pkg.name.localeCompare(b.pkg.name));
@@ -123,7 +113,7 @@ export class PackageUtil {
   ): Promise<Set<T>> {
 
     const root = typeof rootOrPath === 'string' ?
-      this.packageReq<T>(rootOrPath, 'root') :
+      this.packageReq<T>(rootOrPath, false, true) :
       rootOrPath;
 
     const seen = new Map<string, T>();
@@ -145,7 +135,15 @@ export class PackageUtil {
         out.add(dep);
         await visitor.visit?.(req, dep);
         seen.set(key, dep);
-        const children = this.getAllDependencies<T>(req.sourcePath, root.sourcePath);
+        const children = this.getAllDependencies<T>(
+          req.sourcePath,
+          // We consider a module local if its not in the node_modules
+          !req.sourcePath.includes('node_modules') && (
+            // And its the root or we are in a monorepo
+            root.sourcePath === req.sourcePath ||
+            !!root.pkg.workspaces
+          )
+        );
         queue.push(...children.map(x => ({ ...x, parent: dep })));
       }
     }

package/src/root-index.ts

@@ -27,15 +27,6 @@ class $RootIndex extends ManifestIndex {
     return !!this.manifest.monoRepo && !this.manifest.mainFolder;
   }
 
-  /**
-   * Asynchronously load all source files from manifest
-   */
-  async loadSource(): Promise<void> {
-    for (const { import: imp } of this.findSrc()) {
-      await import(imp);
-    }
-  }
-
   /**
    * Get internal id from file name and optionally, class name
    */

package/src/types.ts

@@ -1,3 +1,5 @@
+export type NodeModuleType = 'module' | 'commonjs';
+
 export type ManifestModuleFileType = 'typings' | 'ts' | 'js' | 'json' | 'package-json' | 'unknown' | 'fixture' | 'md';
 export type ManifestModuleFolderType =
   '$root' | '$index' | '$package' |
@@ -5,10 +7,9 @@ export type ManifestModuleFolderType =
   'test/fixtures' | 'support/fixtures' | 'support/resources' |
   '$other' | '$transformer';
 
-export type ManifestProfile = 'compile' | 'test' | 'doc' | 'build' | 'std';
-export type PackageRel = 'dev' | 'prod' | 'peer' | 'opt' | 'root' | 'global';
+export type ManifestModuleRole = 'std' | 'test' | 'doc' | 'compile' | 'build';
 
-export type ManifestModuleFile = [string, ManifestModuleFileType, number] | [string, ManifestModuleFileType, number, ManifestProfile];
+export type ManifestModuleFile = [string, ManifestModuleFileType, number] | [string, ManifestModuleFileType, number, ManifestModuleRole];
 export type ManifestModuleCore = {
   name: string;
   main?: boolean;
@@ -16,7 +17,8 @@ export type ManifestModuleCore = {
   version: string;
   sourceFolder: string;
   outputFolder: string;
-  profiles: ManifestProfile[];
+  prod: boolean;
+  roles: ManifestModuleRole[];
   parents: string[];
   internal?: boolean;
 };
@@ -33,7 +35,7 @@ export type ManifestContext = {
   toolFolder: string;
   compilerFolder: string;
   monoRepo?: boolean;
-  moduleType: 'module' | 'commonjs';
+  moduleType: NodeModuleType;
   packageManager: 'yarn' | 'npm';
   frameworkVersion: string;
   description?: string;
@@ -48,7 +50,7 @@ export type ManifestRoot = ManifestContext & {
 
 export type Package = {
   name: string;
-  type?: 'module' | 'commonjs';
+  type?: NodeModuleType;
   version: string;
   description?: string;
   license?: string;
@@ -76,7 +78,7 @@ export type Package = {
   travetto?: {
     isolated?: boolean;
     displayName?: string;
-    profiles?: ManifestProfile[];
+    roles?: ManifestModuleRole[];
     globalModules?: string[];
     mainSource?: string[];
     docOutput?: string[];
@@ -95,7 +97,7 @@ export type Package = {
 
 type OrProm<T> = T | Promise<T>;
 
-export type PackageVisitReq<T> = { pkg: Package, rel: PackageRel, sourcePath: string, parent?: T };
+export type PackageVisitReq<T> = { pkg: Package, prod: boolean, sourcePath: string, parent?: T, topLevel?: boolean };
 export type PackageVisitor<T> = {
   init?(req: PackageVisitReq<T>): OrProm<undefined | void | PackageVisitReq<T>[]>;
   valid?(req: PackageVisitReq<T>): boolean;

package/src/util.ts

@@ -38,7 +38,7 @@ export class ManifestUtil {
       // If in prod mode, only include std modules
       modules: Object.fromEntries(
         Object.values(manifest.modules)
-          .filter(x => x.profiles.includes('std'))
+          .filter(x => x.prod)
           .map(m => [m.name, m])
       ),
       // Mark output folder/workspace path as portable