@travetto/manifest 4.0.0-rc.0 → 4.0.0-rc.1

package/src/manifest-index.ts

@@ -1,36 +1,11 @@
+import { existsSync } from 'node:fs';
+
 import { ManifestModuleUtil } from './module';
 import { path } from './path';
-
-import {
-  ManifestModule, ManifestModuleCore, ManifestModuleFile,
-  ManifestModuleFileType, ManifestModuleFolderType, ManifestModuleRole, ManifestRoot
-} from './types';
-
 import { ManifestUtil } from './util';
 
-export type FindConfig = {
-  folder?: (folder: ManifestModuleFolderType) => boolean;
-  module?: (module: IndexedModule) => boolean;
-  file?: (file: IndexedFile) => boolean;
-  sourceOnly?: boolean;
-};
-
-export type IndexedFile = {
-  id: string;
-  import: string;
-  module: string;
-  sourceFile: string;
-  outputFile: string;
-  relativeFile: string;
-  role: ManifestModuleRole;
-  type: ManifestModuleFileType;
-};
-
-export type IndexedModule = ManifestModuleCore & {
-  sourcePath: string;
-  outputPath: string;
-  files: Record<ManifestModuleFolderType, IndexedFile[]>;
-};
+import type { ManifestModuleFolderType } from './types/common';
+import type { ManifestModule, ManifestRoot, ManifestModuleFile, IndexedModule, IndexedFile, FindConfig } from './types/manifest';
 
 const TypedObject: {
   keys<T = unknown, K extends keyof T = keyof T>(o: T): K[];
@@ -43,7 +18,7 @@ const TypedObject: {
  */
 export class ManifestIndex {
 
-  #manifestFile: string;
+  #arbitraryLookup?: (parts: string[]) => ManifestModule | undefined;
   #manifest: ManifestRoot;
   #modules: IndexedModule[];
   #modulesByName: Record<string, IndexedModule> = {};
@@ -69,14 +44,8 @@ export class ManifestIndex {
     return this.#outputRoot;
   }
 
-  get manifestFile(): string {
-    return this.#manifestFile;
-  }
-
   init(manifestInput: string): void {
-    const { manifest, file } = ManifestUtil.readManifestSync(manifestInput);
-    this.#manifest = manifest;
-    this.#manifestFile = file;
+    this.#manifest = ManifestUtil.readManifestSync(manifestInput);
     this.#outputRoot = path.resolve(this.#manifest.workspace.path, this.#manifest.build.outputFolder);
     this.#index();
   }
@@ -104,12 +73,14 @@ export class ManifestIndex {
     this.#outputToEntry.clear();
     this.#importToEntry.clear();
     this.#sourceToEntry.clear();
+    this.#arbitraryLookup = undefined;
 
     this.#modules = Object.values(this.#manifest.modules)
       .map(m => ({
         ...m,
         outputPath: this.#resolveOutput(m.outputFolder),
         sourcePath: path.resolve(this.#manifest.workspace.path, m.sourceFolder),
+        children: new Set(),
         // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
         files: Object.fromEntries(
           Object.entries(m.files).map(([folder, files]) => [folder, this.#moduleFiles(m, files ?? [])])
@@ -128,6 +99,13 @@ export class ManifestIndex {
     }
     this.#modulesByName = Object.fromEntries(this.#modules.map(x => [x.name, x]));
     this.#modulesByFolder = Object.fromEntries(this.#modules.map(x => [x.sourceFolder, x]));
+
+    // Store child information
+    for (const mod of this.#modules) {
+      for (const p of mod.parents) {
+        this.#modulesByName[p]?.children.add(mod.name);
+      }
+    }
   }
 
   /**
@@ -138,11 +116,11 @@ export class ManifestIndex {
   }
 
   /**
-   * Get all local modules
+   * Get all workspace modules
    * @returns
    */
-  getLocalModules(): IndexedModule[] {
-    return this.#modules.filter(x => x.local);
+  getWorkspaceModules(): IndexedModule[] {
+    return this.#modules.filter(x => x.workspace);
   }
 
   /**
@@ -237,12 +215,9 @@ export class ManifestIndex {
   /**
    * Build module list from an expression list (e.g. `@travetto/rest,-@travetto/log)
    */
-  getModuleList(mode: 'local' | 'all', exprList: string = ''): Set<string> {
+  getModuleList(mode: 'workspace' | 'all', exprList: string = ''): Set<string> {
     const allMods = Object.keys(this.#manifest.modules);
-    const active = new Set<string>(
-      mode === 'local' ? this.getLocalModules().map(x => x.name) :
-        (mode === 'all' ? allMods : [])
-    );
+    const active = new Set<string>(mode === 'workspace' ? this.getWorkspaceModules().map(x => x.name) : allMods);
 
     for (const expr of exprList.split(/\s*,\s*/g)) {
       const [, neg, mod] = expr.match(/(-|[+])?([^+\- ]+)$/) ?? [];
@@ -257,21 +232,38 @@ export class ManifestIndex {
   }
 
   /**
-   * Get all modules (transitively) that depend on this module
+   * Get all modules, parents or children, (transitively) of the provided root, in a DFS fashion
    */
-  getDependentModules(root: IndexedModule): Set<IndexedModule> {
+  getDependentModules(root: IndexedModule | string, field: 'parents' | 'children'): IndexedModule[] {
     const seen = new Set<string>();
-    const out = new Set<IndexedModule>();
-    const toProcess = [root.name];
+    const out: IndexedModule[] = [];
+    const toProcess = [typeof root === 'string' ? root : root.name];
     while (toProcess.length) {
       const next = toProcess.shift()!;
-      if (seen.has(next)) {
-        continue;
+      if (!seen.has(next)) {
+        seen.add(next);
+        const mod = this.getModule(next)!;
+        toProcess.push(...mod[field]);
+        if (next !== this.#manifest.main.name) { // Do not include self
+          out.push(mod);
+        }
       }
-      const mod = this.getModule(next)!;
-      toProcess.push(...mod.parents);
-      out.add(mod);
     }
     return out;
   }
+
+  /**
+   * Find the module for an arbitrary source file, if it falls under a given workspace module
+   */
+  findModuleForArbitraryFile(file: string): ManifestModule | undefined {
+    const base = this.#manifest.workspace.path;
+    const lookup = this.#arbitraryLookup ??= ManifestUtil.lookupTrie(
+      Object.values(this.#manifest.modules),
+      x => x.sourceFolder.split('/'),
+      sub =>
+        !existsSync(path.resolve(base, ...sub, 'package.json')) &&
+        !existsSync(path.resolve(base, ...sub, '.git'))
+    );
+    return lookup(file.replace(`${base}/`, '').split('/'));
+  }
 }

package/src/module.ts

@@ -1,14 +1,12 @@
 import fs from 'node:fs/promises';
 
 import { path } from './path';
-import {
-  ManifestContext,
-  ManifestModule, ManifestModuleFile, ManifestModuleFileType,
-  ManifestModuleFolderType,
-  ManifestModuleRole
-} from './types';
-import { ModuleDep, ModuleDependencyVisitor } from './dependencies';
 import { PackageUtil } from './package';
+import { PackageModuleVisitor } from './dependencies';
+
+import type { ManifestModuleFileType, ManifestModuleRole, ManifestModuleFolderType } from './types/common';
+import type { ManifestModuleFile, ManifestModule, PackageModule } from './types/manifest';
+import type { ManifestContext } from './types/context';
 
 const EXT_MAPPING: Record<string, ManifestModuleFileType> = {
   '.js': 'js',
@@ -47,9 +45,10 @@ export class ManifestModuleUtil {
   /**
    * Simple file scanning
    */
-  static async scanFolder(folder: string, mainLike = false): Promise<string[]> {
-    if (!mainLike && folder in this.#scanCache) {
-      return this.#scanCache[folder];
+  static async scanFolder(folder: string, full = false): Promise<string[]> {
+    const key = `${folder}|${full}`;
+    if (key in this.#scanCache) {
+      return this.#scanCache[key];
     }
 
     if (!await fs.stat(folder).catch(() => false)) {
@@ -73,7 +72,7 @@ export class ManifestModuleUtil {
       }
 
       for (const sub of await fs.readdir(top)) {
-        const valid = !sub.startsWith('.') && (depth > 0 || mainLike);
+        const valid = !sub.startsWith('.') && (depth > 0 || full);
         const stat = await fs.stat(`${top}/${sub}`);
         if (stat.isFile()) {
           if (valid || STD_TOP_FILES.has(sub)) {
@@ -87,11 +86,7 @@ export class ManifestModuleUtil {
       }
     }
 
-    if (!mainLike) {
-      this.#scanCache[folder] = out;
-    }
-
-    return out;
+    return this.#scanCache[key] = out;
   }
 
   /**
@@ -177,12 +172,13 @@ export class ManifestModuleUtil {
   /**
    * Visit a module and describe files, and metadata
    */
-  static async describeModule(ctx: ManifestContext, dep: ModuleDep): Promise<ManifestModule> {
-    const { main, mainLike, local, name, version, sourcePath, roleSet, prod, parentSet, internal } = dep;
+  static async describeModule(ctx: ManifestContext, mod: PackageModule): Promise<ManifestModule> {
+    const { state, ...rest } = mod;
+    const sourcePath = path.resolve(ctx.workspace.path, rest.sourceFolder);
 
     const files: ManifestModule['files'] = {};
 
-    for (const file of await this.scanFolder(sourcePath, mainLike)) {
+    for (const file of await this.scanFolder(sourcePath, rest.main)) {
       // Group by top folder
       const moduleFile = file.replace(`${sourcePath}/`, '');
       const entry = await this.transformFile(moduleFile, file);
@@ -190,30 +186,20 @@ export class ManifestModuleUtil {
       (files[key] ??= []).push(entry);
     }
 
-    // Refine non-main source, remove anything in root that is source (doesn't include $index)
-    if (!mainLike) {
-      files.$root = files.$root?.filter(([file, type]) => type !== 'ts');
-    }
-
-    const roles = [...roleSet ?? []].sort();
-    const parents = [...parentSet].sort();
-    const outputFolder = `node_modules/${name}`;
-    const sourceFolder = sourcePath === ctx.workspace.path ? '' : sourcePath.replace(`${ctx.workspace.path}/`, '');
-
-    const res: ManifestModule = {
-      main, name, version, local, internal, sourceFolder, outputFolder, roles, parents, prod, files
+    return {
+      ...rest,
+      roles: [...state.roleSet].sort(),
+      parents: [...state.parentSet].sort(),
+      files
     };
-    return res;
   }
 
   /**
    * Produce all modules for a given manifest folder, adding in some given modules when developing framework
    */
   static async produceModules(ctx: ManifestContext): Promise<Record<string, ManifestModule>> {
-    const visitor = new ModuleDependencyVisitor(ctx);
-    const declared = await PackageUtil.visitPackages(visitor);
-    const sorted = [...declared].sort((a, b) => a.name.localeCompare(b.name));
-    const modules = await Promise.all(sorted.map(x => this.describeModule(ctx, x)));
+    const pkgs = await PackageUtil.visitPackages(new PackageModuleVisitor(ctx));
+    const modules = await Promise.all([...pkgs].map(x => this.describeModule(ctx, x)));
     return Object.fromEntries(modules.map(m => [m.name, m]));
   }
 

package/src/package.ts

@@ -1,10 +1,13 @@
 import { createRequire } from 'node:module';
 import { execSync } from 'node:child_process';
 
-import type { ManifestContext, NodePackageManager, Package, PackageVisitor, PackageVisitReq, PackageWorkspaceEntry } from './types';
 import { path } from './path';
 import { ManifestFileUtil } from './file';
 
+import { PackagePath, type Package, type PackageVisitor, type PackageWorkspaceEntry } from './types/package';
+import type { ManifestContext } from './types/context';
+import type { NodePackageManager } from './types/common';
+
 /**
  * Utilities for querying, traversing and reading package.json files.
  */
@@ -31,9 +34,9 @@ export class PackageUtil {
   /**
    * Resolve version path, if file: url
    */
-  static resolveVersionPath(rootPath: string, ver: string): string | undefined {
+  static resolveVersionPath(root: Package, ver: string): string | undefined {
     if (ver.startsWith('file:')) {
-      return path.resolve(rootPath, ver.replace('file:', ''));
+      return path.resolve(this.getPackagePath(root), ver.replace('file:', ''));
     } else {
       return;
     }
@@ -54,31 +57,6 @@ export class PackageUtil {
     throw new Error(`Unable to resolve: ${name}`);
   }
 
-  /**
-   * Build a package visit req
-   */
-  static packageReq<T>(sourcePath: string, prod: boolean, topLevel?: boolean): PackageVisitReq<T> {
-    return { pkg: this.readPackage(sourcePath), sourcePath, prod, topLevel };
-  }
-
-  /**
-   * Extract all dependencies from a package
-   */
-  static getAllDependencies<T = unknown>(modulePath: string, local: boolean): PackageVisitReq<T>[] {
-    const pkg = this.readPackage(modulePath);
-    const children: Record<string, PackageVisitReq<T>> = {};
-    for (const [deps, prod] of [
-      [pkg.dependencies, true],
-      ...(local ? [[pkg.devDependencies, false] as const] : []),
-    ] as const) {
-      for (const [name, version] of Object.entries(deps ?? {})) {
-        const depPath = this.resolveVersionPath(modulePath, version) ?? this.resolvePackagePath(name);
-        children[`${name}#${version}`] = this.packageReq<T>(depPath, prod, false);
-      }
-    }
-    return Object.values(children).sort((a, b) => a.pkg.name.localeCompare(b.pkg.name));
-  }
-
   /**
    * Read a package.json from a given folder
    */
@@ -92,86 +70,69 @@ export class PackageUtil {
 
     res.name ??= 'untitled'; // If a package.json (root-only) is missing a name, allows for npx execution
 
+    res[PackagePath] = modulePath;
     return res;
   }
 
   /**
-   * import a package.json from a given module name
+   * Get the package path
    */
-  static importPackage(moduleName: string): Package {
-    return this.readPackage(this.resolvePackagePath(moduleName));
+  static getPackagePath(pkg: Package): string {
+    return pkg[PackagePath];
   }
 
   /**
    * Visit packages with ability to track duplicates
    */
-  static async visitPackages<T>(visitor: PackageVisitor<T>): Promise<Set<T>> {
-
-    const root = this.packageReq<T>(visitor.rootPath, false, true);
-
-    const seen = new Map<string, T>();
-    const queue: PackageVisitReq<T>[] = [...await visitor.init?.(root) ?? [], root];
-    const out = new Set<T>();
+  static async visitPackages<T>(visitor: PackageVisitor<T>): Promise<Iterable<T>> {
+    const seen = new Set<T>();
+    const queue = [...await visitor.init()];
 
     while (queue.length) {
-      const req = queue.pop();
+      const node = queue.shift()!; // Visit initial set first
 
-      if (!req || (visitor.valid && !visitor.valid(req))) {
+      if (!visitor.valid(node)) {
         continue;
       }
 
-      const key = req.sourcePath;
-      if (seen.has(key)) {
-        await visitor.visit?.(req, seen.get(key)!);
+      visitor.visit(node);
+
+      if (seen.has(node.value)) {
+        continue;
       } else {
-        const dep = await visitor.create(req);
-        out.add(dep);
-        await visitor.visit?.(req, dep);
-        seen.set(key, dep);
-        const children = this.getAllDependencies<T>(
-          req.sourcePath,
-          // We consider a module local if its not in the node_modules
-          !req.sourcePath.includes('node_modules') && (
-            // And its the root or we are in a monorepo
-            root.sourcePath === req.sourcePath ||
-            !!root.pkg.workspaces
-          )
-        );
-        queue.push(...children.map(x => ({ ...x, parent: dep })));
+        seen.add(node.value);
       }
+
+      const children = Object.entries(node.children)
+        .map(([n, v]) => this.readPackage(this.resolveVersionPath(node.pkg, v) ?? this.resolvePackagePath(n)))
+        .map(pkg => ({ ...visitor.create(pkg), parent: node.value }));
+
+      queue.push(...children);
     }
-    return (await visitor.complete?.(out)) ?? out;
+
+    return await visitor.complete(seen);
   }
 
   /**
    * Find workspace values from rootPath
    */
-  static async resolveWorkspaces(ctx: ManifestContext, rootPath: string): Promise<PackageWorkspaceEntry[]> {
-    if (!this.#workspaces[rootPath]) {
-      const cache = path.resolve(ctx.workspace.path, ctx.build.outputFolder, 'workspaces.json');
-      try {
-        return await ManifestFileUtil.readAsJson(cache);
-      } catch (err) {
+  static async resolveWorkspaces(ctx: ManifestContext): Promise<PackageWorkspaceEntry[]> {
+    const rootPath = ctx.workspace.path;
+    const cache = path.resolve(rootPath, ctx.build.outputFolder, 'workspaces.json');
+    return this.#workspaces[rootPath] ??= await ManifestFileUtil.readAsJson<PackageWorkspaceEntry[]>(cache)
+      .catch(async () => {
         let out: PackageWorkspaceEntry[];
         switch (ctx.workspace.manager) {
+          case 'yarn':
           case 'npm': {
             const res = await this.#exec<{ location: string, name: string }[]>(rootPath, 'npm query .workspace');
-            out = res.map(d => ({ sourcePath: d.location, name: d.name }));
-            break;
-          }
-          case 'yarn': {
-            const res = await this.#exec<Record<string, { location: string }>>(rootPath, 'npm query .workspace');
-            out = Object.entries(res).map(([name, { location }]) => ({ sourcePath: location, name }));
+            out = res.map(d => ({ path: path.resolve(ctx.workspace.path, d.location), name: d.name }));
             break;
           }
         }
-
-        this.#workspaces[rootPath] = out;
-
         await ManifestFileUtil.bufferedFileWrite(cache, out);
-      }
-    }
-    return this.#workspaces[rootPath];
+        return out;
+      });
   }
 
   /**

package/src/runtime.ts

@@ -1,6 +1,9 @@
 import { path } from './path';
-import { IndexedModule, ManifestIndex } from './manifest-index';
-import { FunctionMetadata, ManifestContext, ManifestModule } from './types';
+import { ManifestIndex } from './manifest-index';
+
+import type { FunctionMetadata } from './types/common';
+import type { IndexedModule, ManifestModule } from './types/manifest';
+import type { ManifestContext } from './types/context';
 
 const METADATA = Symbol.for('@travetto/manifest:metadata');
 type Metadated = { [METADATA]: FunctionMetadata };
@@ -122,5 +125,19 @@ export const RuntimeContext = build({
    */
   workspaceRelative(...rel: string[]): string {
     return path.resolve(RuntimeIndex.manifest.workspace.path, ...rel);
+  },
+  /**
+   * Produce a workspace path for tooling, with '@' being replaced by node_module/name folder
+   * @param rel The relative path
+   */
+  toolPath(...rel: string[]): string {
+    rel = rel.flatMap(x => x === '@' ? ['node_modules', RuntimeIndex.manifest.main.name] : [x]);
+    return path.resolve(RuntimeIndex.manifest.workspace.path, RuntimeIndex.manifest.build.toolFolder, ...rel);
+  },
+  /**
+   * Are we running from a mono-root?
+   */
+  get monoRoot(): boolean {
+    return !!RuntimeIndex.manifest.workspace.mono && !RuntimeIndex.manifest.main.folder;
   }
 }, ['main', 'workspace']);

package/src/types/common.ts

@@ -0,0 +1,20 @@
+export type NodeModuleType = 'module' | 'commonjs';
+export type NodePackageManager = 'yarn' | 'npm';
+
+export type ManifestModuleFileType = 'typings' | 'ts' | 'js' | 'json' | 'package-json' | 'unknown' | 'fixture' | 'md';
+export type ManifestModuleFolderType =
+  '$root' | '$index' | '$package' |
+  'src' | 'bin' | 'support' | 'resources' | 'test' | 'doc' |
+  'test/fixtures' | 'support/fixtures' | 'support/resources' |
+  '$other' | '$transformer';
+
+export type ManifestModuleRole = 'std' | 'test' | 'doc' | 'compile' | 'build';
+
+export type FunctionMetadata = {
+  id: string;
+  source: string;
+  hash?: number;
+  methods?: Record<string, { hash: number }>;
+  synthetic?: boolean;
+  abstract?: boolean;
+};

package/src/types/context.ts

@@ -0,0 +1,40 @@
+import type { NodeModuleType, NodePackageManager } from './common';
+
+export type ManifestContext = {
+  workspace: {
+    /** Workspace path for module */
+    path: string;
+    /** The module name for the workspace root */
+    name: string;
+    /** Is the workspace a monorepo? */
+    mono?: boolean;
+    /** The module type of the workspace */
+    type: NodeModuleType;
+    /** The package manager of the workspace */
+    manager: NodePackageManager;
+    /** The default env name */
+    defaultEnv: string;
+  };
+  build: {
+    /** Compiler folder, relative to workspace */
+    compilerFolder: string;
+    /** Compiler module folder */
+    compilerModuleFolder: string;
+    /** URL for the compiler server */
+    compilerUrl: string;
+    /** Code output folder, relative to workspace */
+    outputFolder: string;
+    /** Location of development-time tool output */
+    toolFolder: string;
+  };
+  main: {
+    /** Main module for manifest */
+    name: string;
+    /** Folder, relative to workspace for main module */
+    folder: string;
+    /** Description of the main module */
+    description?: string;
+    /** Version of the main module */
+    version: string;
+  };
+};

package/src/types/manifest.ts

@@ -0,0 +1,79 @@
+import type { ManifestModuleFileType, ManifestModuleFolderType, ManifestModuleRole } from './common';
+import type { ManifestContext } from './context';
+import { Package } from './package';
+
+export type ManifestModuleFile = [string, ManifestModuleFileType, number] | [string, ManifestModuleFileType, number, ManifestModuleRole];
+
+export type ManifestDepCore = {
+  /** Package name */
+  name: string;
+  /** Package version */
+  version: string;
+  /** Is this the main module */
+  main?: boolean;
+  /** Is this a module that is part of the workspace */
+  workspace?: boolean;
+  /** Should this module be deployed to prod? */
+  prod: boolean;
+  /** Is the module intended to be published? */
+  internal?: boolean;
+};
+
+export type ManifestModuleCore = ManifestDepCore & {
+  sourceFolder: string;
+  outputFolder: string;
+  roles: ManifestModuleRole[];
+  parents: string[];
+};
+
+export type ManifestModule = ManifestModuleCore & {
+  files: Partial<Record<ManifestModuleFolderType, ManifestModuleFile[]>>;
+};
+
+export type ManifestRoot = ManifestContext & {
+  generated: number;
+  modules: Record<string, ManifestModule>;
+};
+
+export type FindConfig = {
+  folder?: (folder: ManifestModuleFolderType) => boolean;
+  module?: (module: IndexedModule) => boolean;
+  file?: (file: IndexedFile) => boolean;
+  sourceOnly?: boolean;
+};
+
+export type IndexedFile = {
+  id: string;
+  import: string;
+  module: string;
+  sourceFile: string;
+  outputFile: string;
+  relativeFile: string;
+  role: ManifestModuleRole;
+  type: ManifestModuleFileType;
+};
+
+export type IndexedModule = ManifestModuleCore & {
+  sourcePath: string;
+  outputPath: string;
+  files: Record<ManifestModuleFolderType, IndexedFile[]>;
+  children: Set<string>;
+};
+
+/** Module dependency, used in dependency visiting */
+export type PackageModule = Omit<ManifestModule, 'files' | 'parents' | 'roles'> & {
+  state: {
+    /** Role root? */
+    roleRoot?: boolean;
+    /** Travetto package info */
+    travetto?: Package['travetto'];
+    /** Prod dependencies */
+    prodDeps: Set<string>;
+    /** Set of parent package names */
+    parentSet: Set<string>;
+    /** Set of child package names */
+    childSet: Set<string>;
+    /** Defined roles for a given module */
+    roleSet: Set<ManifestModuleRole>;
+  };
+};