@travetto/manifest 3.0.0-rc.3

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 ArcSine Technologies
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,14 @@
+<!-- This file was generated by @travetto/doc and should not be modified directly -->
+<!-- Please modify https://github.com/travetto/travetto/tree/main/module/manifest/DOC.ts and execute "npx trv doc" to rebuild -->
+# Manifest
+## Manifest support
+
+**Install: @travetto/manifest**
+```bash
+npm install @travetto/manifest
+```
+
+This module provides functionality for basic path functionality and common typings for manifests
+
+### Module Indexing
+The bootstrap process will also produce an index of all source files, which allows for fast in-memory scanning.  This allows for all the automatic discovery that is used within the framework.

package/__index__.ts

@@ -0,0 +1,10 @@
+/// <reference path="./src/typings.d.ts" />
+
+export * from './src/path';
+export * from './src/module';
+export * from './src/manifest-index';
+export * from './src/root-index';
+export * from './src/delta';
+export * from './src/package';
+export * from './src/manifest';
+export * from './src/types';

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@travetto/manifest",
+  "version": "3.0.0-rc.3",
+  "description": "Manifest support",
+  "keywords": [
+    "path",
+    "package",
+    "manifest",
+    "travetto",
+    "typescript"
+  ],
+  "homepage": "https://travetto.io",
+  "license": "MIT",
+  "author": {
+    "email": "travetto.framework@gmail.com",
+    "name": "Travetto Framework"
+  },
+  "files": [
+    "__index__.ts",
+    "src",
+    "support"
+  ],
+  "main": "__index__.ts",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "repository": {
+    "url": "https://github.com/travetto/travetto.git",
+    "directory": "module/manifest"
+  },
+  "dependencies": {},
+  "travetto": {
+    "displayName": "Manifest",
+    "docBaseUrl": "https://github.com/travetto/travetto/tree/main"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/delta.ts

@@ -0,0 +1,114 @@
+import { type Stats } from 'fs';
+import fs from 'fs/promises';
+import {
+  ManifestDelta, ManifestDeltaEvent, ManifestDeltaModule,
+  ManifestModule, ManifestModuleFile, ManifestModuleFolderType,
+  ManifestRoot
+} from './types';
+
+const VALID_SOURCE_FOLDERS = new Set<ManifestModuleFolderType>(['bin', 'src', 'test', 'support', '$index', '$package', 'doc']);
+
+/**
+ * Produce delta for the manifest
+ */
+export class ManifestDeltaUtil {
+
+  static #getNewest(stat: { mtimeMs: number, ctimeMs: number }): number {
+    return Math.max(stat.mtimeMs, stat.ctimeMs);
+  }
+
+  /**
+   * Produce delta between two manifest modules, relative to an output folder
+   */
+  static async #deltaModules(outputFolder: string, left: ManifestDeltaModule, right: ManifestDeltaModule): Promise<ManifestDeltaEvent[]> {
+    const out: ManifestDeltaEvent[] = [];
+    const getStat = (f: string): Promise<Stats | void> =>
+      fs.stat(`${outputFolder}/${left.output}/${f.replace(/[.]ts$/, '.js')}`).catch(() => { });
+
+    const add = (file: string, type: ManifestDeltaEvent['type']): unknown =>
+      out.push({ file, module: left.name, type });
+
+    for (const el of Object.keys(left.files)) {
+      if (!(el in right.files)) {
+        const [, , leftTs] = left.files[el];
+        const stat = await getStat(el);
+        if (stat && leftTs < this.#getNewest(stat)) {
+          // If file pre-exists manifest, be cool
+          continue;
+        }
+        add(el, 'added');
+      } else {
+        const [, , leftTs] = left.files[el];
+        const [, , rightTs] = right.files[el];
+        if (leftTs !== rightTs) {
+          const stat = await getStat(el);
+          if (!stat) {
+            add(el, 'missing');
+          } else if (leftTs > this.#getNewest(stat!)) {
+            add(el, 'changed');
+          }
+        } else {
+          const stat = await getStat(el);
+          if (!stat) {
+            add(el, 'missing');
+          } else if (this.#getNewest(stat!) < leftTs) {
+            add(el, 'dirty');
+          }
+        }
+      }
+    }
+    for (const el of Object.keys(right.files)) {
+      if (!(el in left.files)) {
+        const stat = await getStat(el);
+        if (stat) {
+          add(el, 'removed');
+        }
+      }
+    }
+    return out;
+  }
+
+  /**
+   * Collapse all files in a module
+   * @param {ManifestModule} m
+   * @returns {}
+   */
+  static #flattenModuleFiles(m: ManifestModule): Record<string, ManifestModuleFile> {
+    const out: Record<string, ManifestModuleFile> = {};
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    for (const key of Object.keys(m.files) as (ManifestModuleFolderType[])) {
+      if (!VALID_SOURCE_FOLDERS.has(key)) {
+        continue;
+      }
+      for (const [name, type, date] of m.files?.[key] ?? []) {
+        if (type === 'ts' || type === 'js' || type === 'package-json') {
+          out[name] = [name, type, date];
+        }
+      }
+    }
+    return out;
+  }
+
+  /**
+   * Produce delta between ttwo manifest roots, relative to a single output folder
+   */
+  static async produceDelta(outputFolder: string, left: ManifestRoot, right: ManifestRoot): Promise<ManifestDelta> {
+    const deltaLeft = Object.fromEntries(
+      Object.values(left.modules)
+        .map(m => [m.name, { ...m, files: this.#flattenModuleFiles(m) }])
+    );
+
+    const deltaRight = Object.fromEntries(
+      Object.values(right.modules)
+        .map(m => [m.name, { ...m, files: this.#flattenModuleFiles(m) }])
+    );
+
+    const out: Record<string, ManifestDeltaEvent[]> = {};
+
+    for (const [name, lMod] of Object.entries(deltaLeft)) {
+      out[name] = await this.#deltaModules(outputFolder, lMod, deltaRight[name] ?? { files: {}, name });
+    }
+
+    return out;
+  }
+}

package/src/dependencies.ts

@@ -0,0 +1,163 @@
+import { PackageUtil } from './package';
+import { path } from './path';
+import { ManifestContext, ManifestProfile, PackageRel, PackageVisitor, PackageVisitReq } from './types';
+
+export type ModuleDep = {
+  version: string;
+  name: string;
+  main?: boolean;
+  mainSource?: boolean;
+  local?: boolean;
+  internal?: boolean;
+  sourcePath: string;
+  childSet: Map<string, Set<PackageRel>>;
+  parentSet: Set<string>;
+  profileSet: Set<ManifestProfile>;
+};
+
+/**
+ * Used for walking dependencies for collecting modules for the manifest
+ */
+export class ModuleDependencyVisitor implements PackageVisitor<ModuleDep> {
+
+  /**
+   * Get main patterns for detecting if a module should be treated as main
+   */
+  static getMainPatterns(mainModule: string, mergeModules: string[]): RegExp[] {
+    const groups: Record<string, string[]> = { [mainModule]: [] };
+    for (const el of mergeModules) {
+      if (el.includes('/')) {
+        const [grp, sub] = el.split('/');
+        (groups[`${grp}/`] ??= []).push(sub);
+      } else {
+        (groups[el] ??= []);
+      }
+    }
+
+    return Object.entries(groups)
+      .map(([root, subs]) => subs.length ? `${root}(${subs.join('|')})` : root)
+      .map(x => new RegExp(`^${x.replace(/[*]/g, '.*?')}$`));
+  }
+
+  constructor(public ctx: ManifestContext) { }
+
+  #mainPatterns: RegExp[] = [];
+
+  /**
+   * Initialize visitor, and provide global dependencies
+   */
+  async init(req: PackageVisitReq<ModuleDep>): Promise<PackageVisitReq<ModuleDep>[]> {
+    const pkg = PackageUtil.readPackage(req.sourcePath);
+    const workspacePkg = PackageUtil.readPackage(this.ctx.workspacePath);
+    const workspaceModules = pkg.workspaces?.length ? (await PackageUtil.resolveWorkspaces(req.sourcePath)) : [];
+
+    this.#mainPatterns = ModuleDependencyVisitor.getMainPatterns(pkg.name, [
+      ...pkg.travetto?.mainSource ?? [],
+      // Add workspace folders, for tests and docs
+      ...workspaceModules.map(x => x.name)
+    ]);
+
+    const globals = [
+      ...(workspacePkg.travetto?.globalModules ?? []),
+      ...(pkg.travetto?.globalModules ?? [])
+    ]
+      .map(f => PackageUtil.resolvePackagePath(f));
+
+    const workspaceModuleDeps = workspaceModules
+      .map(entry => path.resolve(req.sourcePath, entry.sourcePath));
+
+    return [
+      ...globals,
+      ...workspaceModuleDeps
+    ].map(s => PackageUtil.packageReq(s, 'global'));
+  }
+
+  /**
+   * Is valid dependency for searching
+   */
+  valid(req: PackageVisitReq<ModuleDep>): boolean {
+    return req.sourcePath === path.cwd() || (
+      req.rel !== 'peer' &&
+      (!!req.pkg.travetto || req.rel === 'global')
+    );
+  }
+
+  /**
+   * Create dependency from request
+   */
+  create(req: PackageVisitReq<ModuleDep>): ModuleDep {
+    const { pkg: { name, version, travetto: { profiles = [] } = {}, ...pkg }, sourcePath } = req;
+    const profileSet = new Set<ManifestProfile>([
+      ...profiles ?? []
+    ]);
+    const main = name === this.ctx.mainModule;
+    const mainSource = main || this.#mainPatterns.some(x => x.test(name));
+    const internal = pkg.private === true;
+    const local = internal || mainSource || !sourcePath.includes('node_modules');
+
+    const dep = {
+      name, version, sourcePath, main, mainSource, local, internal,
+      parentSet: new Set([]), childSet: new Map(), profileSet
+    };
+
+    return dep;
+  }
+
+  /**
+   * Visit dependency
+   */
+  visit(req: PackageVisitReq<ModuleDep>, dep: ModuleDep): void {
+    const { parent } = req;
+    if (parent && dep.name !== this.ctx.mainModule) {
+      dep.parentSet.add(parent.name);
+      const set = parent.childSet.get(dep.name) ?? new Set();
+      parent.childSet.set(dep.name, set);
+      set.add(req.rel);
+    }
+  }
+
+  /**
+   * Propagate profile/relationship information through graph
+   */
+  complete(deps: Set<ModuleDep>): Set<ModuleDep> {
+    const mapping = new Map<string, { parent: Set<string>, child: Map<string, Set<PackageRel>>, el: ModuleDep }>();
+    for (const el of deps) {
+      mapping.set(el.name, { parent: new Set(el.parentSet), child: new Map(el.childSet), el });
+    }
+
+    const main = mapping.get(this.ctx.mainModule)!;
+
+    // Visit all direct dependencies and mark
+    for (const [name, relSet] of main.child) {
+      const childDep = mapping.get(name)!.el;
+      if (!relSet.has('dev')) {
+        childDep.profileSet.add('std');
+      }
+    }
+
+    while (mapping.size > 0) {
+      const toProcess = [...mapping.values()].filter(x => x.parent.size === 0);
+      if (!toProcess.length) {
+        throw new Error(`We have reached a cycle for ${[...mapping.keys()]}`);
+      }
+      // Propagate
+      for (const { el, child } of toProcess) {
+        for (const c of child.keys()) {
+          const { el: cDep, parent } = mapping.get(c)!;
+          parent.delete(el.name); // Remove from child
+          for (const prof of el.profileSet) {
+            cDep.profileSet.add(prof);
+          }
+        }
+      }
+      // Remove from mapping
+      for (const { el } of toProcess) {
+        mapping.delete(el.name);
+      }
+    }
+
+    // Color the main folder as std
+    main.el.profileSet.add('std');
+    return deps;
+  }
+}

package/src/manifest-index.ts

@@ -0,0 +1,290 @@
+import fs from 'fs';
+
+import { path } from './path';
+
+import {
+  ManifestModule, ManifestModuleCore, ManifestModuleFile,
+  ManifestModuleFileType, ManifestModuleFolderType, ManifestProfile, ManifestRoot
+} from './types';
+
+type ScanTest = ((full: string) => boolean) | { test: (full: string) => boolean };
+export type FindConfig = {
+  folders?: ManifestModuleFolderType[];
+  filter?: ScanTest;
+  includeIndex?: boolean;
+  profiles?: string[];
+  checkProfile?: boolean;
+};
+
+export type IndexedFile = {
+  id: string;
+  import: string;
+  module: string;
+  source: string;
+  output: string;
+  relative: string;
+  profile: ManifestProfile;
+  type: ManifestModuleFileType;
+};
+
+export type IndexedModule = ManifestModuleCore & {
+  files: Record<ManifestModuleFolderType, IndexedFile[]>;
+  workspaceRelative: string;
+};
+
+/**
+ * Manifest index
+ */
+export class ManifestIndex {
+
+  #manifestFile: string;
+  #manifest: ManifestRoot;
+  #modules: IndexedModule[];
+  #modulesByName: Record<string, IndexedModule> = {};
+  #modulesByFolder: Record<string, IndexedModule> = {};
+  #root: string;
+  #outputToEntry = new Map<string, IndexedFile>();
+  #sourceToEntry = new Map<string, IndexedFile>();
+  #importToEntry = new Map<string, IndexedFile>();
+
+  constructor(root: string, manifest: string | ManifestRoot) {
+    this.init(root, manifest);
+  }
+
+  #resolveOutput(...parts: string[]): string {
+    return path.resolve(this.#root, ...parts);
+  }
+
+  get manifest(): ManifestRoot {
+    return this.#manifest;
+  }
+
+  get root(): string {
+    return this.#root;
+  }
+
+  get manifestFile(): string {
+    return this.#manifestFile;
+  }
+
+  init(root: string, manifestInput: string | ManifestRoot): void {
+    this.#root = root;
+    this.#manifestFile = typeof manifestInput === 'string' ? manifestInput : manifestInput.manifestFile;
+    this.#manifest = typeof manifestInput === 'string' ? JSON.parse(fs.readFileSync(this.#manifestFile, 'utf8')) : manifestInput;
+    this.#index();
+  }
+
+  #moduleFiles(m: ManifestModule, files: ManifestModuleFile[]): IndexedFile[] {
+    return files.map(([f, type, ts, profile = 'std']) => {
+      const source = path.join(m.source, f);
+      const js = (type === 'ts' ? f.replace(/[.]ts$/, '.js') : f);
+      const output = this.#resolveOutput(m.output, js);
+      const modImport = `${m.name}/${js}`;
+      let id = modImport.replace(`${m.name}/`, _ => _.replace(/[/]$/, ':'));
+      if (type === 'ts' || type === 'js') {
+        id = id.replace(/[.]js$/, '');
+      }
+
+      return { id, type, source, output, import: modImport, profile, relative: f, module: m.name };
+    });
+  }
+
+  /**
+   * Get index of all source files
+   */
+  #index(): void {
+    this.#outputToEntry.clear();
+    this.#importToEntry.clear();
+    this.#sourceToEntry.clear();
+
+    this.#modules = Object.values(this.manifest.modules)
+      .map(m => ({
+        ...m,
+        output: this.#resolveOutput(m.output),
+        workspaceRelative: m.source.replace(`${this.#manifest.workspacePath}/`, ''),
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        files: Object.fromEntries(
+          Object.entries(m.files).map(([folder, files]) => [folder, this.#moduleFiles(m, files ?? [])])
+        ) as Record<ManifestModuleFolderType, IndexedFile[]>
+      }));
+
+    for (const mod of this.#modules) {
+      for (const files of Object.values(mod.files ?? {})) {
+        for (const entry of files) {
+          this.#outputToEntry.set(entry.output, entry);
+          this.#sourceToEntry.set(entry.source, entry);
+          this.#importToEntry.set(entry.import, entry);
+          this.#importToEntry.set(entry.import.replace(/[.]js$/, ''), entry);
+        }
+      }
+    }
+    this.#modulesByName = Object.fromEntries(this.#modules.map(x => [x.name, x]));
+    this.#modulesByFolder = Object.fromEntries(this.#modules.map(x => [x.workspaceRelative, x]));
+  }
+
+  /**
+   * Get entry by file (input or output)
+   */
+  getEntry(file: string): IndexedFile | undefined {
+    return this.#outputToEntry.get(file) ?? this.#sourceToEntry.get(file);
+  }
+
+  /**
+   * Get all local modules
+   * @returns
+   */
+  getLocalModules(): IndexedModule[] {
+    return this.#modules.filter(x => x.local);
+  }
+
+  /**
+   * Find files from the index
+   * @param folder The sub-folder to check into
+   * @param filter The filter to determine if this is a valid support file
+   */
+  find(config: FindConfig): IndexedFile[] {
+    const { filter: f, folders } = config;
+    const filter = f ? 'test' in f ? f.test.bind(f) : f : f;
+
+    let idx = this.#modules;
+
+    const checkProfile = config.checkProfile ?? true;
+
+    const activeProfiles = new Set(['std', ...(config.profiles ?? process.env.TRV_PROFILES?.split(/\s*,\s*/g) ?? [])]);
+
+    if (checkProfile) {
+      idx = idx.filter(m => m.profiles.length === 0 || m.profiles.some(p => activeProfiles.has(p)));
+    }
+
+    let searchSpace = folders ?
+      idx.flatMap(m => [...folders.flatMap(fo => m.files[fo] ?? []), ...(config.includeIndex ? (m.files.$index ?? []) : [])]) :
+      idx.flatMap(m => [...Object.values(m.files)].flat());
+
+    if (checkProfile) {
+      searchSpace = searchSpace.filter(fi => activeProfiles.has(fi.profile));
+    }
+
+    return searchSpace
+      .filter(({ type }) => type === 'ts')
+      .filter(({ source }) => filter?.(source) ?? true);
+  }
+
+  /**
+   * Find files from the index
+   * @param filter The filter to determine if this is a valid support file
+   */
+  findSupport(config: Omit<FindConfig, 'folder'>): IndexedFile[] {
+    return this.find({ ...config, folders: ['support'] });
+  }
+
+  /**
+   * Find files from the index
+   * @param filter The filter to determine if this is a valid support file
+   */
+  findSrc(config: Omit<FindConfig, 'folder'> = {}): IndexedFile[] {
+    return this.find({ ...config, includeIndex: true, folders: ['src'] });
+  }
+
+  /**
+   * Find files from the index
+   * @param filter The filter to determine if this is a valid support file
+   */
+  findTest(config: Omit<FindConfig, 'folder'>): IndexedFile[] {
+    return this.find({ ...config, folders: ['test'] });
+  }
+
+  /**
+   * Is module installed?
+   */
+  hasModule(name: string): boolean {
+    return name in this.manifest.modules;
+  }
+
+  /**
+   * Get module
+   */
+  getModule(name: string): IndexedModule | undefined {
+    return this.#modulesByName[name];
+  }
+
+  /**
+   * Get module by folder
+   */
+  getModuleByFolder(folder: string): IndexedModule | undefined {
+    return this.#modulesByFolder[folder];
+  }
+
+  /**
+   * Resolve import
+   */
+  resolveFileImport(name: string): string {
+    name = !name.endsWith('.d.ts') ? name.replace(/[.]ts$/, '.js') : name;
+    return this.#importToEntry.get(name)?.output ?? name;
+  }
+
+  /**
+   * Get indexed module from source file
+   * @param source
+   */
+  getFromSource(source: string): IndexedFile | undefined {
+    return this.#sourceToEntry.get(source);
+  }
+
+  /**
+   * Get indexed module from source file
+   * @param source
+   */
+  getFromImport(imp: string): IndexedFile | undefined {
+    return this.#importToEntry.get(imp);
+  }
+
+  /**
+   * Get module from source file
+   * @param source
+   */
+  getModuleFromSource(source: string): IndexedModule | undefined {
+    const name = this.getFromSource(source)?.module;
+    return name ? this.getModule(name) : undefined;
+  }
+
+  /**
+   * Build module list from an expression list (e.g. `@travetto/app,-@travetto/log)
+   */
+  getModuleList(mode: 'local' | 'all', exprList: string = ''): Set<string> {
+    const allMods = Object.keys(this.manifest.modules);
+    const active = new Set<string>(
+      mode === 'local' ? this.getLocalModules().map(x => x.name) :
+        (mode === 'all' ? allMods : [])
+    );
+
+    for (const expr of exprList.split(/\s*,\s*/g)) {
+      const [, neg, mod] = expr.match(/(-|[+])?([^+\- ]+)$/) ?? [];
+      if (mod) {
+        const patt = new RegExp(`^${mod.replace(/[*]/g, '.*')}$`);
+        for (const m of allMods.filter(x => patt.test(x))) {
+          active[neg ? 'delete' : 'add'](m);
+        }
+      }
+    }
+    return active;
+  }
+
+  /**
+   * Get all modules (transitively) that depend on this module
+   */
+  getDependentModules(root: IndexedModule): Set<IndexedModule> {
+    const seen = new Set<string>();
+    const out = new Set<IndexedModule>();
+    const toProcess = [root.name];
+    while (toProcess.length) {
+      const next = toProcess.shift()!;
+      if (seen.has(next)) {
+        continue;
+      }
+      const mod = this.getModule(next)!;
+      toProcess.push(...mod.parents);
+      out.add(mod);
+    }
+    return out;
+  }
+}