@travetto/manifest 3.0.0-rc.10

package/src/manifest-index.ts

@@ -0,0 +1,318 @@
+import { path } from './path';
+
+import {
+  ManifestModule, ManifestModuleCore, ManifestModuleFile,
+  ManifestModuleFileType, ManifestModuleFolderType, ManifestProfile, ManifestRoot
+} from './types';
+
+import { ManifestUtil } from './util';
+
+type ScanTest = ((full: string) => boolean) | { test: (full: string) => boolean };
+export type FindConfig = {
+  folders?: ManifestModuleFolderType[];
+  filter?: ScanTest;
+  includeIndex?: boolean;
+  profiles?: string[];
+  checkProfile?: boolean;
+};
+
+export type IndexedFile = {
+  id: string;
+  import: string;
+  module: string;
+  sourceFile: string;
+  outputFile: string;
+  relativeFile: string;
+  profile: ManifestProfile;
+  type: ManifestModuleFileType;
+};
+
+export type IndexedModule = ManifestModuleCore & {
+  sourcePath: string;
+  outputPath: string;
+  files: Record<ManifestModuleFolderType, IndexedFile[]>;
+};
+
+/**
+ * Manifest index
+ */
+export class ManifestIndex {
+
+  #manifestFile: string;
+  #manifest: ManifestRoot;
+  #modules: IndexedModule[];
+  #modulesByName: Record<string, IndexedModule> = {};
+  #modulesByFolder: Record<string, IndexedModule> = {};
+  #outputRoot: string;
+  #outputToEntry = new Map<string, IndexedFile>();
+  #sourceToEntry = new Map<string, IndexedFile>();
+  #importToEntry = new Map<string, IndexedFile>();
+
+  constructor(manifest: string) {
+    this.init(manifest);
+  }
+
+  #resolveOutput(...parts: string[]): string {
+    return path.resolve(this.#outputRoot, ...parts);
+  }
+
+  get manifest(): ManifestRoot {
+    return this.#manifest;
+  }
+
+  get outputRoot(): string {
+    return this.#outputRoot;
+  }
+
+  get manifestFile(): string {
+    return this.#manifestFile;
+  }
+
+  init(manifestInput: string): void {
+    const { manifest, file } = ManifestUtil.readManifestSync(manifestInput);
+    this.#manifest = manifest;
+    this.#manifestFile = file;
+    this.#outputRoot = path.resolve(this.#manifest.workspacePath, this.#manifest.outputFolder);
+    this.#index();
+  }
+
+  #moduleFiles(m: ManifestModule, files: ManifestModuleFile[]): IndexedFile[] {
+    return files.map(([f, type, ts, profile = 'std']) => {
+      const sourceFile = path.resolve(this.#manifest.workspacePath, m.sourceFolder, f);
+      const js = (type === 'ts' ? f.replace(/[.]ts$/, '.js') : f);
+      const outputFile = this.#resolveOutput(m.outputFolder, js);
+      const modImport = `${m.name}/${js}`;
+      let id = modImport.replace(`${m.name}/`, _ => _.replace(/[/]$/, ':'));
+      if (type === 'ts' || type === 'js') {
+        id = id.replace(/[.]js$/, '');
+      }
+
+      return { id, type, sourceFile, outputFile, import: modImport, profile, relativeFile: f, module: m.name };
+    });
+  }
+
+  /**
+   * Get index of all source files
+   */
+  #index(): void {
+    this.#outputToEntry.clear();
+    this.#importToEntry.clear();
+    this.#sourceToEntry.clear();
+
+    this.#modules = Object.values(this.#manifest.modules)
+      .map(m => ({
+        ...m,
+        outputPath: this.#resolveOutput(m.outputFolder),
+        sourcePath: path.resolve(this.#manifest.workspacePath, m.sourceFolder),
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        files: Object.fromEntries(
+          Object.entries(m.files).map(([folder, files]) => [folder, this.#moduleFiles(m, files ?? [])])
+        ) as Record<ManifestModuleFolderType, IndexedFile[]>
+      }));
+
+    for (const mod of this.#modules) {
+      for (const files of Object.values(mod.files ?? {})) {
+        for (const entry of files) {
+          this.#outputToEntry.set(entry.outputFile, entry);
+          this.#sourceToEntry.set(entry.sourceFile, entry);
+          this.#importToEntry.set(entry.import, entry);
+          this.#importToEntry.set(entry.import.replace(/[.]js$/, ''), entry);
+          this.#importToEntry.set(entry.import.replace(/[.]js$/, '.ts'), entry);
+        }
+      }
+    }
+    this.#modulesByName = Object.fromEntries(this.#modules.map(x => [x.name, x]));
+    this.#modulesByFolder = Object.fromEntries(this.#modules.map(x => [x.sourceFolder, x]));
+  }
+
+  /**
+   * Get entry by file (input or output)
+   */
+  getEntry(file: string): IndexedFile | undefined {
+    return this.#outputToEntry.get(file) ?? this.#sourceToEntry.get(file);
+  }
+
+  /**
+   * Get all local modules
+   * @returns
+   */
+  getLocalModules(): IndexedModule[] {
+    return this.#modules.filter(x => x.local);
+  }
+
+  /**
+   * Find files from the index
+   * @param folder The sub-folder to check into
+   * @param filter The filter to determine if this is a valid support file
+   */
+  find(config: FindConfig): IndexedFile[] {
+    const { filter: f, folders } = config;
+    const filter = f ? 'test' in f ? f.test.bind(f) : f : f;
+
+    let idx = this.#modules;
+
+    const checkProfile = config.checkProfile ?? true;
+
+    const activeProfiles = new Set(['std', ...(config.profiles ?? process.env.TRV_PROFILES?.split(/\s*,\s*/g) ?? [])]);
+
+    if (checkProfile) {
+      idx = idx.filter(m => m.profiles.length === 0 || m.profiles.some(p => activeProfiles.has(p)));
+    }
+
+    let searchSpace = folders ?
+      idx.flatMap(m => [...folders.flatMap(fo => m.files[fo] ?? []), ...(config.includeIndex ? (m.files.$index ?? []) : [])]) :
+      idx.flatMap(m => [...Object.values(m.files)].flat());
+
+    if (checkProfile) {
+      searchSpace = searchSpace.filter(fi => activeProfiles.has(fi.profile));
+    }
+
+    return searchSpace
+      .filter(({ type }) => type === 'ts')
+      .filter(({ sourceFile: source }) => filter?.(source) ?? true);
+  }
+
+  /**
+   * Find files from the index
+   * @param filter The filter to determine if this is a valid support file
+   */
+  findSupport(config: Omit<FindConfig, 'folder'>): IndexedFile[] {
+    return this.find({ ...config, folders: ['support'] });
+  }
+
+  /**
+   * Find files from the index
+   * @param filter The filter to determine if this is a valid support file
+   */
+  findSrc(config: Omit<FindConfig, 'folder'> = {}): IndexedFile[] {
+    return this.find({ ...config, includeIndex: true, folders: ['src'] });
+  }
+
+  /**
+   * Find files from the index
+   * @param filter The filter to determine if this is a valid support file
+   */
+  findTest(config: Omit<FindConfig, 'folder'>): IndexedFile[] {
+    return this.find({ ...config, folders: ['test'] });
+  }
+
+  /**
+   * Is module installed?
+   */
+  hasModule(name: string): boolean {
+    return name in this.#manifest.modules;
+  }
+
+  /**
+   * Get module
+   */
+  getModule(name: string): IndexedModule | undefined {
+    return this.#modulesByName[name];
+  }
+
+  /**
+   * Get module by folder
+   */
+  getModuleByFolder(folder: string): IndexedModule | undefined {
+    return this.#modulesByFolder[folder];
+  }
+
+  /**
+   * Resolve import
+   */
+  resolveFileImport(name: string): string {
+    return this.#importToEntry.get(name)?.outputFile ?? name;
+  }
+
+  /**
+   * Get indexed module from source file
+   * @param source
+   */
+  getFromSource(source: string): IndexedFile | undefined {
+    return this.#sourceToEntry.get(source);
+  }
+
+  /**
+   * Get indexed module from source file
+   * @param source
+   */
+  getFromImport(imp: string): IndexedFile | undefined {
+    return this.#importToEntry.get(imp);
+  }
+
+  /**
+   * Get module from source file
+   * @param source
+   */
+  getModuleFromSource(source: string): IndexedModule | undefined {
+    const name = this.getFromSource(source)?.module;
+    return name ? this.getModule(name) : undefined;
+  }
+
+  /**
+   * Get module from import name
+   * @param importName
+   */
+  getModuleFromImport(importName: string): IndexedModule | undefined {
+    const name = this.getFromImport(importName)?.module;
+    return name ? this.getModule(name) : undefined;
+  }
+  /**
+   * Build module list from an expression list (e.g. `@travetto/app,-@travetto/log)
+   */
+  getModuleList(mode: 'local' | 'all', exprList: string = ''): Set<string> {
+    const allMods = Object.keys(this.#manifest.modules);
+    const active = new Set<string>(
+      mode === 'local' ? this.getLocalModules().map(x => x.name) :
+        (mode === 'all' ? allMods : [])
+    );
+
+    for (const expr of exprList.split(/\s*,\s*/g)) {
+      const [, neg, mod] = expr.match(/(-|[+])?([^+\- ]+)$/) ?? [];
+      if (mod) {
+        const patt = new RegExp(`^${mod.replace(/[*]/g, '.*')}$`);
+        for (const m of allMods.filter(x => patt.test(x))) {
+          active[neg ? 'delete' : 'add'](m);
+        }
+      }
+    }
+    return active;
+  }
+
+  /**
+   * Get all modules (transitively) that depend on this module
+   */
+  getDependentModules(root: IndexedModule): Set<IndexedModule> {
+    const seen = new Set<string>();
+    const out = new Set<IndexedModule>();
+    const toProcess = [root.name];
+    while (toProcess.length) {
+      const next = toProcess.shift()!;
+      if (seen.has(next)) {
+        continue;
+      }
+      const mod = this.getModule(next)!;
+      toProcess.push(...mod.parents);
+      out.add(mod);
+    }
+    return out;
+  }
+
+  /**
+   * Get local folders that represent the user's controlled input
+   */
+  getLocalInputFolders(): string[] {
+    return this.getLocalModules()
+      .flatMap(x =>
+        (!this.manifest.monoRepo || x.sourcePath !== this.manifest.workspacePath) ?
+          [x.sourcePath] : [...Object.keys(x.files)].filter(y => !y.startsWith('$')).map(y => path.resolve(x.sourcePath, y))
+      );
+  }
+
+  /**
+   * Get local output folders
+   */
+  getLocalOutputFolders(): string[] {
+    return this.getLocalModules().map(x => x.outputPath);
+  }
+}

package/src/module.ts

@@ -0,0 +1,213 @@
+import fs from 'fs/promises';
+
+import { path } from './path';
+import {
+  ManifestContext,
+  ManifestModule, ManifestModuleFile, ManifestModuleFileType,
+  ManifestModuleFolderType, ManifestProfile
+} from './types';
+import { ModuleDep, ModuleDependencyVisitor } from './dependencies';
+import { PackageUtil } from './package';
+
+const EXT_MAPPING: Record<string, ManifestModuleFileType> = {
+  '.js': 'js',
+  '.mjs': 'js',
+  '.cjs': 'js',
+  '.json': 'json',
+  '.ts': 'ts',
+  '.md': 'md'
+};
+
+const INDEX_FILES = new Set([
+  'index.ts',
+  'index.js',
+  '__index__.ts',
+  '__index__.js',
+  '__index.ts',
+  '__index.js'
+]);
+
+export class ManifestModuleUtil {
+
+  static #scanCache: Record<string, string[]> = {};
+
+  static #getNewest(stat: { mtimeMs: number, ctimeMs: number }): number {
+    return Math.max(stat.mtimeMs, stat.ctimeMs);
+  }
+
+  /**
+   * Simple file scanning
+   */
+  static async scanFolder(folder: string, mainSource = false): Promise<string[]> {
+    if (!mainSource && folder in this.#scanCache) {
+      return this.#scanCache[folder];
+    }
+
+    if (!await fs.stat(folder).catch(() => false)) {
+      return [];
+    }
+
+    const topFolders = new Set(mainSource ? [] : ['src', 'bin', 'support']);
+    const topFiles = new Set(mainSource ? [] : [...INDEX_FILES, 'package.json']);
+    const out: string[] = [];
+
+    if (!fs.stat(folder).catch(() => false)) {
+      return out;
+    }
+    const stack: [string, number][] = [[folder, 0]];
+    while (stack.length) {
+      const popped = stack.pop();
+      if (!popped) {
+        continue;
+      }
+
+      const [top, depth] = popped;
+
+      // Don't navigate into sub-folders with package.json's
+      if (top !== folder && await fs.stat(`${top}/package.json`).catch(() => false)) {
+        continue;
+      }
+
+      for (const sub of await fs.readdir(top)) {
+        const stat = await fs.stat(`${top}/${sub}`);
+        if (stat.isFile()) {
+          if (!sub.startsWith('.') && (depth > 0 || !topFiles.size || topFiles.has(sub))) {
+            out.push(`${top}/${sub}`);
+          }
+        } else {
+          if (!sub.includes('node_modules') && !sub.startsWith('.') && (depth > 0 || !topFolders.size || topFolders.has(sub))) {
+            stack.push([`${top}/${sub}`, depth + 1]);
+          }
+        }
+      }
+    }
+
+    if (!mainSource) {
+      this.#scanCache[folder] = out;
+    }
+
+    return out;
+  }
+
+  /**
+   * Get file type for a file name
+   */
+  static getFileType(moduleFile: string): ManifestModuleFileType {
+    if (moduleFile === 'package.json') {
+      return 'package-json';
+    } else if (
+      moduleFile.startsWith('support/fixtures/') ||
+      moduleFile.startsWith('test/fixtures/') ||
+      moduleFile.startsWith('support/resources/')
+    ) {
+      return 'fixture';
+    } else if (moduleFile.endsWith('.d.ts')) {
+      return 'typings';
+    } else {
+      const ext = path.extname(moduleFile);
+      return EXT_MAPPING[ext] ?? 'unknown';
+    }
+  }
+
+  /**
+   * Get file type for a file name
+   */
+  static getFileProfile(moduleFile: string): ManifestProfile | undefined {
+    if (moduleFile.startsWith('support/transform')) {
+      return 'compile';
+    } else if (moduleFile.startsWith('support/test/') || moduleFile.startsWith('test/')) {
+      return 'test';
+    } else if (moduleFile.startsWith('doc/') || moduleFile === 'DOC.ts') {
+      return 'doc';
+    } else {
+      return;
+    }
+  }
+
+  /**
+   * Get folder key
+   */
+  static getFolderKey(moduleFile: string): ManifestModuleFolderType {
+    const folderLocation = moduleFile.indexOf('/');
+    if (folderLocation > 0) {
+      if (moduleFile.startsWith('test/fixtures/')) {
+        return 'test/fixtures';
+      } else if (moduleFile.startsWith('support/fixtures/')) {
+        return 'support/fixtures';
+      } else if (moduleFile.startsWith('support/resources/')) {
+        return 'support/resources';
+      } else if (moduleFile.startsWith('support/transform')) {
+        return '$transformer';
+      }
+      const key = moduleFile.substring(0, folderLocation);
+      switch (key) {
+        case 'src':
+        case 'bin':
+        case 'test':
+        case 'doc':
+        case 'resources':
+        case 'support': return key;
+        default: return '$other';
+      }
+    } else if (moduleFile === 'DOC.ts') {
+      return 'doc';
+    } else if (INDEX_FILES.has(moduleFile)) {
+      return '$index';
+    } else if (moduleFile === 'package.json') {
+      return '$package';
+    } else {
+      return '$root';
+    }
+  }
+
+  /**
+   * Convert file (by ext) to a known file type and also retrieve its latest timestamp
+   */
+  static async transformFile(moduleFile: string, full: string): Promise<ManifestModuleFile> {
+    const res: ManifestModuleFile = [moduleFile, this.getFileType(moduleFile), this.#getNewest(await fs.stat(full))];
+    const profile = this.getFileProfile(moduleFile);
+    return profile ? [...res, profile] : res;
+  }
+
+  /**
+   * Visit a module and describe files, and metadata
+   */
+  static async describeModule(ctx: ManifestContext, dep: ModuleDep): Promise<ManifestModule> {
+    const { main, mainSource, local, name, version, sourcePath, profileSet, parentSet, internal } = dep;
+
+    const files: ManifestModule['files'] = {};
+
+    for (const file of await this.scanFolder(sourcePath, mainSource)) {
+      // Group by top folder
+      const moduleFile = file.replace(`${sourcePath}/`, '');
+      const entry = await this.transformFile(moduleFile, file);
+      const key = this.getFolderKey(moduleFile);
+      (files[key] ??= []).push(entry);
+    }
+
+    // Refine non-main source
+    if (!mainSource) {
+      files.$root = files.$root?.filter(([file, type]) => type !== 'ts');
+    }
+
+    const profiles = [...profileSet].sort();
+    const parents = [...parentSet].sort();
+    const outputFolder = `node_modules/${name}`;
+    const sourceFolder = sourcePath === ctx.workspacePath ? '' : sourcePath.replace(`${ctx.workspacePath}/`, '');
+
+    const res = { main, name, version, local, internal, sourceFolder, outputFolder, files, profiles, parents, };
+    return res;
+  }
+
+  /**
+   * Produce all modules for a given manifest folder, adding in some given modules when developing framework
+   */
+  static async produceModules(ctx: ManifestContext): Promise<Record<string, ManifestModule>> {
+    const visitor = new ModuleDependencyVisitor(ctx);
+    const mainPath = path.resolve(ctx.workspacePath, ctx.mainFolder);
+    const declared = await PackageUtil.visitPackages(mainPath, visitor);
+    const sorted = [...declared].sort((a, b) => a.name.localeCompare(b.name));
+    const modules = await Promise.all(sorted.map(x => this.describeModule(ctx, x)));
+    return Object.fromEntries(modules.map(m => [m.name, m]));
+  }
+}