@cyberismo/data-handler 0.0.6 → 0.0.8

package/src/module-manager.ts

@@ -11,16 +11,14 @@
   License along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
 
-import fs from 'node:fs';
 import { join } from 'node:path';
 import { mkdir, readdir, rm } from 'node:fs/promises';
 
-import git from 'isomorphic-git';
-import http from 'isomorphic-git/http/node/index.js';
+import { simpleGit, type SimpleGit } from 'simple-git';
 
 import { copyDir, deleteDir, pathExists } from './utils/file-utils.js';
-import type { Import } from './commands/index.js';
 import type {
+  Credentials,
   ModuleSetting,
   ModuleSettingOptions,
 } from './interfaces/project-interfaces.js';
@@ -31,8 +29,15 @@ import { readJsonFile } from './utils/json.js';
 import { Validate } from './commands/index.js';
 
 const FILE_PROTOCOL = 'file:';
-// todo: add support for git's default branch.
-const MAIN_BRANCH = 'main';
+const HTTPS_PROTOCOL = 'https:';
+
+// timeout in milliseconds for git client (no stdout / stderr activity)
+const DEFAULT_TIMEOUT = 10000;
+
+// When dependencies are built to a map, use map that has
+//   key: module name,
+//   value: list of unique prefixes
+type DependencyGraph = Map<string, Set<string>>;
 
 /**
  * Class that handles module updates and imports.
@@ -40,10 +45,9 @@ const MAIN_BRANCH = 'main';
 export class ModuleManager {
   private modules: ModuleSetting[] = [];
   private tempModulesDir: string = '';
-  constructor(
-    private project: Project,
-    private importCmd: Import,
-  ) {
+  private defaultBranchCache: Map<string, string> = new Map();
+
+  constructor(private project: Project) {
     this.tempModulesDir = join(this.project.paths.tempFolder, 'modules');
   }
 
@@ -56,87 +60,121 @@ export class ModuleManager {
     await this.project.collectModuleResources();
   }
 
-  // Handles a branch of a repository.
-  private async branch(module: ModuleSetting) {
-    if (module.branch === MAIN_BRANCH || module.branch === '' || !module.branch)
-      return;
+  // Creates a map of what dependencies each module depend from.
+  private async buildDependencyGraph(
+    dependencies: ModuleSetting[],
+  ): Promise<DependencyGraph> {
+    const dependencyNames = dependencies.map((item) => item.name);
+    const dependencyGraph = new Map<string, Set<string>>() as DependencyGraph;
+    for (const dependency of dependencyNames) {
+      dependencyGraph.set(
+        dependency,
+        await this.transientDependencies(dependency),
+      );
+    }
+    return dependencyGraph;
+  }
 
-    await git.checkout({
-      fs,
-      dir: join(this.tempModulesDir, module.name),
-      ref: module.branch,
+  // Returns 'true' if 'moduleName' can be removed.
+  private canBeRemoved(
+    dependencies: DependencyGraph,
+    moduleName: string,
+  ): boolean {
+    let unused = true;
+    dependencies.forEach((transientDependencies, key) => {
+      if (key !== moduleName && transientDependencies.has(moduleName)) {
+        unused = false;
+      }
     });
-    console.error(
-      `... Switched to '${module.branch}' branch for module '${module.name}'`,
-    );
+    return unused;
   }
 
   // Handles cloning of a repository.
   private async clone(
     module: ModuleSetting,
     verbose: boolean = true,
+    credentials?: Credentials,
   ): Promise<string> {
     if (!module.name || module.name === '') {
       module.name = this.repositoryName(module.location);
     }
 
-    let repoUrl: URL;
-    try {
-      repoUrl = new URL(module.location);
-    } catch {
-      throw new Error(`Invalid repository URL: ${module.location}`);
-    }
-
-    if (
-      process.env.CYBERISMO_GIT_USER &&
-      process.env.CYBERISMO_GIT_TOKEN &&
-      module.private
-    ) {
+    const destinationPath = join(this.tempModulesDir, module.name);
+
+    let remote = module.location;
+    if (module.private) {
+      if (
+        credentials &&
+        credentials?.username &&
+        credentials?.token &&
+        module.location.startsWith(HTTPS_PROTOCOL)
+      ) {
+        if (verbose) {
+          console.log(
+            `... Using HTTPS with credentials '${credentials?.username}' for cloning '${module.name}'`,
+          );
+        }
+        try {
+          const repoUrl = new URL(module.location);
+          const user = credentials?.username;
+          const pass = credentials?.token;
+          const host = repoUrl.host;
+          const path = repoUrl.pathname;
+          remote = `https://${user}:${pass}@${host}${path}`;
+        } catch {
+          throw new Error(`Invalid repository URL: ${module.location}`);
+        }
+      } else if (module.location.startsWith('git@')) {
+        if (verbose) {
+          console.log(`... Using SSH for cloning '${module.name}'`);
+        }
+      }
+    } else {
       if (verbose) {
         console.log(
-          `... Using credentials '${process.env.CYBERISMO_GIT_USER}' for cloning '${module.name}'`,
+          `... Using HTTPS without credentials for cloning '${module.name}'`,
         );
       }
-      repoUrl.username = process.env.CYBERISMO_GIT_USER;
-      repoUrl.password = process.env.CYBERISMO_GIT_TOKEN;
     }
-    await git.clone({
-      fs,
-      http,
-      dir: join(this.tempModulesDir, module.name),
-      url: repoUrl.toString(),
-      depth: 1,
-      onAuth: () => {
-        // Turn credentials 'off' when they are not available
-        if (
-          !process.env.CYBERISMO_GIT_USER ||
-          !process.env.CYBERISMO_GIT_TOKEN
-        ) {
-          return undefined;
-        }
-        // Turn credentials 'off' for public repos
-        if (!module.private) {
-          return undefined;
-        }
-        return {
-          username: process.env.CYBERISMO_GIT_USER,
-          password: process.env.CYBERISMO_GIT_TOKEN,
-        };
-      },
-    });
 
-    if (verbose) {
-      console.log(`... Cloned '${module.name}' to a temporary folder`);
+    try {
+      await mkdir(this.tempModulesDir, { recursive: true });
+      const cloneOptions = await this.setCloneOptions(module);
+      await rm(destinationPath, { recursive: true, force: true });
+
+      const git: SimpleGit = simpleGit({
+        timeout: {
+          block: DEFAULT_TIMEOUT,
+        },
+      });
+
+      await git
+        // turn off git prompts
+        .env({ GIT_TERMINAL_PROMPT: 0, GCM_INTERACTIVE: 'never' })
+        .clone(remote, destinationPath, cloneOptions);
+
+      if (verbose) {
+        console.log(`... Cloned '${module.name}' to a temporary folder`);
+      }
+    } catch (error) {
+      if (error instanceof Error)
+        throw new Error(
+          `Failed to clone module '${module.name}': ${error.message}`,
+        );
     }
+
     return module.name;
   }
 
   // Collects all module prefixes from module hierarchy into 'this.modules'.
   // Note that collected result can contain duplicates.
-  private async collectModulePrefixes(modules: ModuleSetting[]) {
+  private async collectModulePrefixes(
+    modules: ModuleSetting[],
+    credentials?: Credentials,
+  ) {
     if (modules) {
       for (const module of modules) {
-        await this.doCollectModulePrefix(module);
+        await this.doCollectModulePrefix(module, credentials);
       }
     }
   }
@@ -151,62 +189,97 @@ export class ModuleManager {
     }
   }
 
+  // Gets the default branch for a repository from remote or cache
+  private async defaultBranch(module: ModuleSetting): Promise<string> {
+    if (this.defaultBranchCache.has(module.location)) {
+      return this.defaultBranchCache.get(module.location)!;
+    }
+    // Set the default branch if branch was not specified
+    if (!module.branch) {
+      const destinationPath = join(this.tempModulesDir, module.name);
+      // Only return path after cloning
+      if (pathExists(destinationPath)) {
+        const git: SimpleGit = simpleGit({
+          timeout: {
+            block: DEFAULT_TIMEOUT,
+          },
+        });
+        const options = ['--abbrev-ref', 'HEAD'];
+        const defaultBranch = await git.cwd(destinationPath).revparse(options);
+        this.defaultBranchCache.set(module.location, defaultBranch);
+        return defaultBranch;
+      }
+    }
+    // The actual default branch will be updated later (after cloning).
+    return '';
+  }
+
+  // Fetches direct dependencies of a module.
+  private async dependencies(moduleName: string): Promise<Set<string>> {
+    const allModules = await this.project.modules();
+    if (!allModules) return new Set();
+    const module = allModules.find((m) => m.name === moduleName);
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
+    }
+    const modulePath = join(module.path, module.name, 'cardsConfig.json');
+    const moduleConfiguration = (await readJsonFile(
+      modulePath,
+    )) as ProjectConfiguration;
+    return moduleConfiguration.modules
+      ? new Set(moduleConfiguration.modules.map((m) => m.name))
+      : new Set();
+  }
+
   // Collects one module's dependency prefixes to 'this.modules'.
   // Note that there can be duplicate entries.
-  private async doCollectModulePrefix(module: ModuleSetting) {
+  private async doCollectModulePrefix(
+    module: ModuleSetting,
+    credentials?: Credentials,
+  ) {
     let moduleRoot = '';
     if (this.isFileModule(module)) {
       const urlStart = FILE_PROTOCOL.length;
       // Remove 'file:' from location
       moduleRoot = module.location.substring(urlStart, module.location.length);
     } else {
-      await this.clone(module, false);
+      await this.clone(module, false, credentials);
       moduleRoot = join(this.tempModulesDir, module.name);
     }
 
     this.modules.push(module);
 
     const configuration = await this.configuration(moduleRoot);
-    await this.collectModulePrefixes(configuration.modules);
+    await this.collectModulePrefixes(configuration.modules, credentials);
   }
 
   // Updates one module that is read from local file system.
   private async handleFileModule(module: ModuleSetting) {
-    this.removeProtocolFromLocation(module);
+    this.stripProtocolFromLocation(module);
     await this.remove(module);
-    await this.importFromFolder(module);
+    await this.importFromFolder(module.location, module.name);
   }
 
   // Updates one module that is received from Git.
   private async handleGitModule(module: ModuleSetting) {
     await this.clone(module);
-    await this.branch(module);
+    const tempLocation = join(this.tempModulesDir, module.name);
     await this.remove(module);
-    await this.importFromTemp(module);
+    await this.importFromFolder(tempLocation, module.name);
   }
 
   // Updates one module.
   private async handleModule(module: ModuleSetting) {
-    return this.isFileModule(module)
-      ? this.handleFileModule(module)
-      : this.handleGitModule(module);
+    return this.isGitModule(module)
+      ? this.handleGitModule(module)
+      : this.handleFileModule(module);
   }
 
-  // Handles importing a module from module settings 'location'
-  private async importFromFolder(module: ModuleSetting) {
-    await this.importCmd.updateExistingModule(module.location);
+  // Imports from a given folder. Is used both for .temp/<module name> and file locations.
+  private async importFromFolder(path: string, name: string) {
+    await this.importFileModule(path);
     console.log(
-      `... Imported module '${module.name}' to '${this.project.configuration.name}'`,
-    );
-  }
-
-  // Handles importing a module from '.temp' folder
-  private async importFromTemp(module: ModuleSetting) {
-    await this.importCmd.updateExistingModule(
-      join(this.tempModulesDir, module.name),
-    );
-    console.log(
-      `... Imported module '${module.name}' to '${this.project.configuration.name}'`,
+      `... Imported module '${name}' to '${this.project.configuration.name}'`,
     );
   }
 
@@ -216,20 +289,61 @@ export class ModuleManager {
     return module.location.startsWith('file:');
   }
 
-  // Prepares '.temp/modules' for cloning
-  private async prepare() {
-    await mkdir(this.tempModulesDir, { recursive: true });
-    for (const file of await readdir(this.tempModulesDir)) {
-      await rm(join(this.tempModulesDir, file), {
-        force: true,
-        recursive: true,
-      });
+  // Returns true if module is imported from git.
+  private isGitModule(module: ModuleSetting): boolean {
+    if (!module.location) return false;
+    return (
+      module.location.startsWith('https:') || module.location.startsWith('git@')
+    );
+  }
+
+  // Collect modules that could be removed from .cards/modules when
+  // 'moduleName' is removed.
+  private async orphanedModules(
+    dependencies: DependencyGraph,
+    moduleName: string,
+  ): Promise<string[]> {
+    const projectModules = this.project.configuration.modules;
+    const removableTransientModules: string[] = [];
+    if (dependencies.has(moduleName)) {
+      const deps = dependencies.get(moduleName);
+      for (const dependency of deps!) {
+        const projectDependency = projectModules.some(
+          (item) => item.name === dependency,
+        );
+        if (projectDependency) continue;
+
+        let orphanModule = true;
+        dependencies.forEach((transientDependencies, key) => {
+          if (key === moduleName) return;
+          if (transientDependencies.has(dependency)) {
+            orphanModule = false;
+          }
+        });
+
+        if (orphanModule) {
+          removableTransientModules.push(dependency);
+        }
+      }
     }
+    return removableTransientModules;
   }
 
-  // Returns whether to use git or file system for handling the module.
-  private protocol(module: ModuleSetting) {
-    return this.isFileModule(module) ? 'file' : 'git';
+  // Prepares '.temp/modules' for cloning
+  private async prepare() {
+    try {
+      await mkdir(this.tempModulesDir, { recursive: true });
+      const files = await readdir(this.tempModulesDir);
+      for (const file of files) {
+        const filePath = join(this.tempModulesDir, file);
+        await rm(filePath, {
+          force: true,
+          recursive: true,
+        });
+      }
+    } catch (error) {
+      throw new Error(`Failed to prepare temporary directory: ${error}`);
+    }
   }
 
   // Handles removing an imported module.
@@ -245,47 +359,32 @@ export class ModuleManager {
     }
   }
 
-  // Remove module files.
-  private async removeModuleFiles(moduleName: string) {
-    const module = await this.project.module(moduleName);
-    if (!module) {
-      throw new Error(`Module '${moduleName}' not found`);
-    }
-    await deleteDir(module.path);
-  }
-
-  // Updates module's 'location' not to have 'protocol:' in the beginning (only for "file:" needed).
-  private removeProtocolFromLocation(module: ModuleSetting) {
-    const protocol = this.protocol(module);
-    module.location = module.location.substring(
-      protocol.length + 1,
-      module.location.length,
-    );
-  }
-
   // Checks for duplicate ModuleSetting entries and throws an error if modules
   // with the same name have different branches or locations.
-  // Treats undefined branch, empty string branch, and "main" branch as equivalent.
+  // Treats undefined branch, empty string branch, and default branch as equivalent.
   // Returns an array with duplicate entries removed
-  private removeDuplicates(modules: ModuleSetting[]): ModuleSetting[] {
+  private async removeDuplicates(
+    modules: ModuleSetting[],
+  ): Promise<ModuleSetting[]> {
     const moduleMap = new Map<string, ModuleSetting>();
 
-    // Assume that empty, or missing branch means 'main'
-    const normalizeBranch = (branch: string | undefined): string => {
-      if (!branch || branch === '' || branch === MAIN_BRANCH) {
-        return MAIN_BRANCH;
-      }
-      return branch;
+    // Normalize branch names by checking against the default branch for each module
+
+    const normalizeBranch = async (module: ModuleSetting) => {
+      return module.branch ? module.branch! : await this.defaultBranch(module);
     };
 
     for (const module of modules) {
       const existingModule = moduleMap.get(module.name);
       if (existingModule) {
-        if (existingModule.private !== module.private) {
+        if (
+          (existingModule.private && !module.private) ||
+          (!existingModule.private && module.private)
+        ) {
           throw new Error(
             `Module conflict: '${module.name}' has different access:\n` +
-              `  - ${existingModule.private || 'undefined'}\n` +
-              `  - ${module.private || 'undefined'}`,
+              `  - ${Boolean(existingModule.private)}\n` +
+              `  - ${Boolean(module.private)}`,
           );
         }
         if (existingModule.location !== module.location) {
@@ -295,8 +394,8 @@ export class ModuleManager {
               `  - ${module.location}`,
           );
         }
-        const existingBranch = normalizeBranch(existingModule.branch);
-        const newBranch = normalizeBranch(module.branch);
+        const existingBranch = await normalizeBranch(existingModule);
+        const newBranch = await normalizeBranch(module);
 
         if (existingBranch !== newBranch) {
           throw new Error(
@@ -312,6 +411,15 @@ export class ModuleManager {
     return Array.from(moduleMap.values());
   }
 
+  // Remove module files.
+  private async removeModuleFiles(moduleName: string) {
+    const module = await this.project.module(moduleName);
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
+    }
+    await deleteDir(module.path);
+  }
+
   // Gets repository name from gitUrl
   private repositoryName(gitUrl: string): string {
     const last = gitUrl.lastIndexOf('/');
@@ -319,6 +427,94 @@ export class ModuleManager {
     return repoName;
   }
 
+  // Sets cloning options with support for default branch.
+  private async setCloneOptions(module: ModuleSetting): Promise<string[]> {
+    const cloneOptions = ['--depth', '1'];
+    const defaultBranch = await this.defaultBranch(module);
+    // Only specify branch if it's different from the default branch
+    if (
+      module.branch &&
+      module.branch !== '' &&
+      module.branch !== defaultBranch
+    ) {
+      cloneOptions.push('--branch', module.branch);
+    }
+    return cloneOptions;
+  }
+
+  // Updates module's 'location' not to have 'protocol:' in the beginning (only for "file:" needed).
+  private stripProtocolFromLocation(module: ModuleSetting) {
+    const protocol = this.isFileModule(module) ? 'file' : 'git';
+    module.location = module.location.substring(
+      protocol.length + 1,
+      module.location.length,
+    );
+  }
+
+  // Fetches all dependencies for a module.
+  private async transientDependencies(
+    moduleName: string,
+  ): Promise<Set<string>> {
+    const dependencies = await this.dependencies(moduleName);
+    let transientDependencies: Set<string> = new Set(dependencies);
+    for (const dependency of dependencies) {
+      const depTransients = await this.transientDependencies(dependency);
+      transientDependencies = new Set([
+        ...transientDependencies,
+        ...depTransients,
+      ]);
+    }
+
+    return transientDependencies;
+  }
+
+  // Updates modules in the project.
+  private async update(module?: ModuleSetting, credentials?: Credentials) {
+    // Prints dots every half second so that user knows that something is ongoing
+    function start() {
+      console.log('... Collecting unique modules. This takes a moment.');
+      return setInterval(() => process.stdout.write(`.`), 500);
+    }
+
+    // Stops the above, and shows results
+    function finished(interval: NodeJS.Timeout, modules: string[]) {
+      clearInterval(interval);
+      if (modules.length > 0) {
+        console.log(`\n... Found modules: ${modules.join(', ')}`);
+      }
+    }
+
+    await this.prepare();
+
+    let modules = module ? [module] : this.project.configuration.modules;
+    if (modules.length === 0) {
+      throw new Error(`No modules in the project!`);
+    }
+    modules = modules.filter((module) => this.isGitModule(module));
+
+    const dotInterval = start();
+
+    let uniqueModules: ModuleSetting[] = [];
+    try {
+      await this.collectModulePrefixes(modules, credentials);
+      uniqueModules = await this.removeDuplicates(this.modules);
+    } finally {
+      finished(
+        dotInterval,
+        uniqueModules.map((item) => item.name),
+      );
+
+      // Update modules parallel.
+      const promises: Promise<void>[] = [];
+      uniqueModules.forEach((module) =>
+        promises.push(this.handleModule(module)),
+      );
+      await Promise.all(promises);
+      await deleteDir(this.tempModulesDir);
+      await this.project.collectModuleResources();
+    }
+  }
+
   // Checks that module prefix is not in use in the project
   private async validatePrefix(modulePrefix: string) {
     // Do not allow modules with same prefixes.
@@ -370,16 +566,24 @@ export class ModuleManager {
    * Imports module from gitUrl.
    * @param source Git URL to import from.
    * @param options Modules setting options.
+   * @param credentials Credentials for private repositories.
    * @returns module prefix as defined in its CardsConfig.json
    */
-  public async importGitModule(source: string, options?: ModuleSettingOptions) {
-    const repoName = await this.clone({
-      name: '',
-      location: source,
-      ...options,
-    });
-    await this.branch({ name: repoName, location: source, ...options });
-    const clonePath = join(this.project.paths.tempFolder, 'modules', repoName);
+  public async importGitModule(
+    source: string,
+    options?: ModuleSettingOptions,
+    credentials?: Credentials,
+  ) {
+    const clonedName = await this.clone(
+      {
+        name: this.repositoryName(source),
+        location: source,
+        ...options,
+      },
+      undefined,
+      credentials,
+    );
+    const clonePath = join(this.tempModulesDir, clonedName);
     const modulePrefix = (await this.configuration(clonePath)).cardKeyPrefix;
     await this.validatePrefix(modulePrefix);
 
@@ -393,51 +597,53 @@ export class ModuleManager {
   }
 
   /**
-   * Updates all imported modules.
+   * Removed module from project.
+   * If module is not used by any other modules, then will remove the module from disk as well.
+   * Otherwise, only updates project configuration.
+   * @param moduleName Name of the module to remove
    */
-  public async update() {
-    // Prints dots every half second so that user knows that something is ongoing
-    function start() {
-      console.log('... Collecting unique modules. This takes a moment.');
-      return setInterval(() => process.stdout.write(`.`), 500);
-    }
+  public async removeModule(moduleName: string) {
+    const projectModules = this.project.configuration.modules;
+    const dependencies = await this.buildDependencyGraph(projectModules);
+    const module = await this.project.module(moduleName);
 
-    // Stops the above, and shows results
-    function finished(interval: NodeJS.Timeout, modules: string[]) {
-      clearInterval(interval);
-      console.log(`\n... Found modules: ${modules.join(', ')}`);
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
     }
 
-    await this.prepare();
-
-    const modules = this.project.configuration.modules;
-    if (modules.length === 0) {
-      throw new Error(`No modules in the project!`);
+    // Project module can always be removed from project configuration,
+    // but modules under .cards/modules must be checked not to be used by
+    // other modules.
+    if (this.canBeRemoved(dependencies, moduleName)) {
+      const orphans = await this.orphanedModules(dependencies, moduleName);
+      await deleteDir(module.path);
+      for (const moduleToDelete of orphans) {
+        const modulePath = join(
+          this.project.paths.modulesFolder,
+          moduleToDelete,
+        );
+        await deleteDir(modulePath);
+      }
+      await this.project.collectModuleResources();
     }
+    await this.project.configuration.removeModule(moduleName);
+  }
 
-    const dotInterval = start();
-
-    // Collect prefixes from project's dependency modules.
-    await this.collectModulePrefixes(modules);
-
-    let uniqueModules: ModuleSetting[] = [];
-    try {
-      uniqueModules = this.removeDuplicates(this.modules);
-    } finally {
-      finished(
-        dotInterval,
-        uniqueModules.map((item) => item.name),
-      );
-
-      // Update modules parallel.
-      const promises: Promise<void>[] = [];
-      uniqueModules.forEach((module) =>
-        promises.push(this.handleModule(module)),
-      );
-      await Promise.all(promises);
+  /**
+   * Imports module from a local file path or a git URL.
+   * @param module Module to update. If not provided, updates all modules.
+   * @param credentials Optional credentials for private repositories.
+   * @returns Module prefix as defined in its CardsConfig.json
+   */
+  public async updateModule(module: ModuleSetting, credentials?: Credentials) {
+    return this.update(module, credentials);
+  }
 
-      await deleteDir(this.tempModulesDir);
-      await this.project.collectModuleResources();
-    }
+  /**
+   * Updates all imported modules.
+   * @param credentials Optional credentials for private modules.
+   */
+  public async updateModules(credentials?: Credentials) {
+    return this.update(undefined, credentials);
   }
 }