@cyberismo/data-handler 0.0.7 → 0.0.8

package/src/macros/report/index.ts

@@ -23,6 +23,7 @@ import type TaskQueue from '../task-queue.js';
 import { ReportResource } from '../../resources/report-resource.js';
 import { resourceName } from '../../utils/resource-utils.js';
 import { generateReportContent } from '../../utils/report.js';
+import { ClingoError } from '@cyberismo/node-clingo';
 
 export interface ReportOptions extends MacroOptions {
   name: string;
@@ -58,16 +59,24 @@ class ReportMacro extends BaseMacro {
         schema: report.schema,
       });
     }
-
-    return generateReportContent({
-      calculate: this.calculate,
-      contentTemplate: report.contentTemplate,
-      queryTemplate: report.queryTemplate,
-      options: {
-        cardKey: context.cardKey,
-        ...options,
-      },
-    });
+    try {
+      return await generateReportContent({
+        calculate: this.calculate,
+        contentTemplate: report.contentTemplate,
+        queryTemplate: report.queryTemplate,
+        options: {
+          cardKey: context.cardKey,
+          ...options,
+        },
+      });
+    } catch (error) {
+      if (error instanceof ClingoError) {
+        throw new Error(
+          `Error running logic program in report '${options.name}':${error.details.errors.join('\n')}`,
+        );
+      }
+      throw error;
+    }
   };
 
   private validate(data: unknown): ReportOptions {

package/src/module-manager.ts

@@ -29,17 +29,24 @@ import { readJsonFile } from './utils/json.js';
 import { Validate } from './commands/index.js';
 
 const FILE_PROTOCOL = 'file:';
-// todo: add support for git's default branch.
-const MAIN_BRANCH = 'main';
+const HTTPS_PROTOCOL = 'https:';
+
 // timeout in milliseconds for git client (no stdout / stderr activity)
 const DEFAULT_TIMEOUT = 10000;
 
+// When dependencies are built to a map, use map that has
+//   key: module name,
+//   value: list of unique prefixes
+type DependencyGraph = Map<string, Set<string>>;
+
 /**
  * Class that handles module updates and imports.
  */
 export class ModuleManager {
   private modules: ModuleSetting[] = [];
   private tempModulesDir: string = '';
+  private defaultBranchCache: Map<string, string> = new Map();
+
   constructor(private project: Project) {
     this.tempModulesDir = join(this.project.paths.tempFolder, 'modules');
   }
@@ -53,6 +60,35 @@ export class ModuleManager {
     await this.project.collectModuleResources();
   }
 
+  // Creates a map of what dependencies each module depend from.
+  private async buildDependencyGraph(
+    dependencies: ModuleSetting[],
+  ): Promise<DependencyGraph> {
+    const dependencyNames = dependencies.map((item) => item.name);
+    const dependencyGraph = new Map<string, Set<string>>() as DependencyGraph;
+    for (const dependency of dependencyNames) {
+      dependencyGraph.set(
+        dependency,
+        await this.transientDependencies(dependency),
+      );
+    }
+    return dependencyGraph;
+  }
+
+  // Returns 'true' if 'moduleName' can be removed.
+  private canBeRemoved(
+    dependencies: DependencyGraph,
+    moduleName: string,
+  ): boolean {
+    let unused = true;
+    dependencies.forEach((transientDependencies, key) => {
+      if (key !== moduleName && transientDependencies.has(moduleName)) {
+        unused = false;
+      }
+    });
+    return unused;
+  }
+
   // Handles cloning of a repository.
   private async clone(
     module: ModuleSetting,
@@ -67,10 +103,15 @@ export class ModuleManager {
 
     let remote = module.location;
     if (module.private) {
-      if (credentials && credentials?.username && credentials?.token) {
+      if (
+        credentials &&
+        credentials?.username &&
+        credentials?.token &&
+        module.location.startsWith(HTTPS_PROTOCOL)
+      ) {
         if (verbose) {
           console.log(
-            `... Using credentials '${credentials?.username}' for cloning '${module.name}'`,
+            `... Using HTTPS with credentials '${credentials?.username}' for cloning '${module.name}'`,
           );
         }
         try {
@@ -83,16 +124,22 @@ export class ModuleManager {
         } catch {
           throw new Error(`Invalid repository URL: ${module.location}`);
         }
-      } else {
+      } else if (module.location.startsWith('git@')) {
         if (verbose) {
-          console.log(`... Not using credentials for cloning '${module.name}'`);
+          console.log(`... Using SSH for cloning '${module.name}'`);
         }
       }
+    } else {
+      if (verbose) {
+        console.log(
+          `... Using HTTPS without credentials for cloning '${module.name}'`,
+        );
+      }
     }
 
     try {
       await mkdir(this.tempModulesDir, { recursive: true });
-      const cloneOptions = this.setCloneOptions(module);
+      const cloneOptions = await this.setCloneOptions(module);
       await rm(destinationPath, { recursive: true, force: true });
 
       const git: SimpleGit = simpleGit({
@@ -110,7 +157,6 @@ export class ModuleManager {
         console.log(`... Cloned '${module.name}' to a temporary folder`);
       }
     } catch (error) {
-      console.error(error);
       if (error instanceof Error)
         throw new Error(
           `Failed to clone module '${module.name}': ${error.message}`,
@@ -143,6 +189,48 @@ export class ModuleManager {
     }
   }
 
+  // Gets the default branch for a repository from remote or cache
+  private async defaultBranch(module: ModuleSetting): Promise<string> {
+    if (this.defaultBranchCache.has(module.location)) {
+      return this.defaultBranchCache.get(module.location)!;
+    }
+    // Set the default branch if branch was not specified
+    if (!module.branch) {
+      const destinationPath = join(this.tempModulesDir, module.name);
+      // Only return path after cloning
+      if (pathExists(destinationPath)) {
+        const git: SimpleGit = simpleGit({
+          timeout: {
+            block: DEFAULT_TIMEOUT,
+          },
+        });
+        const options = ['--abbrev-ref', 'HEAD'];
+        const defaultBranch = await git.cwd(destinationPath).revparse(options);
+        this.defaultBranchCache.set(module.location, defaultBranch);
+        return defaultBranch;
+      }
+    }
+    // The actual default branch will be updated later (after cloning).
+    return '';
+  }
+
+  // Fetches direct dependencies of a module.
+  private async dependencies(moduleName: string): Promise<Set<string>> {
+    const allModules = await this.project.modules();
+    if (!allModules) return new Set();
+    const module = allModules.find((m) => m.name === moduleName);
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
+    }
+    const modulePath = join(module.path, module.name, 'cardsConfig.json');
+    const moduleConfiguration = (await readJsonFile(
+      modulePath,
+    )) as ProjectConfiguration;
+    return moduleConfiguration.modules
+      ? new Set(moduleConfiguration.modules.map((m) => m.name))
+      : new Set();
+  }
+
   // Collects one module's dependency prefixes to 'this.modules'.
   // Note that there can be duplicate entries.
   private async doCollectModulePrefix(
@@ -167,16 +255,17 @@ export class ModuleManager {
 
   // Updates one module that is read from local file system.
   private async handleFileModule(module: ModuleSetting) {
-    this.removeProtocolFromLocation(module);
+    this.stripProtocolFromLocation(module);
     await this.remove(module);
-    await this.importFromFolder(module);
+    await this.importFromFolder(module.location, module.name);
   }
 
   // Updates one module that is received from Git.
   private async handleGitModule(module: ModuleSetting) {
     await this.clone(module);
+    const tempLocation = join(this.tempModulesDir, module.name);
     await this.remove(module);
-    await this.importFromTemp(module);
+    await this.importFromFolder(tempLocation, module.name);
   }
 
   // Updates one module.
@@ -186,19 +275,11 @@ export class ModuleManager {
       : this.handleFileModule(module);
   }
 
-  // Handles importing a module from module settings 'location'
-  private async importFromFolder(module: ModuleSetting) {
-    await this.importFileModule(module.location);
-    console.log(
-      `... Imported module '${module.name}' to '${this.project.configuration.name}'`,
-    );
-  }
-
-  // Handles importing a module from '.temp' folder
-  private async importFromTemp(module: ModuleSetting) {
-    await this.importFileModule(join(this.tempModulesDir, module.name));
+  // Imports from a given folder. Is used both for .temp/<module name> and file locations.
+  private async importFromFolder(path: string, name: string) {
+    await this.importFileModule(path);
     console.log(
-      `... Imported module '${module.name}' to '${this.project.configuration.name}'`,
+      `... Imported module '${name}' to '${this.project.configuration.name}'`,
     );
   }
 
@@ -211,7 +292,41 @@ export class ModuleManager {
   // Returns true if module is imported from git.
   private isGitModule(module: ModuleSetting): boolean {
     if (!module.location) return false;
-    return module.location.startsWith('https:');
+    return (
+      module.location.startsWith('https:') || module.location.startsWith('git@')
+    );
+  }
+
+  // Collect modules that could be removed from .cards/modules when
+  // 'moduleName' is removed.
+  private async orphanedModules(
+    dependencies: DependencyGraph,
+    moduleName: string,
+  ): Promise<string[]> {
+    const projectModules = this.project.configuration.modules;
+    const removableTransientModules: string[] = [];
+    if (dependencies.has(moduleName)) {
+      const deps = dependencies.get(moduleName);
+      for (const dependency of deps!) {
+        const projectDependency = projectModules.some(
+          (item) => item.name === dependency,
+        );
+        if (projectDependency) continue;
+
+        let orphanModule = true;
+        dependencies.forEach((transientDependencies, key) => {
+          if (key === moduleName) return;
+          if (transientDependencies.has(dependency)) {
+            orphanModule = false;
+          }
+        });
+
+        if (orphanModule) {
+          removableTransientModules.push(dependency);
+        }
+      }
+    }
+    return removableTransientModules;
   }
 
   // Prepares '.temp/modules' for cloning
@@ -231,11 +346,6 @@ export class ModuleManager {
     }
   }
 
-  // Returns whether to use git or file system for handling the module.
-  private protocol(module: ModuleSetting) {
-    return this.isFileModule(module) ? 'file' : 'git';
-  }
-
   // Handles removing an imported module.
   private async remove(module: ModuleSetting) {
     try {
@@ -249,37 +359,19 @@ export class ModuleManager {
     }
   }
 
-  // Remove module files.
-  private async removeModuleFiles(moduleName: string) {
-    const module = await this.project.module(moduleName);
-    if (!module) {
-      throw new Error(`Module '${moduleName}' not found`);
-    }
-    await deleteDir(module.path);
-  }
-
-  // Updates module's 'location' not to have 'protocol:' in the beginning (only for "file:" needed).
-  private removeProtocolFromLocation(module: ModuleSetting) {
-    const protocol = this.protocol(module);
-    module.location = module.location.substring(
-      protocol.length + 1,
-      module.location.length,
-    );
-  }
-
   // Checks for duplicate ModuleSetting entries and throws an error if modules
   // with the same name have different branches or locations.
-  // Treats undefined branch, empty string branch, and "main" branch as equivalent.
+  // Treats undefined branch, empty string branch, and default branch as equivalent.
   // Returns an array with duplicate entries removed
-  private removeDuplicates(modules: ModuleSetting[]): ModuleSetting[] {
+  private async removeDuplicates(
+    modules: ModuleSetting[],
+  ): Promise<ModuleSetting[]> {
     const moduleMap = new Map<string, ModuleSetting>();
 
-    // Assume that empty, or missing branch means 'main'
-    const normalizeBranch = (branch: string | undefined): string => {
-      if (!branch || branch === '' || branch === MAIN_BRANCH) {
-        return MAIN_BRANCH;
-      }
-      return branch;
+    // Normalize branch names by checking against the default branch for each module
+
+    const normalizeBranch = async (module: ModuleSetting) => {
+      return module.branch ? module.branch! : await this.defaultBranch(module);
     };
 
     for (const module of modules) {
@@ -291,8 +383,8 @@ export class ModuleManager {
         ) {
           throw new Error(
             `Module conflict: '${module.name}' has different access:\n` +
-              `  - ${existingModule.private === true ? 'true' : 'false'}\n` +
-              `  - ${module.private === true ? 'true' : 'false'}`,
+              `  - ${Boolean(existingModule.private)}\n` +
+              `  - ${Boolean(module.private)}`,
           );
         }
         if (existingModule.location !== module.location) {
@@ -302,8 +394,8 @@ export class ModuleManager {
               `  - ${module.location}`,
           );
         }
-        const existingBranch = normalizeBranch(existingModule.branch);
-        const newBranch = normalizeBranch(module.branch);
+        const existingBranch = await normalizeBranch(existingModule);
+        const newBranch = await normalizeBranch(module);
 
         if (existingBranch !== newBranch) {
           throw new Error(
@@ -319,6 +411,15 @@ export class ModuleManager {
     return Array.from(moduleMap.values());
   }
 
+  // Remove module files.
+  private async removeModuleFiles(moduleName: string) {
+    const module = await this.project.module(moduleName);
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
+    }
+    await deleteDir(module.path);
+  }
+
   // Gets repository name from gitUrl
   private repositoryName(gitUrl: string): string {
     const last = gitUrl.lastIndexOf('/');
@@ -326,19 +427,47 @@ export class ModuleManager {
     return repoName;
   }
 
-  // Sets cloning options.
-  private setCloneOptions(module: ModuleSetting) {
+  // Sets cloning options with support for default branch.
+  private async setCloneOptions(module: ModuleSetting): Promise<string[]> {
     const cloneOptions = ['--depth', '1'];
+    const defaultBranch = await this.defaultBranch(module);
+    // Only specify branch if it's different from the default branch
     if (
       module.branch &&
       module.branch !== '' &&
-      module.branch !== MAIN_BRANCH
+      module.branch !== defaultBranch
     ) {
       cloneOptions.push('--branch', module.branch);
     }
     return cloneOptions;
   }
 
+  // Updates module's 'location' not to have 'protocol:' in the beginning (only for "file:" needed).
+  private stripProtocolFromLocation(module: ModuleSetting) {
+    const protocol = this.isFileModule(module) ? 'file' : 'git';
+    module.location = module.location.substring(
+      protocol.length + 1,
+      module.location.length,
+    );
+  }
+
+  // Fetches all dependencies for a module.
+  private async transientDependencies(
+    moduleName: string,
+  ): Promise<Set<string>> {
+    const dependencies = await this.dependencies(moduleName);
+    let transientDependencies: Set<string> = new Set(dependencies);
+    for (const dependency of dependencies) {
+      const depTransients = await this.transientDependencies(dependency);
+      transientDependencies = new Set([
+        ...transientDependencies,
+        ...depTransients,
+      ]);
+    }
+
+    return transientDependencies;
+  }
+
   // Updates modules in the project.
   private async update(module?: ModuleSetting, credentials?: Credentials) {
     // Prints dots every half second so that user knows that something is ongoing
@@ -350,7 +479,9 @@ export class ModuleManager {
     // Stops the above, and shows results
     function finished(interval: NodeJS.Timeout, modules: string[]) {
       clearInterval(interval);
-      console.log(`\n... Found modules: ${modules.join(', ')}`);
+      if (modules.length > 0) {
+        console.log(`\n... Found modules: ${modules.join(', ')}`);
+      }
     }
 
     await this.prepare();
@@ -366,8 +497,7 @@ export class ModuleManager {
     let uniqueModules: ModuleSetting[] = [];
     try {
       await this.collectModulePrefixes(modules, credentials);
-
-      uniqueModules = this.removeDuplicates(this.modules);
+      uniqueModules = await this.removeDuplicates(this.modules);
     } finally {
       finished(
         dotInterval,
@@ -380,7 +510,6 @@ export class ModuleManager {
         promises.push(this.handleModule(module)),
       );
       await Promise.all(promises);
-
       await deleteDir(this.tempModulesDir);
       await this.project.collectModuleResources();
     }
@@ -467,6 +596,39 @@ export class ModuleManager {
     return modulePrefix;
   }
 
+  /**
+   * Removed module from project.
+   * If module is not used by any other modules, then will remove the module from disk as well.
+   * Otherwise, only updates project configuration.
+   * @param moduleName Name of the module to remove
+   */
+  public async removeModule(moduleName: string) {
+    const projectModules = this.project.configuration.modules;
+    const dependencies = await this.buildDependencyGraph(projectModules);
+    const module = await this.project.module(moduleName);
+
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
+    }
+
+    // Project module can always be removed from project configuration,
+    // but modules under .cards/modules must be checked not to be used by
+    // other modules.
+    if (this.canBeRemoved(dependencies, moduleName)) {
+      const orphans = await this.orphanedModules(dependencies, moduleName);
+      await deleteDir(module.path);
+      for (const moduleToDelete of orphans) {
+        const modulePath = join(
+          this.project.paths.modulesFolder,
+          moduleToDelete,
+        );
+        await deleteDir(modulePath);
+      }
+      await this.project.collectModuleResources();
+    }
+    await this.project.configuration.removeModule(moduleName);
+  }
+
   /**
    * Imports module from a local file path or a git URL.
    * @param module Module to update. If not provided, updates all modules.

package/src/project-settings.ts

@@ -57,18 +57,9 @@ export class ProjectConfiguration implements ProjectSettings {
 
   // Sets configuration values from file.
   private readSettings() {
-    let settings;
-    try {
-      settings = readJsonFileSync(this.settingPath) as ProjectConfiguration;
-    } catch {
-      throw new Error(
-        `Invalid path '${this.settingPath}' to configuration file`,
-      );
-    }
+    const settings = readJsonFileSync(this.settingPath) as ProjectConfiguration;
     if (!settings) {
-      throw new Error(
-        `Invalid path '${this.settingPath}' to configuration file`,
-      );
+      throw new Error(`File at '${this.settingPath}' is not a valid JSON file`);
     }
 
     const valid =