@cyberismo/data-handler 0.0.7 → 0.0.9

package/src/module-manager.ts

@@ -29,10 +29,12 @@ import { readJsonFile } from './utils/json.js';
 import { Validate } from './commands/index.js';
 
 const FILE_PROTOCOL = 'file:';
-// todo: add support for git's default branch.
-const MAIN_BRANCH = 'main';
-// timeout in milliseconds for git client (no stdout / stderr activity)
-const DEFAULT_TIMEOUT = 10000;
+const HTTPS_PROTOCOL = 'https:';
+
+// When dependencies are built to a map, use map that has
+//   key: module name,
+//   value: list of unique prefixes
+type DependencyGraph = Map<string, Set<string>>;
 
 /**
  * Class that handles module updates and imports.
@@ -40,6 +42,8 @@ const DEFAULT_TIMEOUT = 10000;
 export class ModuleManager {
   private modules: ModuleSetting[] = [];
   private tempModulesDir: string = '';
+  private defaultBranchCache: Map<string, string> = new Map();
+
   constructor(private project: Project) {
     this.tempModulesDir = join(this.project.paths.tempFolder, 'modules');
   }
@@ -53,6 +57,35 @@ export class ModuleManager {
     await this.project.collectModuleResources();
   }
 
+  // Creates a map of what dependencies each module depend from.
+  private async buildDependencyGraph(
+    dependencies: ModuleSetting[],
+  ): Promise<DependencyGraph> {
+    const dependencyNames = dependencies.map((item) => item.name);
+    const dependencyGraph = new Map<string, Set<string>>() as DependencyGraph;
+    for (const dependency of dependencyNames) {
+      dependencyGraph.set(
+        dependency,
+        await this.transientDependencies(dependency),
+      );
+    }
+    return dependencyGraph;
+  }
+
+  // Returns 'true' if 'moduleName' can be removed.
+  private canBeRemoved(
+    dependencies: DependencyGraph,
+    moduleName: string,
+  ): boolean {
+    let unused = true;
+    dependencies.forEach((transientDependencies, key) => {
+      if (key !== moduleName && transientDependencies.has(moduleName)) {
+        unused = false;
+      }
+    });
+    return unused;
+  }
+
   // Handles cloning of a repository.
   private async clone(
     module: ModuleSetting,
@@ -67,10 +100,15 @@ export class ModuleManager {
 
     let remote = module.location;
     if (module.private) {
-      if (credentials && credentials?.username && credentials?.token) {
+      if (
+        credentials &&
+        credentials?.username &&
+        credentials?.token &&
+        module.location.startsWith(HTTPS_PROTOCOL)
+      ) {
         if (verbose) {
           console.log(
-            `... Using credentials '${credentials?.username}' for cloning '${module.name}'`,
+            `... Using HTTPS with credentials '${credentials?.username}' for cloning '${module.name}'`,
           );
         }
         try {
@@ -83,21 +121,27 @@ export class ModuleManager {
         } catch {
           throw new Error(`Invalid repository URL: ${module.location}`);
         }
-      } else {
+      } else if (module.location.startsWith('git@')) {
         if (verbose) {
-          console.log(`... Not using credentials for cloning '${module.name}'`);
+          console.log(`... Using SSH for cloning '${module.name}'`);
         }
       }
+    } else {
+      if (verbose) {
+        console.log(
+          `... Using HTTPS without credentials for cloning '${module.name}'`,
+        );
+      }
     }
 
     try {
       await mkdir(this.tempModulesDir, { recursive: true });
-      const cloneOptions = this.setCloneOptions(module);
+      const cloneOptions = await this.setCloneOptions(module);
       await rm(destinationPath, { recursive: true, force: true });
 
       const git: SimpleGit = simpleGit({
         timeout: {
-          block: DEFAULT_TIMEOUT,
+          block: this.gitTimeout(),
         },
       });
 
@@ -110,7 +154,6 @@ export class ModuleManager {
         console.log(`... Cloned '${module.name}' to a temporary folder`);
       }
     } catch (error) {
-      console.error(error);
       if (error instanceof Error)
         throw new Error(
           `Failed to clone module '${module.name}': ${error.message}`,
@@ -143,6 +186,61 @@ export class ModuleManager {
     }
   }
 
+  // Gets the default branch for a repository from remote or cache
+  private async defaultBranch(module: ModuleSetting): Promise<string> {
+    if (this.defaultBranchCache.has(module.location)) {
+      return this.defaultBranchCache.get(module.location)!;
+    }
+    // Set the default branch if branch was not specified
+    if (!module.branch) {
+      const destinationPath = join(this.tempModulesDir, module.name);
+      // Only return path after cloning
+      if (pathExists(destinationPath)) {
+        const git: SimpleGit = simpleGit({
+          timeout: {
+            block: this.gitTimeout(),
+          },
+        });
+        const options = ['--abbrev-ref', 'HEAD'];
+        const defaultBranch = await git.cwd(destinationPath).revparse(options);
+        this.defaultBranchCache.set(module.location, defaultBranch);
+        return defaultBranch;
+      }
+    }
+    // The actual default branch will be updated later (after cloning).
+    return '';
+  }
+
+  // Fetches direct dependencies of a module.
+  private async dependencies(moduleName: string): Promise<Set<string>> {
+    const allModules = await this.project.modules();
+    if (!allModules) return new Set();
+    const module = allModules.find((m) => m.name === moduleName);
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
+    }
+    const modulePath = join(module.path, module.name, 'cardsConfig.json');
+    const moduleConfiguration = (await readJsonFile(
+      modulePath,
+    )) as ProjectConfiguration;
+    return moduleConfiguration.modules
+      ? new Set(moduleConfiguration.modules.map((m) => m.name))
+      : new Set();
+  }
+
+  // Increase timeout for CI environments and add platform-specific adjustments
+  private gitTimeout(): number {
+    const baseTimeout = 15000;
+    const isCI = process.env.CI;
+    const isWindows = process.platform === 'win32';
+
+    let timeout = baseTimeout;
+    if (isCI) timeout *= 2; // Double timeout in CI
+    if (isWindows) timeout *= 1.5; // 50% more time on Windows
+
+    return timeout;
+  }
+
   // Collects one module's dependency prefixes to 'this.modules'.
   // Note that there can be duplicate entries.
   private async doCollectModulePrefix(
@@ -167,16 +265,17 @@ export class ModuleManager {
 
   // Updates one module that is read from local file system.
   private async handleFileModule(module: ModuleSetting) {
-    this.removeProtocolFromLocation(module);
+    this.stripProtocolFromLocation(module);
     await this.remove(module);
-    await this.importFromFolder(module);
+    await this.importFromFolder(module.location, module.name);
   }
 
   // Updates one module that is received from Git.
   private async handleGitModule(module: ModuleSetting) {
     await this.clone(module);
+    const tempLocation = join(this.tempModulesDir, module.name);
     await this.remove(module);
-    await this.importFromTemp(module);
+    await this.importFromFolder(tempLocation, module.name);
   }
 
   // Updates one module.
@@ -186,19 +285,11 @@ export class ModuleManager {
       : this.handleFileModule(module);
   }
 
-  // Handles importing a module from module settings 'location'
-  private async importFromFolder(module: ModuleSetting) {
-    await this.importFileModule(module.location);
+  // Imports from a given folder. Is used both for .temp/<module name> and file locations.
+  private async importFromFolder(path: string, name: string) {
+    await this.importFileModule(path);
     console.log(
-      `... Imported module '${module.name}' to '${this.project.configuration.name}'`,
-    );
-  }
-
-  // Handles importing a module from '.temp' folder
-  private async importFromTemp(module: ModuleSetting) {
-    await this.importFileModule(join(this.tempModulesDir, module.name));
-    console.log(
-      `... Imported module '${module.name}' to '${this.project.configuration.name}'`,
+      `... Imported module '${name}' to '${this.project.configuration.name}'`,
     );
   }
 
@@ -211,7 +302,41 @@ export class ModuleManager {
   // Returns true if module is imported from git.
   private isGitModule(module: ModuleSetting): boolean {
     if (!module.location) return false;
-    return module.location.startsWith('https:');
+    return (
+      module.location.startsWith('https:') || module.location.startsWith('git@')
+    );
+  }
+
+  // Collect modules that could be removed from .cards/modules when
+  // 'moduleName' is removed.
+  private async orphanedModules(
+    dependencies: DependencyGraph,
+    moduleName: string,
+  ): Promise<string[]> {
+    const projectModules = this.project.configuration.modules;
+    const removableTransientModules: string[] = [];
+    if (dependencies.has(moduleName)) {
+      const deps = dependencies.get(moduleName);
+      for (const dependency of deps!) {
+        const projectDependency = projectModules.some(
+          (item) => item.name === dependency,
+        );
+        if (projectDependency) continue;
+
+        let orphanModule = true;
+        dependencies.forEach((transientDependencies, key) => {
+          if (key === moduleName) return;
+          if (transientDependencies.has(dependency)) {
+            orphanModule = false;
+          }
+        });
+
+        if (orphanModule) {
+          removableTransientModules.push(dependency);
+        }
+      }
+    }
+    return removableTransientModules;
   }
 
   // Prepares '.temp/modules' for cloning
@@ -231,11 +356,6 @@ export class ModuleManager {
     }
   }
 
-  // Returns whether to use git or file system for handling the module.
-  private protocol(module: ModuleSetting) {
-    return this.isFileModule(module) ? 'file' : 'git';
-  }
-
   // Handles removing an imported module.
   private async remove(module: ModuleSetting) {
     try {
@@ -249,37 +369,19 @@ export class ModuleManager {
     }
   }
 
-  // Remove module files.
-  private async removeModuleFiles(moduleName: string) {
-    const module = await this.project.module(moduleName);
-    if (!module) {
-      throw new Error(`Module '${moduleName}' not found`);
-    }
-    await deleteDir(module.path);
-  }
-
-  // Updates module's 'location' not to have 'protocol:' in the beginning (only for "file:" needed).
-  private removeProtocolFromLocation(module: ModuleSetting) {
-    const protocol = this.protocol(module);
-    module.location = module.location.substring(
-      protocol.length + 1,
-      module.location.length,
-    );
-  }
-
   // Checks for duplicate ModuleSetting entries and throws an error if modules
   // with the same name have different branches or locations.
-  // Treats undefined branch, empty string branch, and "main" branch as equivalent.
+  // Treats undefined branch, empty string branch, and default branch as equivalent.
   // Returns an array with duplicate entries removed
-  private removeDuplicates(modules: ModuleSetting[]): ModuleSetting[] {
+  private async removeDuplicates(
+    modules: ModuleSetting[],
+  ): Promise<ModuleSetting[]> {
     const moduleMap = new Map<string, ModuleSetting>();
 
-    // Assume that empty, or missing branch means 'main'
-    const normalizeBranch = (branch: string | undefined): string => {
-      if (!branch || branch === '' || branch === MAIN_BRANCH) {
-        return MAIN_BRANCH;
-      }
-      return branch;
+    // Normalize branch names by checking against the default branch for each module
+
+    const normalizeBranch = async (module: ModuleSetting) => {
+      return module.branch ? module.branch! : await this.defaultBranch(module);
     };
 
     for (const module of modules) {
@@ -291,8 +393,8 @@ export class ModuleManager {
         ) {
           throw new Error(
             `Module conflict: '${module.name}' has different access:\n` +
-              `  - ${existingModule.private === true ? 'true' : 'false'}\n` +
-              `  - ${module.private === true ? 'true' : 'false'}`,
+              `  - ${Boolean(existingModule.private)}\n` +
+              `  - ${Boolean(module.private)}`,
           );
         }
         if (existingModule.location !== module.location) {
@@ -302,8 +404,8 @@ export class ModuleManager {
               `  - ${module.location}`,
           );
         }
-        const existingBranch = normalizeBranch(existingModule.branch);
-        const newBranch = normalizeBranch(module.branch);
+        const existingBranch = await normalizeBranch(existingModule);
+        const newBranch = await normalizeBranch(module);
 
         if (existingBranch !== newBranch) {
           throw new Error(
@@ -319,6 +421,15 @@ export class ModuleManager {
     return Array.from(moduleMap.values());
   }
 
+  // Remove module files.
+  private async removeModuleFiles(moduleName: string) {
+    const module = await this.project.module(moduleName);
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
+    }
+    await deleteDir(module.path);
+  }
+
   // Gets repository name from gitUrl
   private repositoryName(gitUrl: string): string {
     const last = gitUrl.lastIndexOf('/');
@@ -326,19 +437,47 @@ export class ModuleManager {
     return repoName;
   }
 
-  // Sets cloning options.
-  private setCloneOptions(module: ModuleSetting) {
+  // Sets cloning options with support for default branch.
+  private async setCloneOptions(module: ModuleSetting): Promise<string[]> {
     const cloneOptions = ['--depth', '1'];
+    const defaultBranch = await this.defaultBranch(module);
+    // Only specify branch if it's different from the default branch
     if (
       module.branch &&
       module.branch !== '' &&
-      module.branch !== MAIN_BRANCH
+      module.branch !== defaultBranch
     ) {
       cloneOptions.push('--branch', module.branch);
     }
     return cloneOptions;
   }
 
+  // Updates module's 'location' not to have 'protocol:' in the beginning (only for "file:" needed).
+  private stripProtocolFromLocation(module: ModuleSetting) {
+    const protocol = this.isFileModule(module) ? 'file' : 'git';
+    module.location = module.location.substring(
+      protocol.length + 1,
+      module.location.length,
+    );
+  }
+
+  // Fetches all dependencies for a module.
+  private async transientDependencies(
+    moduleName: string,
+  ): Promise<Set<string>> {
+    const dependencies = await this.dependencies(moduleName);
+    let transientDependencies: Set<string> = new Set(dependencies);
+    for (const dependency of dependencies) {
+      const depTransients = await this.transientDependencies(dependency);
+      transientDependencies = new Set([
+        ...transientDependencies,
+        ...depTransients,
+      ]);
+    }
+
+    return transientDependencies;
+  }
+
   // Updates modules in the project.
   private async update(module?: ModuleSetting, credentials?: Credentials) {
     // Prints dots every half second so that user knows that something is ongoing
@@ -350,7 +489,9 @@ export class ModuleManager {
     // Stops the above, and shows results
     function finished(interval: NodeJS.Timeout, modules: string[]) {
       clearInterval(interval);
-      console.log(`\n... Found modules: ${modules.join(', ')}`);
+      if (modules.length > 0) {
+        console.log(`\n... Found modules: ${modules.join(', ')}`);
+      }
     }
 
     await this.prepare();
@@ -366,8 +507,7 @@ export class ModuleManager {
     let uniqueModules: ModuleSetting[] = [];
     try {
       await this.collectModulePrefixes(modules, credentials);
-
-      uniqueModules = this.removeDuplicates(this.modules);
+      uniqueModules = await this.removeDuplicates(this.modules);
     } finally {
       finished(
         dotInterval,
@@ -380,7 +520,6 @@ export class ModuleManager {
         promises.push(this.handleModule(module)),
       );
       await Promise.all(promises);
-
       await deleteDir(this.tempModulesDir);
       await this.project.collectModuleResources();
     }
@@ -467,6 +606,39 @@ export class ModuleManager {
     return modulePrefix;
   }
 
+  /**
+   * Removed module from project.
+   * If module is not used by any other modules, then will remove the module from disk as well.
+   * Otherwise, only updates project configuration.
+   * @param moduleName Name of the module to remove
+   */
+  public async removeModule(moduleName: string) {
+    const projectModules = this.project.configuration.modules;
+    const dependencies = await this.buildDependencyGraph(projectModules);
+    const module = await this.project.module(moduleName);
+
+    if (!module) {
+      throw new Error(`Module '${moduleName}' not found`);
+    }
+
+    // Project module can always be removed from project configuration,
+    // but modules under .cards/modules must be checked not to be used by
+    // other modules.
+    if (this.canBeRemoved(dependencies, moduleName)) {
+      const orphans = await this.orphanedModules(dependencies, moduleName);
+      await deleteDir(module.path);
+      for (const moduleToDelete of orphans) {
+        const modulePath = join(
+          this.project.paths.modulesFolder,
+          moduleToDelete,
+        );
+        await deleteDir(modulePath);
+      }
+      await this.project.collectModuleResources();
+    }
+    await this.project.configuration.removeModule(moduleName);
+  }
+
   /**
    * Imports module from a local file path or a git URL.
    * @param module Module to update. If not provided, updates all modules.

package/src/permissions/action-guard.ts

@@ -10,7 +10,7 @@
     License along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import type { Calculate } from '../commands/index.js';
+import type { CalculationEngine } from '../containers/project/calculation-engine.js';
 import type { DeniedOperationCollection } from '../types/queries.js';
 
 export type Action = keyof DeniedOperationCollection;
@@ -25,7 +25,7 @@ function checkOperation<T extends { errorMessage: string }>(data: Array<T>) {
  * This class is used to guard actions from being used without permissions
  */
 export class ActionGuard {
-  constructor(private calculate: Calculate) {}
+  constructor(private calculate: CalculationEngine) {}
 
   /**
    * Checks whether an action can be done
@@ -39,7 +39,7 @@ export class ActionGuard {
     param?: string,
   ) {
     await this.calculate.generate();
-    const cards = await this.calculate.runQuery('card', {
+    const cards = await this.calculate.runQuery('card', 'localApp', {
       cardKey,
     });
     if (cards.length === 0) {

package/src/project-settings.ts

@@ -57,18 +57,9 @@ export class ProjectConfiguration implements ProjectSettings {
 
   // Sets configuration values from file.
   private readSettings() {
-    let settings;
-    try {
-      settings = readJsonFileSync(this.settingPath) as ProjectConfiguration;
-    } catch {
-      throw new Error(
-        `Invalid path '${this.settingPath}' to configuration file`,
-      );
-    }
+    const settings = readJsonFileSync(this.settingPath) as ProjectConfiguration;
     if (!settings) {
-      throw new Error(
-        `Invalid path '${this.settingPath}' to configuration file`,
-      );
+      throw new Error(`File at '${this.settingPath}' is not a valid JSON file`);
     }
 
     const valid =

package/src/resources/card-type-resource.ts

@@ -153,6 +153,50 @@ export class CardTypeResource extends FileResource {
     }
   }
 
+  // Apply state mapping to all cards using this card type.
+  // Checks that all states in the current workflow are updated.
+  private async handleWorkflowChange<Type>(
+    stateMapping: Record<string, string>,
+    op: ChangeOperation<Type>,
+  ) {
+    await this.verifyStateMapping(stateMapping, op);
+    const cards = await this.collectCards(
+      {
+        metadata: true,
+        content: true,
+      },
+      this.content.name,
+    );
+
+    const unmappedStates: string[] = [];
+
+    // Update each card's workflowState if it has a mapping
+    const updatePromises = cards.map(async (card) => {
+      if (card.metadata && card.metadata.workflowState) {
+        const currentState = card.metadata.workflowState as string;
+        const newState = stateMapping[currentState];
+
+        if (newState && newState !== currentState) {
+          this.logger.info(
+            `Updating card '${card.key}': ${currentState} -> ${newState}`,
+          );
+          card.metadata.workflowState = newState;
+          await this.project.updateCardMetadata(card, card.metadata);
+        } else if (!newState && !unmappedStates.includes(currentState)) {
+          unmappedStates.push(currentState);
+        }
+      }
+    });
+
+    await Promise.all(updatePromises);
+
+    if (unmappedStates.length > 0) {
+      this.logger.warn(
+        `Found unmapped states that were not updated: ${unmappedStates.join(', ')}`,
+      );
+    }
+  }
+
   // Checks if field type exists in this card type.
   private async hasFieldType(field: Partial<CustomField>): Promise<boolean> {
     return this.data.customFields.some((item) => item.name === field.name);
@@ -315,6 +359,57 @@ export class CardTypeResource extends FileResource {
     }
   }
 
+  // Verifies that:
+  // - all states in the current workflow are covered in the state mapping
+  // - the states are correct
+  private async verifyStateMapping<Type>(
+    stateMapping: Record<string, string>,
+    op: ChangeOperation<Type>,
+  ) {
+    const currentWorkflowName = op.target as string;
+    const currentWorkflow =
+      await this.project.resource<Workflow>(currentWorkflowName);
+    if (!currentWorkflow) {
+      throw new Error(
+        `Workflow '${currentWorkflowName}' does not exist in the project`,
+      );
+    }
+
+    const newWorkflow = await this.project.resource<Workflow>(op.to as string);
+    if (!newWorkflow) {
+      throw new Error(`Workflow '${op.to}' does not exist in the project`);
+    }
+
+    const currentWorkflowStates = currentWorkflow.states.map(
+      (state) => state.name,
+    );
+    const mappedSourceStates = Object.keys(stateMapping);
+    const unmappedCurrentStates = currentWorkflowStates.filter(
+      (stateName) => !mappedSourceStates.includes(stateName),
+    );
+
+    if (unmappedCurrentStates.length > 0) {
+      throw new Error(
+        `State mapping validation failed: The following states exist in the current workflow '${currentWorkflowName}' ` +
+          `but are not mapped from in the state mapping JSON file: ${unmappedCurrentStates.join(', ')}. ` +
+          `Please ensure all states in the current workflow are accounted for in the mapping to ensure all cards are properly updated.`,
+      );
+    }
+
+    // Also verify that all target states exist in the new workflow
+    const newWorkflowStates = newWorkflow.states.map((state) => state.name);
+    const mappedTargetStates = Object.values(stateMapping);
+    const invalidTargetStates = mappedTargetStates.filter(
+      (stateName) => !newWorkflowStates.includes(stateName),
+    );
+
+    if (invalidTargetStates.length > 0) {
+      throw new Error(
+        `State mapping validation failed: The following target states in the mapping do not exist in the new workflow '${op.to}': ${invalidTargetStates.join(', ')}.`,
+      );
+    }
+  }
+
   /**
    * Creates a new card type object. Base class writes the object to disk automatically.
    * @param workflowName Workflow name that this card type uses.
@@ -404,7 +499,12 @@ export class CardTypeResource extends FileResource {
         content.optionallyVisibleFields as Type[],
       ) as string[];
     } else if (key === 'workflow') {
+      const changeOp = op as ChangeOperation<string>;
+      const stateMapping = changeOp.mappingTable?.stateMapping || {};
       content.workflow = super.handleScalar(op) as string;
+      if (Object.keys(stateMapping).length > 0) {
+        await this.handleWorkflowChange(stateMapping, changeOp);
+      }
     } else if (key === 'customFields') {
       await this.validateFieldType(key, op);
       content.customFields = super.handleArray(

package/src/resources/file-resource.ts

@@ -137,6 +137,16 @@ export class FileResource extends ResourceObject {
     return cards;
   }
 
+  // Get (resource folder) type name
+  protected get getType() {
+    return super.getType;
+  }
+
+  // Get logger instance
+  protected get logger() {
+    return super.getLogger(this.getType);
+  }
+
   // Initialize the resource.
   protected initialize() {
     if (this.resourceName.type === '') {
@@ -301,9 +311,9 @@ export class FileResource extends ResourceObject {
   ) {
     function toValue(op: Operation<Type>) {
       if (op.name === 'rank') return op.newIndex;
-      if (op.name === 'add') return op.target;
-      if (op.name === 'remove') return op.target;
-      if (op.name === 'change') return op.to;
+      if (op.name === 'add') return JSON.stringify(op.target);
+      if (op.name === 'remove') return JSON.stringify(op.target);
+      if (op.name === 'change') return JSON.stringify(op.to);
     }
 
     // Check that new name is valid.
@@ -320,7 +330,9 @@ export class FileResource extends ResourceObject {
     } catch (error) {
       if (error instanceof Error) {
         const errorValue = typeof op === 'object' ? toValue(op) : op;
-        throw new Error(`Cannot ${op.name} '${key}' --> '${errorValue}'`);
+        throw new Error(
+          `Cannot ${op.name} '${key}' --> '${errorValue}: ${error.message}'`,
+        );
       }
     }