@cyberismo/data-handler 0.0.11 → 0.0.13

package/src/commands/create.ts

@@ -44,6 +44,8 @@ import { ReportResource } from '../resources/report-resource.js';
 import { TemplateResource } from '../resources/template-resource.js';
 import { WorkflowResource } from '../resources/workflow-resource.js';
 
+const MODULES_PATH = `${sep}modules${sep}`;
+
 // todo: Is there a easy to way to make JSON schema into a TypeScript interface/type?
 //       Check this out: https://www.npmjs.com/package/json-schema-to-ts
 
@@ -65,6 +67,7 @@ export class Create {
         cardKeyPrefix: '$PROJECT-PREFIX',
         name: '$PROJECT-NAME',
         modules: [],
+        hubs: [],
       },
       name: Project.projectConfigFileName,
     },
@@ -122,13 +125,13 @@ export class Create {
       ? await templateObject.findSpecificCard(card)
       : undefined;
     if (card && !specificCard) {
-      throw Error(
+      throw new Error(
         `Card '${card}' was not found from template '${origTemplateName}'`,
       );
     }
 
     if (templateObject.templateFolder().includes(`${sep}modules${sep}`)) {
-      throw Error(`Cannot add cards to imported module templates`);
+      throw new Error(`Cannot add cards to imported module templates`);
     }
 
     // Collect all add-card promises and settle them in parallel.
@@ -159,6 +162,14 @@ export class Create {
     }
   }
 
+  /**
+   * Adds a new hub location.
+   * @param hubUrl URL of the hub
+   */
+  public async addHubLocation(hubUrl: string) {
+    return this.project.configuration.addHub(hubUrl);
+  }
+
   /**
    * Adds an attachment to a card.
    * @param cardKey card ID
@@ -176,12 +187,10 @@ export class Create {
       );
     }
     const attachmentFolder = await this.project.cardAttachmentFolder(cardKey);
-    if (!attachmentFolder) {
-      throw new Error(`Attachment folder for '${cardKey}' not found`);
-    }
 
     // Imported templates cannot be modified.
-    if (attachmentFolder.includes(`${sep}modules${sep}`)) {
+    // @todo: make MODULES_PATH project level constant
+    if (attachmentFolder.includes(MODULES_PATH)) {
       throw new Error(`Cannot modify imported module`);
     }
 
@@ -528,20 +537,23 @@ export class Create {
         ),
       );
     });
-    Create.JSONFileContent.forEach(async (entry) => {
-      if ('cardKeyPrefix' in entry.content) {
-        if (entry.content.cardKeyPrefix.includes('$PROJECT-PREFIX')) {
-          entry.content.cardKeyPrefix = projectPrefix.toLowerCase();
-        }
-        if (entry.content.name.includes('$PROJECT-NAME')) {
-          entry.content.name = projectName;
+
+    await Promise.all(
+      Create.JSONFileContent.map(async (entry) => {
+        if ('cardKeyPrefix' in entry.content) {
+          if (entry.content.cardKeyPrefix.includes('$PROJECT-PREFIX')) {
+            entry.content.cardKeyPrefix = projectPrefix.toLowerCase();
+          }
+          if (entry.content.name.includes('$PROJECT-NAME')) {
+            entry.content.name = projectName;
+          }
         }
-      }
-      await writeJsonFile(
-        join(projectPath, entry.path, entry.name),
-        entry.content,
-      );
-    });
+        await writeJsonFile(
+          join(projectPath, entry.path, entry.name),
+          entry.content,
+        );
+      }),
+    );
 
     try {
       await writeFile(

package/src/commands/fetch.ts

@@ -0,0 +1,152 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation.
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+  FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+  details. You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { mkdir } from 'node:fs/promises';
+import { resolve, sep } from 'node:path';
+import type { Project } from '../containers/project.js';
+
+import { writeJsonFile } from '../utils/json.js';
+import { validateJson } from '../utils/validate.js';
+import { type ModuleSetting } from '../interfaces/project-interfaces.js';
+import { errorFunction, getChildLogger } from '../utils/log-utils.js';
+
+const FETCH_TIMEOUT = 30000; // 30s timeout for fetching a hub file.
+const MAX_RESPONSE_SIZE = 1024 * 1024; // 1MB limit for safety
+const HUB_SCHEMA = 'hubSchema';
+const MODULE_LIST_FILE = 'moduleList.json';
+const TEMP_FOLDER = `.temp`;
+
+export const MODULE_LIST_FULL_PATH = `${TEMP_FOLDER}/${MODULE_LIST_FILE}`;
+
+export class Fetch {
+  constructor(private project: Project) {}
+
+  private get logger() {
+    return getChildLogger({
+      module: 'fetch',
+    });
+  }
+
+  private async fetchJSON(location: string, schemaId: string) {
+    try {
+      const url = new URL(`${location}/${MODULE_LIST_FILE}`);
+      if (!['http:', 'https:'].includes(url.protocol)) {
+        throw new Error(
+          `Invalid protocol: ${url.protocol}. Only HTTP and HTTPS are supported.`,
+        );
+      }
+
+      this.logger.info(`Fetching module list from: ${url.toString()}`);
+      const response = await fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          Accept: 'application/json',
+          'User-Agent': 'Cyberismo/1.0',
+        },
+        signal: AbortSignal.timeout(FETCH_TIMEOUT),
+      });
+
+      if (!response.ok) {
+        throw new Error(
+          `HTTP ${response.status}: ${response.statusText} when fetching from ${url.toString()}`,
+        );
+      }
+
+      // Check content length before downloading
+      const contentLength = response.headers.get('content-length');
+      if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+        throw new Error(
+          `Response too large: ${contentLength} bytes (max: ${MAX_RESPONSE_SIZE})`,
+        );
+      }
+
+      const contentType = response.headers.get('content-type');
+      if (!contentType?.includes('application/json')) {
+        this.logger.warn(`Expected JSON response, got: ${contentType}`);
+      }
+
+      const json = await response.json();
+      // Validate the incoming JSON before saving it into a file.
+      await validateJson(json, { schemaId: schemaId });
+
+      // Validate JSON structure and prevent prototype pollution
+      if (typeof json !== 'object' || json === null || Array.isArray(json)) {
+        throw new Error('Response must be a JSON object');
+      }
+
+      // Additional size check after JSON parsing
+      if (JSON.stringify(json).length > MAX_RESPONSE_SIZE) {
+        throw new Error('JSON content too large after parsing');
+      }
+
+      return json;
+    } catch (error) {
+      this.logger.error(
+        error,
+        `Failed to fetch module list from ${location}: ${errorFunction(error)}`,
+      );
+      throw error;
+    }
+  }
+
+  /**
+   * Fetches modules from modules hub(s) and writes them to a file.
+   */
+  public async fetchHubs() {
+    const hubs = this.project.configuration.hubs;
+
+    const moduleMap: Map<string, ModuleSetting> = new Map([]);
+
+    for (const hub of hubs) {
+      const json = await this.fetchJSON(hub.location, HUB_SCHEMA);
+      json.modules.forEach((module: ModuleSetting) => {
+        if (!moduleMap.has(module.name)) {
+          moduleMap.set(module.name, module);
+        } else {
+          this.logger.info(
+            `Skipping module '${module.name}' since it was already listed.`,
+          );
+        }
+      });
+    }
+
+    try {
+      const fullPath = resolve(this.project.basePath, MODULE_LIST_FULL_PATH);
+      const normalizedBasePath = resolve(this.project.basePath);
+
+      // Ensure the file is written within the project directory (prevent path traversal)
+      if (
+        !fullPath.startsWith(normalizedBasePath + sep) &&
+        fullPath !== normalizedBasePath
+      ) {
+        throw new Error(
+          'Invalid file path: attempting to write outside project directory',
+        );
+      }
+
+      await mkdir(resolve(this.project.basePath, TEMP_FOLDER), {
+        recursive: true,
+      });
+      await writeJsonFile(fullPath, {
+        modules: Array.from(moduleMap.values()),
+      });
+      this.logger.info(`Module list written to: ${fullPath}`);
+    } catch (error) {
+      this.logger.error(
+        error,
+        `Failed to write module list to local file: ${errorFunction(error)}`,
+      );
+      throw error;
+    }
+  }
+}

package/src/commands/import.ts

@@ -136,6 +136,7 @@ export class Import {
   ) {
     const beforeImportValidateErrors = await Validate.getInstance().validate(
       this.project.basePath,
+      () => this.project,
     );
     const gitModule = source.startsWith('https') || source.startsWith('git@');
     const modulePrefix = gitModule
@@ -164,6 +165,7 @@ export class Import {
     // Validate the project after module has been imported
     const afterImportValidateErrors = await Validate.getInstance().validate(
       this.project.basePath,
+      () => this.project,
     );
     if (afterImportValidateErrors.length > beforeImportValidateErrors.length) {
       console.error(

package/src/commands/index.ts

@@ -15,6 +15,7 @@ import { Calculate } from './calculate.js';
 import { Create } from './create.js';
 import { Edit } from './edit.js';
 import { Export } from './export.js';
+import { Fetch } from './fetch.js';
 import { Import } from './import.js';
 import { Move } from './move.js';
 import { Remove } from './remove.js';
@@ -29,6 +30,7 @@ export {
   Create,
   Edit,
   Export,
+  Fetch,
   Import,
   Move,
   Remove,

package/src/commands/remove.ts

@@ -53,9 +53,6 @@ export class Remove {
     }
 
     const attachmentFolder = await this.project.cardAttachmentFolder(cardKey);
-    if (!attachmentFolder) {
-      throw new Error(`Card '${cardKey}' not found`);
-    }
 
     // Imported templates cannot be modified.
     if (attachmentFolder.includes(MODULES_PATH)) {
@@ -95,13 +92,16 @@ export class Remove {
       },
     );
     const promiseContainer: Promise<void>[] = [];
-    allCards.filter((item) => {
-      item.metadata?.links.forEach(async (link) => {
+
+    for (const item of allCards) {
+      const links = item.metadata?.links ?? [];
+      for (const link of links) {
         if (link.cardKey === cardKey) {
           promiseContainer.push(this.removeLink(item.key, link.cardKey));
         }
-      });
-    });
+      }
+    }
+
     await Promise.all(promiseContainer);
 
     // Calculations need to be updated before card is removed.
@@ -182,6 +182,11 @@ export class Remove {
     await this.project.updateCardMetadataKey(sourceCardKey, 'links', newLinks);
   }
 
+  // Remove a hub from project.
+  private async removeHubLocation(name: string) {
+    await this.project.configuration.removeHub(name);
+  }
+
   /**
    * Removes either attachment, card, imported module, link or resource from project.
    * @param type Type of resource
@@ -221,14 +226,15 @@ export class Remove {
       return resource?.delete();
     } else {
       // Something else than resources...
-      if (type == 'attachment')
+      if (type === 'attachment')
         return this.removeAttachment(targetName, rest[0]);
-      else if (type == 'card') return this.removeCard(targetName);
-      else if (type == 'link')
+      else if (type === 'card') return this.removeCard(targetName);
+      else if (type === 'link')
         return this.removeLink(targetName, rest[0], rest[1], rest.at(2));
-      else if (type == 'module')
+      else if (type === 'module')
         return this.moduleManager.removeModule(targetName);
-      else if (type == 'label') return this.removeLabel(targetName, rest[0]);
+      else if (type === 'label') return this.removeLabel(targetName, rest[0]);
+      else if (type === 'hub') return this.removeHubLocation(targetName);
     }
     throw new Error(`Unknown resource type '${type}'`);
   }

package/src/commands/rename.ts

@@ -160,17 +160,17 @@ export class Rename {
     );
 
     // Then replace all values that match in the conversion map.
-    files.forEach(async (item) => {
-      const target = join(item.parentPath, item.name);
-      let fileContent = await readFile(target, {
-        encoding: 'utf-8',
-      });
-      for (const [key, value] of conversionMap) {
-        const re = new RegExp(key, 'g');
-        fileContent = fileContent.replaceAll(re, value);
-      }
-      await writeFile(target, fileContent);
-    });
+    await Promise.all(
+      files.map(async (item) => {
+        const target = join(item.parentPath, item.name);
+        let fileContent = await readFile(target, { encoding: 'utf-8' });
+        for (const [key, value] of conversionMap) {
+          const re = new RegExp(key, 'g');
+          fileContent = fileContent.replace(re, value);
+        }
+        await writeFile(target, fileContent);
+      }),
+    );
   }
 
   // Changes the name of a resource to match the new prefix.

package/src/commands/show.ts

@@ -18,6 +18,8 @@ import { join, resolve } from 'node:path';
 import { spawn } from 'node:child_process';
 import { readFile, writeFile } from 'node:fs/promises';
 
+import { MODULE_LIST_FULL_PATH } from './fetch.js';
+
 import mime from 'mime-types';
 
 import type { attachmentPayload } from '../interfaces/request-status-interfaces.js';
@@ -26,6 +28,8 @@ import type {
   Card,
   CardListContainer,
   ModuleContent,
+  HubSetting,
+  ModuleSettingFromHub,
   ProjectFetchCardDetails,
   ProjectMetadata,
   Resource,
@@ -53,6 +57,8 @@ import ReportMacro from '../macros/report/index.js';
 import TaskQueue from '../macros/task-queue.js';
 import { evaluateMacros } from '../macros/index.js';
 import { FolderResource } from '../resources/folder-resource.js';
+import { readJsonFile } from '../utils/json.js';
+import { getChildLogger } from '../utils/log-utils.js';
 
 /**
  * Show command.
@@ -76,6 +82,12 @@ export class Show {
     ]);
   }
 
+  private get logger() {
+    return getChildLogger({
+      module: 'show',
+    });
+  }
+
   // Collect all labels from cards.
   private collectLabels = (cards: Card[]): string[] => {
     return cards.reduce<string[]>((labels, card) => {
@@ -320,6 +332,8 @@ export class Show {
 
   /**
    * Shows the content of a file in a resource.
+   * TODO: To be removed
+   * @deprecated
    * @param resourceName Name of the resource.
    * @param fileName Name of the file to show.
    * @returns the content of the file.
@@ -348,6 +362,8 @@ export class Show {
 
   /**
    * Shows all file names in a folder resource.
+   * TODO: To be removed
+   * @deprecated
    * @param resourceName Name of the resource.
    * @returns all file names in the resource.
    */
@@ -369,6 +385,54 @@ export class Show {
     }
     return resource.showFileNames();
   }
+
+  /**
+   * Shows importable modules.
+   * @param showAll - When true, shows all importable modules, even if they have already been imported
+   * @param showDetails - When true, shows all properties of modules, not just name.
+   * @returns list of modules; the list content depends on the parameters provided
+   *          by default it is a list of module names that could be imported into the project,
+   *          with 'showDetails' true, instead of name, the list consists of full details of the modules
+   *          with 'showAll' true, the list consists of all modules in the hubs, even if they have already been imported
+   *          Note that the two boolean options can be combined.
+   */
+  public async showImportableModules(
+    showAll?: boolean,
+    showDetails?: boolean,
+  ): Promise<ModuleSettingFromHub[]> {
+    try {
+      const moduleList = (
+        await readJsonFile(
+          resolve(this.project.basePath, MODULE_LIST_FULL_PATH),
+        )
+      ).modules;
+      const currentModules = await this.project.modules();
+      const nonImportedModules = moduleList.filter(
+        (item: ModuleSettingFromHub) => {
+          return !currentModules.some((module) => item.name === module.name);
+        },
+      );
+
+      if (showAll && showDetails) {
+        return moduleList;
+      }
+      if (showAll) {
+        return moduleList?.map((item: ModuleSettingFromHub) => item?.name);
+      }
+      if (showDetails) {
+        return nonImportedModules;
+      }
+      // By default return the non-imported modules
+      return nonImportedModules.map((item: ModuleSettingFromHub) => item?.name);
+    } catch (error) {
+      if (error instanceof Error) {
+        this.logger.error(error.message);
+      }
+      // Module list doesn't exist, return empty list
+      return [];
+    }
+  }
+
   /**
    * Returns all unique labels in a project
    * @returns labels in a list
@@ -406,6 +470,14 @@ export class Show {
     return moduleDetails;
   }
 
+  /**
+   * Shows hubs of the project.
+   * @returns list of hubs.
+   */
+  public showHubs(): HubSetting[] {
+    return this.project.configuration.hubs;
+  }
+
   /**
    * Returns all project cards in the project. Cards don't have content and nor metadata.
    * @note AppUi uses this method.

package/src/commands/update.ts

@@ -14,6 +14,7 @@ import type {
   AddOperation,
   ChangeOperation,
   Operation,
+  OperationFor,
   RankOperation,
   RemoveOperation,
   UpdateOperations,
@@ -44,7 +45,6 @@ export class Update {
     optionalDetail?: Type, // todo: for 'rank' it might be reasonable to accept also 'number'
     mappingTable?: { stateMapping: Record<string, string> },
   ) {
-    const resource = Project.resourceObject(this.project, resourceName(name));
     const op: Operation<Type> = {
       name: operation,
       target: '' as Type,
@@ -76,7 +76,25 @@ export class Update {
         : undefined;
     }
 
-    await resource?.update(key, op);
+    await this.applyResourceOperation(name, key, op);
+  }
+
+  /**
+   * Update single resource property
+   * This is similar to updateValue, but allows the operation to be fully specified
+   * @param name Name of the resource to operate on.
+   * @param key Property to change in resource JSON
+   * @param operation The full operation object
+   * @template Type Type of the target of the operation
+   * @template T Type of operation ('add', 'remove', 'change', 'rank')
+   */
+  public async applyResourceOperation<Type, T extends UpdateOperations>(
+    name: string,
+    key: string,
+    operation: OperationFor<Type, T>,
+  ) {
+    const resource = Project.resourceObject(this.project, resourceName(name));
+    await resource?.update(key, operation);
     this.project.collectLocalResources();
   }
 }

package/src/commands/validate.ts

@@ -14,7 +14,6 @@
 // node
 import { type Dirent } from 'node:fs';
 import { basename, dirname, extname, join, parse, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
 import { readdir } from 'node:fs/promises';
 
 // dependencies
@@ -52,7 +51,7 @@ const SHORT_TEXT_MAX_LENGTH = 80;
 
 import * as EmailValidator from 'email-validator';
 import { evaluateMacros } from '../macros/index.js';
-const baseDir = dirname(fileURLToPath(import.meta.url));
+const baseDir = import.meta.dirname;
 const subFoldersToValidate = ['.cards', 'cardRoot'];
 
 export interface LengthProvider {
@@ -522,9 +521,13 @@ export class Validate {
    * Validates that a given directory path (and its children) conform to a JSON schema.
    * @note Validates also content in the directory tree, if .schema file is found.
    * @param projectPath path to validate.
+   * @param projectFn function that returns a Project instance.
    * @returns string containing all validation errors
    */
-  public async validate(projectPath: string): Promise<string> {
+  public async validate(
+    projectPath: string,
+    projectFn: () => Project,
+  ): Promise<string> {
     let validationErrors = '';
     this.validatedFieldTypes.clear();
     this.validatedWorkflows.clear();
@@ -546,7 +549,7 @@ export class Validate {
         return validationErrors;
       } else {
         const errorMsg: string[] = [];
-        const project = new Project(projectPath);
+        const project = projectFn();
 
         // Then, validate that each 'contentSchema' children as well.
         const result = await this.readAndValidateContentFiles(
@@ -606,17 +609,17 @@ export class Validate {
             });
           }
         }
-        if (errorMsg.length) {
-          validationErrors += errorMsg
-            .filter(this.removeDuplicateEntries)
-            .join('\n');
-        }
         // Validate that there are no duplicate card keys
         for (const [key, count] of cardIds) {
           if (count > 1) {
-            validationErrors += `Duplicate card key '${key}' found ${count} times\n`;
+            errorMsg.push(`Duplicate card key '${key}' found ${count} times`);
           }
         }
+        if (errorMsg.length) {
+          validationErrors += errorMsg
+            .filter(this.removeDuplicateEntries)
+            .join('\n');
+        }
       }
     } catch (error) {
       validationErrors += errorFunction(error);

package/src/containers/card-container.ts

@@ -18,9 +18,9 @@ import type { Dirent } from 'node:fs';
 import { readdir, readFile, writeFile } from 'node:fs/promises';
 
 import { findParentPath } from '../utils/card-utils.js';
+import { getFilesSync } from '../utils/file-utils.js';
 import { readJsonFile } from '../utils/json.js';
 import { writeJsonFile } from '../utils/json.js';
-import { getFilesSync } from '../utils/file-utils.js';
 
 // interfaces
 import {