@cyberismo/data-handler 0.0.11 → 0.0.12

package/src/commands/fetch.ts

@@ -0,0 +1,152 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation.
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+  FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+  details. You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { mkdir } from 'node:fs/promises';
+import { resolve, sep } from 'node:path';
+import type { Project } from '../containers/project.js';
+
+import { writeJsonFile } from '../utils/json.js';
+import { validateJson } from '../utils/validate.js';
+import { type ModuleSetting } from '../interfaces/project-interfaces.js';
+import { errorFunction, getChildLogger } from '../utils/log-utils.js';
+
+const FETCH_TIMEOUT = 30000; // 30s timeout for fetching a hub file.
+const MAX_RESPONSE_SIZE = 1024 * 1024; // 1MB limit for safety
+const HUB_SCHEMA = 'hubSchema';
+const MODULE_LIST_FILE = 'moduleList.json';
+const TEMP_FOLDER = `.temp`;
+
+export const MODULE_LIST_FULL_PATH = `${TEMP_FOLDER}/${MODULE_LIST_FILE}`;
+
+export class Fetch {
+  constructor(private project: Project) {}
+
+  private get logger() {
+    return getChildLogger({
+      module: 'fetch',
+    });
+  }
+
+  private async fetchJSON(location: string, schemaId: string) {
+    try {
+      const url = new URL(`${location}/${MODULE_LIST_FILE}`);
+      if (!['http:', 'https:'].includes(url.protocol)) {
+        throw new Error(
+          `Invalid protocol: ${url.protocol}. Only HTTP and HTTPS are supported.`,
+        );
+      }
+
+      this.logger.info(`Fetching module list from: ${url.toString()}`);
+      const response = await fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          Accept: 'application/json',
+          'User-Agent': 'Cyberismo/1.0',
+        },
+        signal: AbortSignal.timeout(FETCH_TIMEOUT),
+      });
+
+      if (!response.ok) {
+        throw new Error(
+          `HTTP ${response.status}: ${response.statusText} when fetching from ${url.toString()}`,
+        );
+      }
+
+      // Check content length before downloading
+      const contentLength = response.headers.get('content-length');
+      if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+        throw new Error(
+          `Response too large: ${contentLength} bytes (max: ${MAX_RESPONSE_SIZE})`,
+        );
+      }
+
+      const contentType = response.headers.get('content-type');
+      if (!contentType?.includes('application/json')) {
+        this.logger.warn(`Expected JSON response, got: ${contentType}`);
+      }
+
+      const json = await response.json();
+      // Validate the incoming JSON before saving it into a file.
+      await validateJson(json, { schemaId: schemaId });
+
+      // Validate JSON structure and prevent prototype pollution
+      if (typeof json !== 'object' || json === null || Array.isArray(json)) {
+        throw new Error('Response must be a JSON object');
+      }
+
+      // Additional size check after JSON parsing
+      if (JSON.stringify(json).length > MAX_RESPONSE_SIZE) {
+        throw new Error('JSON content too large after parsing');
+      }
+
+      return json;
+    } catch (error) {
+      this.logger.error(
+        error,
+        `Failed to fetch module list from ${location}: ${errorFunction(error)}`,
+      );
+      throw error;
+    }
+  }
+
+  /**
+   * Fetches modules from modules hub(s) and writes them to a file.
+   */
+  public async fetchHubs() {
+    const hubs = this.project.configuration.hubs;
+
+    const moduleMap: Map<string, ModuleSetting> = new Map([]);
+
+    for (const hub of hubs) {
+      const json = await this.fetchJSON(hub.location, HUB_SCHEMA);
+      json.modules.forEach((module: ModuleSetting) => {
+        if (!moduleMap.has(module.name)) {
+          moduleMap.set(module.name, module);
+        } else {
+          this.logger.info(
+            `Skipping module '${module.name}' since it was already listed.`,
+          );
+        }
+      });
+    }
+
+    try {
+      const fullPath = resolve(this.project.basePath, MODULE_LIST_FULL_PATH);
+      const normalizedBasePath = resolve(this.project.basePath);
+
+      // Ensure the file is written within the project directory (prevent path traversal)
+      if (
+        !fullPath.startsWith(normalizedBasePath + sep) &&
+        fullPath !== normalizedBasePath
+      ) {
+        throw new Error(
+          'Invalid file path: attempting to write outside project directory',
+        );
+      }
+
+      await mkdir(resolve(this.project.basePath, TEMP_FOLDER), {
+        recursive: true,
+      });
+      await writeJsonFile(fullPath, {
+        modules: Array.from(moduleMap.values()),
+      });
+      this.logger.info(`Module list written to: ${fullPath}`);
+    } catch (error) {
+      this.logger.error(
+        error,
+        `Failed to write module list to local file: ${errorFunction(error)}`,
+      );
+      throw error;
+    }
+  }
+}

package/src/commands/index.ts

@@ -15,6 +15,7 @@ import { Calculate } from './calculate.js';
 import { Create } from './create.js';
 import { Edit } from './edit.js';
 import { Export } from './export.js';
+import { Fetch } from './fetch.js';
 import { Import } from './import.js';
 import { Move } from './move.js';
 import { Remove } from './remove.js';
@@ -29,6 +30,7 @@ export {
   Create,
   Edit,
   Export,
+  Fetch,
   Import,
   Move,
   Remove,

package/src/commands/remove.ts

@@ -53,9 +53,6 @@ export class Remove {
     }
 
     const attachmentFolder = await this.project.cardAttachmentFolder(cardKey);
-    if (!attachmentFolder) {
-      throw new Error(`Card '${cardKey}' not found`);
-    }
 
     // Imported templates cannot be modified.
     if (attachmentFolder.includes(MODULES_PATH)) {
@@ -95,13 +92,16 @@ export class Remove {
       },
     );
     const promiseContainer: Promise<void>[] = [];
-    allCards.filter((item) => {
-      item.metadata?.links.forEach(async (link) => {
+
+    for (const item of allCards) {
+      const links = item.metadata?.links ?? [];
+      for (const link of links) {
         if (link.cardKey === cardKey) {
           promiseContainer.push(this.removeLink(item.key, link.cardKey));
         }
-      });
-    });
+      }
+    }
+
     await Promise.all(promiseContainer);
 
     // Calculations need to be updated before card is removed.
@@ -182,6 +182,11 @@ export class Remove {
     await this.project.updateCardMetadataKey(sourceCardKey, 'links', newLinks);
   }
 
+  // Remove a hub from project.
+  private async removeHubLocation(name: string) {
+    await this.project.configuration.removeHub(name);
+  }
+
   /**
    * Removes either attachment, card, imported module, link or resource from project.
    * @param type Type of resource
@@ -221,14 +226,15 @@ export class Remove {
       return resource?.delete();
     } else {
       // Something else than resources...
-      if (type == 'attachment')
+      if (type === 'attachment')
         return this.removeAttachment(targetName, rest[0]);
-      else if (type == 'card') return this.removeCard(targetName);
-      else if (type == 'link')
+      else if (type === 'card') return this.removeCard(targetName);
+      else if (type === 'link')
         return this.removeLink(targetName, rest[0], rest[1], rest.at(2));
-      else if (type == 'module')
+      else if (type === 'module')
         return this.moduleManager.removeModule(targetName);
-      else if (type == 'label') return this.removeLabel(targetName, rest[0]);
+      else if (type === 'label') return this.removeLabel(targetName, rest[0]);
+      else if (type === 'hub') return this.removeHubLocation(targetName);
     }
     throw new Error(`Unknown resource type '${type}'`);
   }

package/src/commands/rename.ts

@@ -160,17 +160,17 @@ export class Rename {
     );
 
     // Then replace all values that match in the conversion map.
-    files.forEach(async (item) => {
-      const target = join(item.parentPath, item.name);
-      let fileContent = await readFile(target, {
-        encoding: 'utf-8',
-      });
-      for (const [key, value] of conversionMap) {
-        const re = new RegExp(key, 'g');
-        fileContent = fileContent.replaceAll(re, value);
-      }
-      await writeFile(target, fileContent);
-    });
+    await Promise.all(
+      files.map(async (item) => {
+        const target = join(item.parentPath, item.name);
+        let fileContent = await readFile(target, { encoding: 'utf-8' });
+        for (const [key, value] of conversionMap) {
+          const re = new RegExp(key, 'g');
+          fileContent = fileContent.replace(re, value);
+        }
+        await writeFile(target, fileContent);
+      }),
+    );
   }
 
   // Changes the name of a resource to match the new prefix.

package/src/commands/show.ts

@@ -18,6 +18,8 @@ import { join, resolve } from 'node:path';
 import { spawn } from 'node:child_process';
 import { readFile, writeFile } from 'node:fs/promises';
 
+import { MODULE_LIST_FULL_PATH } from './fetch.js';
+
 import mime from 'mime-types';
 
 import type { attachmentPayload } from '../interfaces/request-status-interfaces.js';
@@ -26,6 +28,8 @@ import type {
   Card,
   CardListContainer,
   ModuleContent,
+  HubSetting,
+  ModuleSettingFromHub,
   ProjectFetchCardDetails,
   ProjectMetadata,
   Resource,
@@ -53,6 +57,8 @@ import ReportMacro from '../macros/report/index.js';
 import TaskQueue from '../macros/task-queue.js';
 import { evaluateMacros } from '../macros/index.js';
 import { FolderResource } from '../resources/folder-resource.js';
+import { readJsonFile } from '../utils/json.js';
+import { getChildLogger } from '../utils/log-utils.js';
 
 /**
  * Show command.
@@ -76,6 +82,12 @@ export class Show {
     ]);
   }
 
+  private get logger() {
+    return getChildLogger({
+      module: 'show',
+    });
+  }
+
   // Collect all labels from cards.
   private collectLabels = (cards: Card[]): string[] => {
     return cards.reduce<string[]>((labels, card) => {
@@ -369,6 +381,54 @@ export class Show {
     }
     return resource.showFileNames();
   }
+
+  /**
+   * Shows importable modules.
+   * @param showAll - When true, shows all importable modules, even if they have already been imported
+   * @param showDetails - When true, shows all properties of modules, not just name.
+   * @returns list of modules; the list content depends on the parameters provided
+   *          by default it is a list of module names that could be imported into the project,
+   *          with 'showDetails' true, instead of name, the list consists of full details of the modules
+   *          with 'showAll' true, the list consists of all modules in the hubs, even if they have already been imported
+   *          Note that the two boolean options can be combined.
+   */
+  public async showImportableModules(
+    showAll?: boolean,
+    showDetails?: boolean,
+  ): Promise<ModuleSettingFromHub[]> {
+    try {
+      const moduleList = (
+        await readJsonFile(
+          resolve(this.project.basePath, MODULE_LIST_FULL_PATH),
+        )
+      ).modules;
+      const currentModules = await this.project.modules();
+      const nonImportedModules = moduleList.filter(
+        (item: ModuleSettingFromHub) => {
+          return !currentModules.some((module) => item.name === module.name);
+        },
+      );
+
+      if (showAll && showDetails) {
+        return moduleList;
+      }
+      if (showAll) {
+        return moduleList?.map((item: ModuleSettingFromHub) => item?.name);
+      }
+      if (showDetails) {
+        return nonImportedModules;
+      }
+      // By default return the non-imported modules
+      return nonImportedModules.map((item: ModuleSettingFromHub) => item?.name);
+    } catch (error) {
+      if (error instanceof Error) {
+        this.logger.error(error.message);
+      }
+      // Module list doesn't exist, return empty list
+      return [];
+    }
+  }
+
   /**
    * Returns all unique labels in a project
    * @returns labels in a list
@@ -406,6 +466,14 @@ export class Show {
     return moduleDetails;
   }
 
+  /**
+   * Shows hubs of the project.
+   * @returns list of hubs.
+   */
+  public showHubs(): HubSetting[] {
+    return this.project.configuration.hubs;
+  }
+
   /**
    * Returns all project cards in the project. Cards don't have content and nor metadata.
    * @note AppUi uses this method.

package/src/commands/validate.ts

@@ -14,7 +14,6 @@
 // node
 import { type Dirent } from 'node:fs';
 import { basename, dirname, extname, join, parse, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
 import { readdir } from 'node:fs/promises';
 
 // dependencies
@@ -52,7 +51,7 @@ const SHORT_TEXT_MAX_LENGTH = 80;
 
 import * as EmailValidator from 'email-validator';
 import { evaluateMacros } from '../macros/index.js';
-const baseDir = dirname(fileURLToPath(import.meta.url));
+const baseDir = import.meta.dirname;
 const subFoldersToValidate = ['.cards', 'cardRoot'];
 
 export interface LengthProvider {
@@ -606,17 +605,17 @@ export class Validate {
             });
           }
         }
-        if (errorMsg.length) {
-          validationErrors += errorMsg
-            .filter(this.removeDuplicateEntries)
-            .join('\n');
-        }
         // Validate that there are no duplicate card keys
         for (const [key, count] of cardIds) {
           if (count > 1) {
-            validationErrors += `Duplicate card key '${key}' found ${count} times\n`;
+            errorMsg.push(`Duplicate card key '${key}' found ${count} times`);
           }
         }
+        if (errorMsg.length) {
+          validationErrors += errorMsg
+            .filter(this.removeDuplicateEntries)
+            .join('\n');
+        }
       }
     } catch (error) {
       validationErrors += errorFunction(error);

package/src/containers/project/calculation-engine.ts

@@ -13,7 +13,7 @@
 
 // node
 import { basename, join, resolve } from 'node:path';
-import { readFile } from 'node:fs/promises';
+import { readFile, writeFile } from 'node:fs/promises';
 
 import { sanitizeSvgBase64 } from '../../utils/sanitize-svg.js';
 import { instance } from '@viz-js/viz';
@@ -57,6 +57,7 @@ import {
   solve,
   setProgram,
   removeProgram,
+  buildProgram,
 } from '@cyberismo/node-clingo';
 import { generateReportContent } from '../../utils/report.js';
 import { lpFiles, graphvizReport } from '@cyberismo/assets';
@@ -110,6 +111,22 @@ export class CalculationEngine {
     return createCardFacts(card, this.project);
   }
 
+  /**
+   * Exports logic program to a given file
+   * @param destination Destination file path
+   * @param programs Programs or categories to export
+   * @param query Query to export, if not provided, all programs will be exported
+   */
+  public async exportLogicProgram(
+    destination: string,
+    programs: string[],
+    query?: QueryName,
+  ) {
+    let logicProgram = query ? this.queryContent(query) : '';
+    logicProgram += await buildProgram('', programs);
+    await writeFile(destination, logicProgram);
+  }
+
   // // Wrapper to run onCreation query.
   private async creationQuery(cardKeys: string[], context: Context) {
     if (!cardKeys) return undefined;
@@ -503,6 +520,13 @@ export class CalculationEngine {
     return this.parseClingoResult(clingoOutput);
   }
 
+  private queryContent(queryName: QueryName, options?: unknown) {
+    const content = lpFiles.queries[queryName];
+    const handlebars = Handlebars.create();
+    const compiled = handlebars.compile(content);
+    return compiled(options || {});
+  }
+
   /**
    * Runs a pre-defined query.
    * @param queryName Name of the query file without extension
@@ -514,13 +538,7 @@ export class CalculationEngine {
     context: Context = 'localApp',
     options?: unknown,
   ): Promise<QueryResult<T>[]> {
-    let content = lpFiles.queries[queryName];
-    const handlebars = Handlebars.create();
-    const compiled = handlebars.compile(content);
-    content = compiled(options || {});
-    if (!content) {
-      throw new Error(`Query file ${queryName} not found`);
-    }
+    const content = this.queryContent(queryName, options);
 
     this.logger.trace({ queryName }, 'Running query');
     const clingoOutput = await this.run(content, context);