@cyberismo/data-handler 0.0.15 → 0.0.17

package/src/commands/create.ts

@@ -15,31 +15,18 @@
 import { join, resolve } from 'node:path';
 import { mkdir, writeFile } from 'node:fs/promises';
 
+import { SCHEMA_VERSION } from '@cyberismo/assets';
 import { errorFunction } from '../utils/error-utils.js';
 import { Project } from '../containers/project.js';
-import { Validate } from './index.js';
+import { Validate } from './validate.js';
 
 import { EMPTY_RANK, sortItems } from '../utils/lexorank.js';
 import { isModulePath } from '../utils/card-utils.js';
-import type {
-  DataType,
-  Link,
-  LinkType,
-} from '../interfaces/resource-interfaces.js';
+import type { DataType } from '../interfaces/resource-interfaces.js';
 import type { Card, ProjectFile } from '../interfaces/project-interfaces.js';
 import { resourceName, resourceNameToString } from '../utils/resource-utils.js';
 import { writeJsonFile } from '../utils/json.js';
 
-import { CalculationResource } from '../resources/calculation-resource.js';
-import { CardTypeResource } from '../resources/card-type-resource.js';
-import { FieldTypeResource } from '../resources/field-type-resource.js';
-import { GraphModelResource } from '../resources/graph-model-resource.js';
-import { GraphViewResource } from '../resources/graph-view-resource.js';
-import { LinkTypeResource } from '../resources/link-type-resource.js';
-import { ReportResource } from '../resources/report-resource.js';
-import { TemplateResource } from '../resources/template-resource.js';
-import { WorkflowResource } from '../resources/workflow-resource.js';
-
 // todo: Is there a easy to way to make JSON schema into a TypeScript interface/type?
 //       Check this out: https://www.npmjs.com/package/json-schema-to-ts
 
@@ -58,6 +45,7 @@ export class Create {
     {
       path: '.cards/local',
       content: {
+        schemaVersion: SCHEMA_VERSION,
         cardKeyPrefix: '$PROJECT-PREFIX',
         name: '$PROJECT-NAME',
         modules: [],
@@ -107,9 +95,9 @@ export class Create {
     if (cardTypeName === undefined) {
       throw new Error(`Input validation error: card type cannot be empty`);
     }
-    const templateResource = new TemplateResource(
-      this.project,
-      resourceName(templateName),
+    const templateResource = this.project.resources.byType(
+      templateName,
+      'templates',
     );
     const templateObject = templateResource.templateObject();
     const specificCard = card ? templateObject.findCard(card) : undefined;
@@ -181,11 +169,9 @@ export class Create {
    * @param calculationName name for the calculation resource
    */
   public async createCalculation(calculationName: string) {
-    const calculation = new CalculationResource(
-      this.project,
-      resourceName(calculationName),
-    );
-    await calculation.create();
+    return this.project.resources
+      .byType(calculationName, 'calculations')
+      .create();
   }
 
   /**
@@ -198,15 +184,15 @@ export class Create {
     templateName: string,
     parentCardKey?: string,
   ): Promise<Card[]> {
-    const templateResource = new TemplateResource(
-      this.project,
-      resourceName(templateName),
+    const templateResource = this.project.resources.byType(
+      templateName,
+      'templates',
     );
 
     Validate.getInstance().validResourceName(
       'templates',
       resourceNameToString(resourceName(templateName)),
-      await this.project.projectPrefixes(),
+      this.project.allModulePrefixes(),
     );
 
     await templateResource.validate();
@@ -236,12 +222,9 @@ export class Create {
    * @param workflowName workflow name to use in the card type.
    */
   public async createCardType(cardTypeName: string, workflowName: string) {
-    const cardType = new CardTypeResource(
-      this.project,
-      resourceName(cardTypeName),
-    );
-
-    await cardType.createCardType(workflowName);
+    return this.project.resources
+      .byType(cardTypeName, 'cardTypes')
+      .createCardType(workflowName);
   }
 
   /**
@@ -250,11 +233,9 @@ export class Create {
    * @param dataType data type for the field type
    */
   public async createFieldType(fieldTypeName: string, dataType: DataType) {
-    const fieldType = new FieldTypeResource(
-      this.project,
-      resourceName(fieldTypeName),
-    );
-    await fieldType.createFieldType(dataType);
+    return this.project.resources
+      .byType(fieldTypeName, 'fieldTypes')
+      .createFieldType(dataType);
   }
 
   /**
@@ -262,23 +243,17 @@ export class Create {
    * @param graphModelName name for the graph model.
    */
   public async createGraphModel(graphModelName: string) {
-    const graphModel = new GraphModelResource(
-      this.project,
-      resourceName(graphModelName),
-    );
-    await graphModel.create();
+    return this.project.resources
+      .byType(graphModelName, 'graphModels')
+      .create();
   }
 
   /**
    * Creates a new graph view.
-   * @param graphModelName name for the graph view.
+   * @param graphViewName name for the graph view.
    */
   public async createGraphView(graphViewName: string) {
-    const graphView = new GraphViewResource(
-      this.project,
-      resourceName(graphViewName),
-    );
-    await graphView.create();
+    return this.project.resources.byType(graphViewName, 'graphViews').create();
   }
 
   /**
@@ -307,11 +282,7 @@ export class Create {
    * @param linkTypeName name for the link type.
    */
   public async createLinkType(linkTypeName: string) {
-    const linkType = new LinkTypeResource(
-      this.project,
-      resourceName(linkTypeName),
-    );
-    await linkType.create();
+    return this.project.resources.byType(linkTypeName, 'linkTypes').create();
   }
 
   /**
@@ -335,10 +306,9 @@ export class Create {
     const card = this.project.findCard(cardKey);
     const destinationCard = this.project.findCard(destinationCardKey);
     // make sure the link type exists
-    const linkTypeObject = this.project.resource<LinkType>(linkType);
-    if (!linkTypeObject) {
-      throw new Error(`Link type '${linkType}' does not exist in the project`);
-    }
+    const linkTypeObject = this.project.resources
+      .byType(linkType, 'linkTypes')
+      .show();
 
     // make sure that if linkDescription is not enabled, linkDescription is not provided
     if (
@@ -387,7 +357,7 @@ export class Create {
       );
     }
 
-    const links: Link[] = card.metadata?.links || [];
+    const links = card.metadata?.links || [];
     links.push({
       linkType,
       cardKey: destinationCardKey,
@@ -416,6 +386,12 @@ export class Create {
 
     const projectFolders: string[] = ['.cards/local', 'cardRoot'];
 
+    if (!Validate.validateFolder(projectPath)) {
+      throw new Error(
+        `Input validation error: folder name '${projectPath}' is invalid`,
+      );
+    }
+
     if (
       projectPrefix === undefined ||
       projectPrefix.length < 3 ||
@@ -486,8 +462,7 @@ export class Create {
    * @param name name of the report
    */
   public async createReport(name: string) {
-    const report = new ReportResource(this.project, resourceName(name));
-    await report.createReport();
+    return this.project.resources.byType(name, 'reports').createReport();
   }
 
   /**
@@ -496,14 +471,9 @@ export class Create {
    * @param templateContent JSON content for the template file.
    */
   public async createTemplate(templateName: string, templateContent: string) {
-    const template = new TemplateResource(
-      this.project,
-      resourceName(templateName),
-    );
-
-    await template.create(
-      templateContent ? JSON.parse(templateContent) : undefined,
-    );
+    return this.project.resources
+      .byType(templateName, 'templates')
+      .create(templateContent ? JSON.parse(templateContent) : undefined);
   }
 
   /**
@@ -512,13 +482,8 @@ export class Create {
    * @param workflowContent workflow content JSON
    */
   public async createWorkflow(workflowName: string, workflowContent: string) {
-    const workflow = new WorkflowResource(
-      this.project,
-      resourceName(workflowName),
-    );
-
-    await workflow.create(
-      workflowContent ? JSON.parse(workflowContent) : undefined,
-    );
+    return this.project.resources
+      .byType(workflowName, 'workflows')
+      .create(workflowContent ? JSON.parse(workflowContent) : undefined);
   }
 }

package/src/commands/export.ts

@@ -31,12 +31,20 @@ import { generateReportContent } from '../utils/report.js';
 import { getStaticDirectoryPath, pdfReport } from '@cyberismo/assets';
 import { Project } from '../containers/project.js';
 import type { QueryResult } from '../types/queries.js';
-import type { Show } from './index.js';
+import type { Show } from './show.js';
 import { sortItems } from '../utils/lexorank.js';
 
 const attachmentFolder: string = 'a';
 
+/**
+ * Handles all export commands.
+ */
 export class Export {
+  /**
+   * Creates an instance of export.
+   * @param project Project to use
+   * @param showCmd Instance of Export command to use.
+   */
   constructor(
     protected project: Project,
     protected showCmd: Show,
@@ -97,6 +105,56 @@ export class Export {
     return content;
   }
 
+  // Runs Ascii Doctor converter --> to PDF
+  private async runAsciidoctorPdf(content: string): Promise<Buffer> {
+    const staticRootDir = await getStaticDirectoryPath();
+    const proc = spawn(
+      'asciidoctor-pdf',
+      [
+        '-a',
+        'pdf-theme=cyberismo',
+        '-a',
+        `pdf-themesdir=${join(staticRootDir, 'pdf-themes')}`,
+        '-a',
+        `pdf-fontsdir=${join(staticRootDir, 'pdf-themes', 'fonts')};GEM_FONTS_DIR`,
+        '-',
+      ],
+      {
+        timeout: 100000,
+        shell: process.platform === 'win32',
+      },
+    );
+    proc.stdin.end(content);
+    const result = await new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = [];
+      proc.stdout.on('data', (chunk) => {
+        chunks.push(chunk);
+      });
+      proc.stderr.on('data', (chunk) => {
+        process.stderr.write(chunk);
+      });
+      proc.on('error', (error) => {
+        if ('code' in error && error.code === 'ENOENT') {
+          reject(
+            new Error(
+              'Asciidoctor-pdf not found. Please install asciidoctor-pdf to use this feature.',
+            ),
+          );
+        }
+        reject(error);
+      });
+      proc.on('close', (code) => {
+        if (code === 0) {
+          resolve(Buffer.concat(chunks));
+        } else {
+          reject(new Error(`Asciidoctor-pdf failed with code ${code}`));
+        }
+      });
+    });
+    return result;
+  }
+
+  // Adds cards to an ADOC file as additional content.
   private async toAdocFileAsContent(path: string, cards: Card[]) {
     for (const card of cards) {
       let fileContent = '';
@@ -108,9 +166,9 @@ export class Export {
       }
 
       if (card.metadata) {
-        const cardTypeForCard = await this.project.resource<CardType>(
-          card.metadata?.cardType,
-        );
+        const cardTypeForCard = this.project.resources
+          .byType(card.metadata?.cardType, 'cardTypes')
+          .show();
         const metaDataContent = this.metaToAdoc(card, cardTypeForCard);
         fileContent += metaDataContent;
       }
@@ -177,6 +235,7 @@ export class Export {
    * Convert treeQueryResult object into a Card object and add content, metadata & attachments
    * Handles card children recursively
    * @param treeQueryResult tree query result object
+   * @returns Tree query result as a Card.
    */
   protected async treeQueryResultToCard(
     treeQueryResult: QueryResult<'tree'>,
@@ -218,54 +277,6 @@ export class Export {
     return card;
   }
 
-  private async runAsciidoctorPdf(content: string): Promise<Buffer> {
-    const staticRootDir = await getStaticDirectoryPath();
-    const proc = spawn(
-      'asciidoctor-pdf',
-      [
-        '-a',
-        'pdf-theme=cyberismo',
-        '-a',
-        `pdf-themesdir=${join(staticRootDir, 'pdf-themes')}`,
-        '-a',
-        `pdf-fontsdir=${join(staticRootDir, 'pdf-themes', 'fonts')};GEM_FONTS_DIR`,
-        '-',
-      ],
-      {
-        timeout: 100000,
-        shell: process.platform === 'win32',
-      },
-    );
-    proc.stdin.end(content);
-    const result = await new Promise<Buffer>((resolve, reject) => {
-      const chunks: Buffer[] = [];
-      proc.stdout.on('data', (chunk) => {
-        chunks.push(chunk);
-      });
-      proc.stderr.on('data', (chunk) => {
-        process.stderr.write(chunk);
-      });
-      proc.on('error', (error) => {
-        if ('code' in error && error.code === 'ENOENT') {
-          reject(
-            new Error(
-              'Asciidoctor-pdf not found. Please install asciidoctor-pdf to use this feature.',
-            ),
-          );
-        }
-        reject(error);
-      });
-      proc.on('close', (code) => {
-        if (code === 0) {
-          resolve(Buffer.concat(chunks));
-        } else {
-          reject(new Error(`Asciidoctor-pdf failed with code ${code}`));
-        }
-      });
-    });
-    return result;
-  }
-
   /**
    * Exports the card(s) to pdf.
    * @param destination Path to where the resulting file(s) will be created.

package/src/commands/fetch.ts

@@ -13,16 +13,28 @@
 
 import { mkdir } from 'node:fs/promises';
 import { resolve, sep } from 'node:path';
-import type { Project } from '../containers/project.js';
 
-import { writeJsonFile } from '../utils/json.js';
-import { validateJson } from '../utils/validate.js';
-import { type ModuleSetting } from '../interfaces/project-interfaces.js';
-import { errorFunction } from '../utils/error-utils.js';
 import { getChildLogger } from '../utils/log-utils.js';
+import { readJsonFile, writeJsonFile } from '../utils/json.js';
+import { validateJson } from '../utils/validate.js';
+
+import type { ModuleSetting } from '../interfaces/project-interfaces.js';
+import type { Project } from '../containers/project.js';
 
-const FETCH_TIMEOUT = 30000; // 30s timeout for fetching a hub file.
-const MAX_RESPONSE_SIZE = 1024 * 1024; // 1MB limit for safety
+// Hub structure
+interface HubVersionInfo {
+  location: string;
+  version: number;
+}
+
+// Structure of .temp/moduleList.json file.
+interface ModuleListFile {
+  modules: ModuleSetting[];
+  hubs: HubVersionInfo[];
+}
+
+const FETCH_TIMEOUT_MS = 30 * 1000; // 30s timeout for fetching a hub file.
+const MAX_RESPONSE_SIZE_MB = 1024 * 1024; // 1MB limit for safety
 const HUB_SCHEMA = 'hubSchema';
 const MODULE_LIST_FILE = 'moduleList.json';
 const TEMP_FOLDER = `.temp`;
@@ -30,7 +42,10 @@ const TEMP_FOLDER = `.temp`;
 export const MODULE_LIST_FULL_PATH = `${TEMP_FOLDER}/${MODULE_LIST_FILE}`;
 
 export class Fetch {
-  constructor(private project: Project) {}
+  private moduleListPath;
+  constructor(private project: Project) {
+    this.moduleListPath = resolve(this.project.basePath, MODULE_LIST_FULL_PATH);
+  }
 
   private get logger() {
     return getChildLogger({
@@ -38,6 +53,38 @@ export class Fetch {
     });
   }
 
+  // Checks the version of the remote moduleList.json.
+  private async checkRemoteVersion(
+    location: string,
+  ): Promise<number | undefined> {
+    try {
+      const url = new URL(`${location}/${MODULE_LIST_FILE}`);
+      if (!['http:', 'https:'].includes(url.protocol)) {
+        return undefined;
+      }
+
+      const response = await fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          Accept: 'application/json',
+          'User-Agent': 'Cyberismo/1.0',
+        },
+        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      });
+
+      if (!response.ok) {
+        return undefined;
+      }
+
+      const json = await response.json();
+      return json.version;
+    } catch (error) {
+      this.logger.error(error, `Could not check hub version for ${location} }`);
+      return undefined;
+    }
+  }
+
+  // Fetches one hub's data as JSON.
   private async fetchJSON(location: string, schemaId: string) {
     try {
       const url = new URL(`${location}/${MODULE_LIST_FILE}`);
@@ -47,14 +94,14 @@ export class Fetch {
         );
       }
 
-      this.logger.info(`Fetching module list from: ${url.toString()}`);
+      this.logger.info(`Fetching module list from hub: ${url.toString()}`);
       const response = await fetch(url.toString(), {
         method: 'GET',
         headers: {
           Accept: 'application/json',
           'User-Agent': 'Cyberismo/1.0',
         },
-        signal: AbortSignal.timeout(FETCH_TIMEOUT),
+        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
       });
 
       if (!response.ok) {
@@ -65,9 +112,9 @@ export class Fetch {
 
       // Check content length before downloading
       const contentLength = response.headers.get('content-length');
-      if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+      if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE_MB) {
         throw new Error(
-          `Response too large: ${contentLength} bytes (max: ${MAX_RESPONSE_SIZE})`,
+          `Response too large: ${contentLength} bytes (max: ${MAX_RESPONSE_SIZE_MB})`,
         );
       }
 
@@ -79,14 +126,10 @@ export class Fetch {
       const json = await response.json();
       // Validate the incoming JSON before saving it into a file.
       await validateJson(json, { schemaId: schemaId });
-
-      // Validate JSON structure and prevent prototype pollution
       if (typeof json !== 'object' || json === null || Array.isArray(json)) {
         throw new Error('Response must be a JSON object');
       }
-
-      // Additional size check after JSON parsing
-      if (JSON.stringify(json).length > MAX_RESPONSE_SIZE) {
+      if (JSON.stringify(json).length > MAX_RESPONSE_SIZE_MB) {
         throw new Error('JSON content too large after parsing');
       }
 
@@ -94,41 +137,109 @@ export class Fetch {
     } catch (error) {
       this.logger.error(
         error,
-        `Failed to fetch module list from ${location}: ${errorFunction(error)}`,
+        `Failed to fetch module list from hub ${location}`,
       );
       throw error;
     }
   }
 
+  // Checks if the local moduleList.json needs to be updated by comparing
+  // each hub's version with the stored version.
+  private async fetchModuleList(): Promise<boolean> {
+    try {
+      const configuredHubs = this.project.configuration.hubs;
+      if (configuredHubs.length === 0) {
+        return false;
+      }
+
+      const localData = (await readJsonFile(
+        this.moduleListPath,
+      )) as ModuleListFile;
+      const localHubs = localData.hubs || [];
+      if (localHubs.length !== configuredHubs.length) {
+        this.logger.info('Hub configuration changed, fetching module list');
+        return true;
+      }
+
+      // Check each hub's version
+      for (const configHub of configuredHubs) {
+        const localHub = localHubs.find(
+          (hub) => hub.location === configHub.location,
+        );
+
+        if (!localHub) {
+          this.logger.info(
+            `New hub detected: ${configHub.location}, fetching module list`,
+          );
+          return true;
+        }
+
+        const remoteVersion = await this.checkRemoteVersion(configHub.location);
+        if (remoteVersion === undefined) {
+          const hubName = configHub.displayName || configHub.location;
+          this.logger.info(`Hub ${hubName} has no version data, skipped.`);
+          continue;
+        }
+
+        if (remoteVersion > localHub.version) {
+          this.logger.info(
+            `Hub ${configHub.location} has newer version (remote: ${remoteVersion}, local: ${localHub.version}), fetching module list`,
+          );
+          return true;
+        }
+      }
+
+      this.logger.info('Module list is up to date');
+      return false;
+    } catch (error) {
+      this.logger.error(
+        error,
+        `Error when checking versions for hub module list`,
+      );
+      return true;
+    }
+  }
+
+  /**
+   * Ensures the module list is up to date by fetching if needed.
+   */
+  public async ensureModuleListUpToDate() {
+    await this.fetchHubs();
+  }
+
   /**
    * Fetches modules from modules hub(s) and writes them to a file.
+   * Only fetches if the remote version is newer than the local version.
    */
   public async fetchHubs() {
-    const hubs = this.project.configuration.hubs;
+    const needsFetch = await this.fetchModuleList();
+    if (!needsFetch) {
+      return;
+    }
 
+    const hubs = this.project.configuration.hubs;
     const moduleMap: Map<string, ModuleSetting> = new Map([]);
+    const hubVersions: HubVersionInfo[] = [];
 
     for (const hub of hubs) {
       const json = await this.fetchJSON(hub.location, HUB_SCHEMA);
       json.modules.forEach((module: ModuleSetting) => {
         if (!moduleMap.has(module.name)) {
           moduleMap.set(module.name, module);
-        } else {
-          this.logger.info(
-            `Skipping module '${module.name}' since it was already listed.`,
-          );
         }
       });
+
+      hubVersions.push({
+        location: hub.location,
+        version: json.version || 1,
+      });
     }
 
     try {
-      const fullPath = resolve(this.project.basePath, MODULE_LIST_FULL_PATH);
       const normalizedBasePath = resolve(this.project.basePath);
-
-      // Ensure the file is written within the project directory (prevent path traversal)
       if (
-        !fullPath.startsWith(normalizedBasePath + sep) &&
-        fullPath !== normalizedBasePath
+        !this.moduleListPath.startsWith(normalizedBasePath + sep) &&
+        this.moduleListPath !== normalizedBasePath
       ) {
         throw new Error(
           'Invalid file path: attempting to write outside project directory',
@@ -138,15 +249,13 @@ export class Fetch {
       await mkdir(resolve(this.project.basePath, TEMP_FOLDER), {
         recursive: true,
       });
-      await writeJsonFile(fullPath, {
+      await writeJsonFile(this.moduleListPath, {
         modules: Array.from(moduleMap.values()),
+        hubs: hubVersions,
       });
-      this.logger.info(`Module list written to: ${fullPath}`);
+      this.logger.info(`Module list written to: ${this.moduleListPath}`);
     } catch (error) {
-      this.logger.error(
-        error,
-        `Failed to write module list to local file: ${errorFunction(error)}`,
-      );
+      this.logger.error(error, `Failed to write module list to local file`);
       throw error;
     }
   }