@cyberismo/data-handler 0.0.16 → 0.0.17

package/src/commands/fetch.ts

@@ -13,16 +13,28 @@
 
 import { mkdir } from 'node:fs/promises';
 import { resolve, sep } from 'node:path';
-import type { Project } from '../containers/project.js';
 
-import { writeJsonFile } from '../utils/json.js';
-import { validateJson } from '../utils/validate.js';
-import { type ModuleSetting } from '../interfaces/project-interfaces.js';
-import { errorFunction } from '../utils/error-utils.js';
 import { getChildLogger } from '../utils/log-utils.js';
+import { readJsonFile, writeJsonFile } from '../utils/json.js';
+import { validateJson } from '../utils/validate.js';
+
+import type { ModuleSetting } from '../interfaces/project-interfaces.js';
+import type { Project } from '../containers/project.js';
 
-const FETCH_TIMEOUT = 30000; // 30s timeout for fetching a hub file.
-const MAX_RESPONSE_SIZE = 1024 * 1024; // 1MB limit for safety
+// Hub structure
+interface HubVersionInfo {
+  location: string;
+  version: number;
+}
+
+// Structure of .temp/moduleList.json file.
+interface ModuleListFile {
+  modules: ModuleSetting[];
+  hubs: HubVersionInfo[];
+}
+
+const FETCH_TIMEOUT_MS = 30 * 1000; // 30s timeout for fetching a hub file.
+const MAX_RESPONSE_SIZE_MB = 1024 * 1024; // 1MB limit for safety
 const HUB_SCHEMA = 'hubSchema';
 const MODULE_LIST_FILE = 'moduleList.json';
 const TEMP_FOLDER = `.temp`;
@@ -30,7 +42,10 @@ const TEMP_FOLDER = `.temp`;
 export const MODULE_LIST_FULL_PATH = `${TEMP_FOLDER}/${MODULE_LIST_FILE}`;
 
 export class Fetch {
-  constructor(private project: Project) {}
+  private moduleListPath;
+  constructor(private project: Project) {
+    this.moduleListPath = resolve(this.project.basePath, MODULE_LIST_FULL_PATH);
+  }
 
   private get logger() {
     return getChildLogger({
@@ -38,6 +53,38 @@ export class Fetch {
     });
   }
 
+  // Checks the version of the remote moduleList.json.
+  private async checkRemoteVersion(
+    location: string,
+  ): Promise<number | undefined> {
+    try {
+      const url = new URL(`${location}/${MODULE_LIST_FILE}`);
+      if (!['http:', 'https:'].includes(url.protocol)) {
+        return undefined;
+      }
+
+      const response = await fetch(url.toString(), {
+        method: 'GET',
+        headers: {
+          Accept: 'application/json',
+          'User-Agent': 'Cyberismo/1.0',
+        },
+        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      });
+
+      if (!response.ok) {
+        return undefined;
+      }
+
+      const json = await response.json();
+      return json.version;
+    } catch (error) {
+      this.logger.error(error, `Could not check hub version for ${location} }`);
+      return undefined;
+    }
+  }
+
+  // Fetches one hub's data as JSON.
   private async fetchJSON(location: string, schemaId: string) {
     try {
       const url = new URL(`${location}/${MODULE_LIST_FILE}`);
@@ -47,14 +94,14 @@ export class Fetch {
         );
       }
 
-      this.logger.info(`Fetching module list from: ${url.toString()}`);
+      this.logger.info(`Fetching module list from hub: ${url.toString()}`);
       const response = await fetch(url.toString(), {
         method: 'GET',
         headers: {
           Accept: 'application/json',
           'User-Agent': 'Cyberismo/1.0',
         },
-        signal: AbortSignal.timeout(FETCH_TIMEOUT),
+        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
       });
 
       if (!response.ok) {
@@ -65,9 +112,9 @@ export class Fetch {
 
       // Check content length before downloading
       const contentLength = response.headers.get('content-length');
-      if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+      if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE_MB) {
         throw new Error(
-          `Response too large: ${contentLength} bytes (max: ${MAX_RESPONSE_SIZE})`,
+          `Response too large: ${contentLength} bytes (max: ${MAX_RESPONSE_SIZE_MB})`,
         );
       }
 
@@ -79,14 +126,10 @@ export class Fetch {
       const json = await response.json();
       // Validate the incoming JSON before saving it into a file.
       await validateJson(json, { schemaId: schemaId });
-
-      // Validate JSON structure and prevent prototype pollution
       if (typeof json !== 'object' || json === null || Array.isArray(json)) {
         throw new Error('Response must be a JSON object');
       }
-
-      // Additional size check after JSON parsing
-      if (JSON.stringify(json).length > MAX_RESPONSE_SIZE) {
+      if (JSON.stringify(json).length > MAX_RESPONSE_SIZE_MB) {
         throw new Error('JSON content too large after parsing');
       }
 
@@ -94,41 +137,109 @@ export class Fetch {
     } catch (error) {
       this.logger.error(
         error,
-        `Failed to fetch module list from ${location}: ${errorFunction(error)}`,
+        `Failed to fetch module list from hub ${location}`,
       );
       throw error;
     }
   }
 
+  // Checks if the local moduleList.json needs to be updated by comparing
+  // each hub's version with the stored version.
+  private async fetchModuleList(): Promise<boolean> {
+    try {
+      const configuredHubs = this.project.configuration.hubs;
+      if (configuredHubs.length === 0) {
+        return false;
+      }
+
+      const localData = (await readJsonFile(
+        this.moduleListPath,
+      )) as ModuleListFile;
+      const localHubs = localData.hubs || [];
+      if (localHubs.length !== configuredHubs.length) {
+        this.logger.info('Hub configuration changed, fetching module list');
+        return true;
+      }
+
+      // Check each hub's version
+      for (const configHub of configuredHubs) {
+        const localHub = localHubs.find(
+          (hub) => hub.location === configHub.location,
+        );
+
+        if (!localHub) {
+          this.logger.info(
+            `New hub detected: ${configHub.location}, fetching module list`,
+          );
+          return true;
+        }
+
+        const remoteVersion = await this.checkRemoteVersion(configHub.location);
+        if (remoteVersion === undefined) {
+          const hubName = configHub.displayName || configHub.location;
+          this.logger.info(`Hub ${hubName} has no version data, skipped.`);
+          continue;
+        }
+
+        if (remoteVersion > localHub.version) {
+          this.logger.info(
+            `Hub ${configHub.location} has newer version (remote: ${remoteVersion}, local: ${localHub.version}), fetching module list`,
+          );
+          return true;
+        }
+      }
+
+      this.logger.info('Module list is up to date');
+      return false;
+    } catch (error) {
+      this.logger.error(
+        error,
+        `Error when checking versions for hub module list`,
+      );
+      return true;
+    }
+  }
+
+  /**
+   * Ensures the module list is up to date by fetching if needed.
+   */
+  public async ensureModuleListUpToDate() {
+    await this.fetchHubs();
+  }
+
   /**
    * Fetches modules from modules hub(s) and writes them to a file.
+   * Only fetches if the remote version is newer than the local version.
    */
   public async fetchHubs() {
-    const hubs = this.project.configuration.hubs;
+    const needsFetch = await this.fetchModuleList();
+    if (!needsFetch) {
+      return;
+    }
 
+    const hubs = this.project.configuration.hubs;
     const moduleMap: Map<string, ModuleSetting> = new Map([]);
+    const hubVersions: HubVersionInfo[] = [];
 
     for (const hub of hubs) {
       const json = await this.fetchJSON(hub.location, HUB_SCHEMA);
       json.modules.forEach((module: ModuleSetting) => {
         if (!moduleMap.has(module.name)) {
           moduleMap.set(module.name, module);
-        } else {
-          this.logger.info(
-            `Skipping module '${module.name}' since it was already listed.`,
-          );
         }
       });
+
+      hubVersions.push({
+        location: hub.location,
+        version: json.version || 1,
+      });
     }
 
     try {
-      const fullPath = resolve(this.project.basePath, MODULE_LIST_FULL_PATH);
       const normalizedBasePath = resolve(this.project.basePath);
-
-      // Ensure the file is written within the project directory (prevent path traversal)
       if (
-        !fullPath.startsWith(normalizedBasePath + sep) &&
-        fullPath !== normalizedBasePath
+        !this.moduleListPath.startsWith(normalizedBasePath + sep) &&
+        this.moduleListPath !== normalizedBasePath
       ) {
         throw new Error(
           'Invalid file path: attempting to write outside project directory',
@@ -138,15 +249,13 @@ export class Fetch {
       await mkdir(resolve(this.project.basePath, TEMP_FOLDER), {
         recursive: true,
       });
-      await writeJsonFile(fullPath, {
+      await writeJsonFile(this.moduleListPath, {
         modules: Array.from(moduleMap.values()),
+        hubs: hubVersions,
       });
-      this.logger.info(`Module list written to: ${fullPath}`);
+      this.logger.info(`Module list written to: ${this.moduleListPath}`);
     } catch (error) {
-      this.logger.error(
-        error,
-        `Failed to write module list to local file: ${errorFunction(error)}`,
-      );
+      this.logger.error(error, `Failed to write module list to local file`);
       throw error;
     }
   }

package/src/commands/import.ts

@@ -20,6 +20,7 @@ import type {
   Credentials,
   ModuleSettingOptions,
 } from '../interfaces/project-interfaces.js';
+import type { Fetch } from './fetch.js';
 import type { Project } from '../containers/project.js';
 
 /**
@@ -36,6 +37,7 @@ export class Import {
   constructor(
     private project: Project,
     private createCmd: Create,
+    private fetchCmd: Fetch,
   ) {
     this.moduleManager = new ModuleManager(this.project);
   }
@@ -133,12 +135,17 @@ export class Import {
    * @param options Additional options for module import. Optional.
    *        branch: Git branch for module from Git.
    *        private: If true, uses credentials to clone the repository
+   * @param skipMigrationLog If true, skip logging to migration log (used during project creation)
    */
   public async importModule(
     source: string,
     destination?: string,
     options?: ModuleSettingOptions,
+    skipMigrationLog = false,
   ) {
+    // Ensure module list is up to date before importing
+    await this.fetchCmd.ensureModuleListUpToDate();
+
     const beforeImportValidateErrors = await Validate.getInstance().validate(
       this.project.basePath,
       () => this.project,
@@ -168,7 +175,7 @@ export class Import {
     );
 
     // Add module as a dependency.
-    await this.project.importModule(moduleSettings);
+    await this.project.importModule(moduleSettings, skipMigrationLog);
 
     // Validate the project after module has been imported
     const afterImportValidateErrors = await Validate.getInstance().validate(
@@ -189,6 +196,9 @@ export class Import {
    * @throws if module is not part of the project
    */
   public async updateModule(moduleName: string, credentials?: Credentials) {
+    // Ensure module list is up to date before updating
+    await this.fetchCmd.ensureModuleListUpToDate();
+
     const module = this.project.configuration.modules.find(
       (item) => item.name === moduleName,
     );
@@ -203,6 +213,8 @@ export class Import {
    * @param credentials Optional credentials for private modules.
    */
   public async updateAllModules(credentials?: Credentials) {
+    // Ensure module list is up to date before updating all modules
+    await this.fetchCmd.ensureModuleListUpToDate();
     return this.moduleManager.updateModules(credentials);
   }
 }

package/src/commands/remove.ts

@@ -14,6 +14,7 @@
 import { ActionGuard } from '../permissions/action-guard.js';
 import { isModuleCard } from '../utils/card-utils.js';
 import { ModuleManager } from '../module-manager.js';
+import type { Fetch } from './fetch.js';
 import type { Project } from '../containers/project.js';
 import type { RemovableResourceTypes } from '../interfaces/project-interfaces.js';
 
@@ -26,7 +27,10 @@ export class Remove {
    * Creates a new instance of Remove command.
    * @param project Project instance to use
    */
-  constructor(private project: Project) {
+  constructor(
+    private project: Project,
+    private fetchCmd: Fetch,
+  ) {
     this.moduleManager = new ModuleManager(this.project);
   }
 
@@ -164,6 +168,11 @@ export class Remove {
     targetName: string,
     ...rest: any[] // eslint-disable-line @typescript-eslint/no-explicit-any
   ) {
+    // Ensure module list is up to date when removing modules
+    if (type === 'module') {
+      await this.fetchCmd.ensureModuleListUpToDate();
+    }
+
     if (type === 'attachment' && rest.length !== 1 && !rest[0]) {
       throw new Error(
         `Input validation error: must pass argument 'detail' if requesting to remove attachment`,

package/src/commands/rename.ts

@@ -17,6 +17,10 @@ import { join } from 'node:path';
 import { rename, readdir, readFile, writeFile } from 'node:fs/promises';
 
 import type { Card } from '../interfaces/project-interfaces.js';
+import {
+  ConfigurationLogger,
+  ConfigurationOperation,
+} from '../utils/configuration-logger.js';
 import { isTemplateCard } from '../utils/card-utils.js';
 import { type Project, ResourcesFrom } from '../containers/project.js';
 import { resourceName } from '../utils/resource-utils.js';
@@ -273,6 +277,17 @@ export class Rename {
     this.project.resources.changed();
     console.info('Collected renamed resources');
 
+    // Remove these when operations properly update card cache
+    this.project.cardsCache.clear();
+    await this.project.populateCaches();
+
+    await ConfigurationLogger.log(
+      this.project.basePath,
+      ConfigurationOperation.PROJECT_RENAME,
+      this.to,
+      {},
+    );
+
     return this.project.calculationEngine.generate();
   }
 }

package/src/commands/show.ts

@@ -18,7 +18,7 @@ import { join, resolve } from 'node:path';
 import { spawn } from 'node:child_process';
 import { writeFile } from 'node:fs/promises';
 
-import { MODULE_LIST_FULL_PATH } from './fetch.js';
+import { type Fetch, MODULE_LIST_FULL_PATH } from './fetch.js';
 
 import type { attachmentPayload } from '../interfaces/request-status-interfaces.js';
 import type {
@@ -74,7 +74,10 @@ export class Show {
     workflows: (from) => this.resourceNames('workflows', from),
   };
 
-  constructor(private project: Project) {}
+  constructor(
+    private project: Project,
+    private fetchCmd: Fetch,
+  ) {}
 
   private get logger() {
     return getChildLogger({
@@ -326,6 +329,9 @@ export class Show {
     showDetails?: boolean,
   ): Promise<ModuleSettingFromHub[]> {
     try {
+      // Ensure module list is up to date before showing
+      await this.fetchCmd.ensureModuleListUpToDate();
+
       const moduleList = (
         await readJsonFile(
           resolve(this.project.basePath, MODULE_LIST_FULL_PATH),
@@ -399,7 +405,9 @@ export class Show {
    * Shows hubs of the project.
    * @returns list of hubs.
    */
-  public showHubs(): HubSetting[] {
+  public async showHubs(): Promise<HubSetting[]> {
+    // Ensure module list is up to date before showing
+    await this.fetchCmd.ensureModuleListUpToDate();
     return this.project.configuration.hubs;
   }
 

package/src/commands/validate.ts

@@ -222,7 +222,7 @@ export class Validate {
   ): Promise<string[]> {
     const message: string[] = [];
     try {
-      const prefixes = project.projectPrefixes();
+      const prefixes = project.allModulePrefixes();
       const files = await readdir(path, {
         withFileTypes: true,
       });
@@ -554,7 +554,7 @@ export class Validate {
         cards.push(...project.allTemplateCards());
 
         const cardIds = new Map<string, number>();
-        const allPrefixes = await project.projectPrefixes();
+        const allPrefixes = project.allModulePrefixes();
 
         for (const card of cards) {
           if (cardIds.has(card.key)) {
@@ -819,11 +819,7 @@ export class Validate {
 
     // Validate that all metadata keys are either predefined fields or valid field type names
     for (const key of Object.keys(card.metadata)) {
-      if (
-        (isPredefinedField(key) as boolean) ||
-        key === 'labels' ||
-        key === 'links'
-      ) {
+      if (isPredefinedField(key) as boolean) {
         continue;
       }
       try {

package/src/containers/card-container.ts

@@ -19,17 +19,19 @@ import { writeFile } from 'node:fs/promises';
 import { CardCache } from './project/card-cache.js';
 import { cardPathParts } from '../utils/card-utils.js';
 import { deleteDir } from '../utils/file-utils.js';
+import { getChildLogger } from '../utils/log-utils.js';
 import { writeJsonFile } from '../utils/json.js';
 
 import type {
   CardAttachment,
   Card,
+  CardMetadata,
   FetchCardDetails,
 } from '../interfaces/project-interfaces.js';
 
 import asciidoctor from '@asciidoctor/core';
 
-import { ROOT } from '../utils/constants.js';
+import { isPredefinedField, ROOT } from '../utils/constants.js';
 
 /**
  * Card container base class. Used for both Project and Template.
@@ -38,17 +40,19 @@ import { ROOT } from '../utils/constants.js';
 export class CardContainer {
   public basePath: string;
   protected cardCache: CardCache;
-  protected containerName: string;
   protected prefix: string;
 
+  protected static get logger() {
+    return getChildLogger({ module: 'CardContainer' });
+  }
+
   static cardContentFile = 'index.adoc';
   static cardMetadataFile = 'index.json';
   static projectConfigFileName = 'cardsConfig.json';
   static schemaContentFile = '.schema';
 
-  constructor(path: string, prefix: string, name: string) {
+  constructor(path: string, prefix: string) {
     this.basePath = path;
-    this.containerName = name;
     this.prefix = prefix;
     this.cardCache = new CardCache(this.prefix);
   }
@@ -226,13 +230,41 @@ export class CardContainer {
     if (card.metadata != null) {
       const metadataFile = join(card.path, CardContainer.cardMetadataFile);
       card.metadata!.lastUpdated = new Date().toISOString();
-      await writeJsonFile(metadataFile, card.metadata);
+
+      const sanitizedMetadata = CardContainer.sanitizeMetadata(card);
+      await writeJsonFile(metadataFile, sanitizedMetadata);
       return this.cardCache.updateCardMetadata(card.key, card.metadata);
     }
     return false;
   }
 
   /**
+   * Removes non-metadata fields that should not be persisted.
+   *
+   * @param metadata The metadata object to sanitize
+   * @returns Clean metadata object with only valid metadata fields
+   */
+  private static sanitizeMetadata(card: Card): CardMetadata {
+    const sanitized: Record<string, unknown> = {};
+
+    if (card.metadata) {
+      for (const [key, value] of Object.entries(card.metadata)) {
+        // Keys are not filtered out if they are: predefined, or field types
+        if (isPredefinedField(key) || key.includes('/')) {
+          sanitized[key] = value;
+        } else {
+          this.logger.warn(
+            `Card ${card.key} had extra metadata key ${key} with value ${value}. Key was removed`,
+          );
+        }
+        // Everything else is filtered out
+      }
+    }
+
+    return sanitized as CardMetadata;
+  }
+
+  /*
    * Show root cards from a given path.
    * @param path The path to get cards from
    * @returns an array of root-level cards (each with their children populated).

package/src/containers/project/project-paths.ts

@@ -64,6 +64,10 @@ export class ProjectPaths {
     return join(this.resourcesFolder, 'cardsConfig.json');
   }
 
+  public get configurationChangesLog(): string {
+    return join(this.migrationLogFolder, 'current', 'migrationLog.jsonl');
+  }
+
   public get fieldTypesFolder(): string {
     return join(this.resourcesFolder, 'fieldTypes');
   }
@@ -84,6 +88,10 @@ export class ProjectPaths {
     return join(this.path, '.logs', 'cyberismo_data-handler.log');
   }
 
+  public get migrationLogFolder(): string {
+    return join(this.resourcesFolder, 'migrations');
+  }
+
   public get modulesFolder(): string {
     return join(this.path, '.cards', 'modules');
   }

package/src/containers/project/resource-cache.ts

@@ -194,15 +194,16 @@ export class ResourceCache {
   }
 
   // Collect all module resources from the filesystem
-  // Only collects modules that are registered in the project configuration
-  // todo: For future:
-  // Should it also try to collect what is in .local/modules and then log for disparities?
   private collectModuleResources() {
     try {
-      const registeredModules = this.project.configuration.modules.map(
-        (m) => m.name,
-      );
-      if (registeredModules.length === 0) {
+      const moduleEntries = readdirSync(this.project.paths.modulesFolder, {
+        withFileTypes: true,
+      });
+      const moduleNames = moduleEntries
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => entry.name);
+
+      if (moduleNames.length === 0) {
         return;
       }
 
@@ -218,13 +219,15 @@ export class ResourceCache {
         'workflows',
       ];
 
-      for (const moduleName of registeredModules) {
+      for (const moduleName of moduleNames) {
         for (const type of resourceTypes) {
           this.collectResourcesOfType(type, 'module', moduleName);
         }
       }
     } catch {
-      ResourceCache.logger.warn(`.cards/modules folder is missing`);
+      ResourceCache.logger.debug(
+        `.cards/modules folder is missing or inaccessible`,
+      );
     }
   }