@cyberismo/data-handler 0.0.15 → 0.0.17

package/src/commands/validate.ts

@@ -12,7 +12,7 @@
 */
 
 // node
-import { type Dirent } from 'node:fs';
+import type { Dirent } from 'node:fs';
 import { basename, dirname, extname, join, parse, resolve } from 'node:path';
 import { readdir } from 'node:fs/promises';
 
@@ -33,13 +33,14 @@ import type {
   CustomField,
   FieldType,
   ReportMetadata,
-  ResourceContent,
+  AnyResourceContent,
   Workflow,
 } from '../interfaces/resource-interfaces.js';
 import { errorFunction } from '../utils/error-utils.js';
 import { isTemplateCard } from '../utils/card-utils.js';
+import { isPredefinedField } from '../utils/constants.js';
 import { pathExists } from '../utils/file-utils.js';
-import { Project } from '../containers/project.js';
+import type { Project } from '../containers/project.js';
 import { readJsonFile } from '../utils/json.js';
 import { type ResourceName, resourceName } from '../utils/resource-utils.js';
 
@@ -51,7 +52,7 @@ const SHORT_TEXT_MAX_LENGTH = 80;
 
 import * as EmailValidator from 'email-validator';
 import { evaluateMacros } from '../macros/index.js';
-const baseDir = import.meta.dirname;
+const baseDir = import.meta.dirname ?? new URL('.', import.meta.url).pathname;
 const subFoldersToValidate = ['.cards', 'cardRoot'];
 
 export interface LengthProvider {
@@ -82,6 +83,9 @@ export class Validate {
   static dotSchemaSchemaId = '/dotSchema';
   static parameterSchemaFile = 'parameterSchema.json';
 
+  /**
+   * Creates an instance of Validate command.
+   */
   constructor() {
     Validate.baseFolder = pathExists(
       join(process.cwd(), '../../schema', 'cardTreeDirectorySchema.json'),
@@ -117,7 +121,7 @@ export class Validate {
   private checkResourceName(
     file: Dirent,
     content:
-      | ResourceContent
+      | AnyResourceContent
       | CustomField
       | DotSchemaContent
       | ProjectSettings
@@ -133,7 +137,7 @@ export class Validate {
       file.name !== Validate.dotSchemaSchemaId &&
       file.name !== Validate.parameterSchemaFile
     ) {
-      const namedContent = content as ResourceContent | ReportMetadata;
+      const namedContent = content as AnyResourceContent | ReportMetadata;
       if (!namedContent.name) {
         errors.push(
           `File '${file.name}' does not contain 'name' property. Cannot validate resource's 'name'.`,
@@ -167,20 +171,7 @@ export class Validate {
     return join(file.parentPath, file.name);
   }
 
-  // Puts resource to a local cache if found and returns the resource.
-  // If value is already cached, returns from cache.
-  private getAndCacheResource<Type>(
-    project: Project,
-    cachedValues: Map<string, Type>,
-    valueName: string,
-  ): Type | undefined {
-    const resource = project.resource<Type>(valueName);
-    if (resource) {
-      cachedValues.set(valueName, resource);
-    }
-    return resource;
-  }
-
+  // Parses validator messages
   private parseValidatorMessage(errorObject: object[]): string {
     let parsedErrorMessage = '';
     // todo: get schema name here?
@@ -231,7 +222,7 @@ export class Validate {
   ): Promise<string[]> {
     const message: string[] = [];
     try {
-      const prefixes = await project.projectPrefixes();
+      const prefixes = project.allModulePrefixes();
       const files = await readdir(path, {
         withFileTypes: true,
       });
@@ -390,11 +381,13 @@ export class Validate {
     const errors: string[] = [];
     if (cardType && fieldArray) {
       const validationPromises = fieldArray.map(async (field) => {
-        const fieldType = this.getAndCacheResource(
-          project,
-          this.validatedFieldTypes,
-          field,
-        );
+        let fieldType;
+        try {
+          fieldType = project.resources.byType(field, 'fieldTypes').show();
+        } catch {
+          fieldType = undefined;
+        }
+
         if (!fieldType) {
           return `Card type '${cardType.name}' has invalid reference to unknown ${nameOfArray} '${field}'`;
         }
@@ -471,7 +464,7 @@ export class Validate {
 
   /**
    * Validates that new identifier of a resource is according to naming convention.
-   * @param identifier: resource identifier
+   * @param identifier Resource identifier
    * returns true if identifier is valid, and false otherwise.
    */
   public static isValidIdentifierName(identifier: string): boolean {
@@ -561,6 +554,7 @@ export class Validate {
         cards.push(...project.allTemplateCards());
 
         const cardIds = new Map<string, number>();
+        const allPrefixes = project.allModulePrefixes();
 
         for (const card of cards) {
           if (cardIds.has(card.key)) {
@@ -571,16 +565,19 @@ export class Validate {
 
           if (card.metadata) {
             if (!isTemplateCard(card)) {
-              const validWorkflow = this.validateWorkflowState(project, card);
+              const validWorkflow = await this.validateWorkflowState(
+                project,
+                card,
+              );
               if (validWorkflow.length !== 0) {
                 errorMsg.push(validWorkflow);
               }
             }
           }
-
           const validCustomFields = await this.validateCustomFields(
             project,
             card,
+            allPrefixes,
           );
           if (validCustomFields.length !== 0) {
             errorMsg.push(validCustomFields);
@@ -663,6 +660,9 @@ export class Validate {
    * @param resourceType Type of resource
    * @param name Name of resource
    * @param prefixes currently used project prefixes
+   * @throws when resource is not from given prefixes,
+   *         or when actual resource does not match the type,
+   *         or when identifier name is not valid
    * @returns resource name as valid resource name; throws in error cases.
    */
   public validResourceName(
@@ -709,11 +709,13 @@ export class Validate {
    * Validates that card's custom fields are according to schema and have correct data in them.
    * @param project currently used Project
    * @param card specific card
-   * @returns string containing all validation errors
+   * @throws when card does not have metadata
+   * @returns validation errors, if any.
    */
   public async validateCustomFields(
     project: Project,
     card: Card,
+    prefixes: string[],
   ): Promise<string> {
     const validationErrors: string[] = [];
 
@@ -723,11 +725,14 @@ export class Validate {
       );
     }
 
-    const cardType = this.getAndCacheResource(
-      project,
-      this.validatedCardTypes,
-      card.metadata?.cardType,
-    );
+    let cardType;
+    try {
+      cardType = project.resources
+        .byType(card.metadata?.cardType, 'cardTypes')
+        .show();
+    } catch {
+      cardType = undefined;
+    }
 
     if (!cardType) {
       validationErrors.push(
@@ -753,11 +758,19 @@ export class Validate {
     validationErrors.push(...fieldErrors);
 
     for (const field of cardType.customFields) {
-      const found = await project.resourceExists('fieldTypes', field.name);
-      if (!found) {
+      let fieldType;
+      try {
+        fieldType = await project.resources
+          .byType(field.name, 'fieldTypes')
+          .show();
+      } catch {
+        fieldType = undefined;
+      }
+      if (!fieldType) {
         validationErrors.push(
-          `Custom field '${field.name}' from card type '${cardType.name}' not found from project`,
+          `In card '${card.key}' field '${field.name}' is missing from project\n`,
         );
+        continue;
       }
       if (field.isCalculated) {
         if (card.metadata[field.name] !== undefined) {
@@ -775,19 +788,6 @@ export class Validate {
         }
       }
 
-      const fieldType = await this.getAndCacheResource(
-        project,
-        this.validatedFieldTypes,
-        field.name,
-      );
-
-      if (!fieldType) {
-        validationErrors.push(
-          `In card '${card.key}' field '${field.name}' is missing from project\n`,
-        );
-        continue;
-      }
-
       if (!this.validType(card.metadata[field.name], fieldType)) {
         const typeOfValue = typeof card.metadata[field.name];
         let fieldValue = card.metadata[field.name];
@@ -817,12 +817,40 @@ export class Validate {
       }
     }
 
+    // Validate that all metadata keys are either predefined fields or valid field type names
+    for (const key of Object.keys(card.metadata)) {
+      if (isPredefinedField(key) as boolean) {
+        continue;
+      }
+      try {
+        this.validResourceName('fieldTypes', key, prefixes);
+      } catch {
+        validationErrors.push(
+          `Card '${card.key}' has invalid metadata key '${key}'`,
+        );
+        continue;
+      }
+      // Check that the card's fieldType exists in the project
+      let fieldType;
+      try {
+        fieldType = await project.resources.byType(key, 'fieldTypes').show();
+      } catch {
+        fieldType = undefined;
+      }
+      if (!fieldType) {
+        validationErrors.push(
+          `Card '${card.key}' has field '${key}' that does not exist in the project`,
+        );
+      }
+    }
+
     return validationErrors.join('\n');
   }
 
   /**
    * Validates the labels of a card
    * @param card card to validate. Card must have metadata.
+   * @returns validation errors, if any.
    */
   public validateCardLabels(card: Card): string {
     const validationErrors: string[] = [];
@@ -856,9 +884,12 @@ export class Validate {
    * Template cards are expected to have empty workflow state.
    * @param project Project object.
    * @param card Card object to validate
-   * @returns string containing all validation errors
+   * @returns validation errors, if any.
    */
-  public validateWorkflowState(project: Project, card: Card): string {
+  public async validateWorkflowState(
+    project: Project,
+    card: Card,
+  ): Promise<string> {
     const validationErrors: string[] = [];
 
     if (!card.metadata) {
@@ -867,12 +898,14 @@ export class Validate {
       );
     }
 
-    // Use caches for cardTypes and workflows, to avoid re-reading the same JSON files multiple times.
-    const cardType = this.getAndCacheResource(
-      project,
-      this.validatedCardTypes,
-      card.metadata?.cardType || '',
-    );
+    let cardType;
+    try {
+      cardType = card.metadata?.cardType
+        ? project.resources.byType(card.metadata?.cardType, 'cardTypes').show()
+        : undefined;
+    } catch {
+      cardType = undefined;
+    }
     if (!cardType) {
       validationErrors.push(
         `Card '${card.key}' has invalid card type '${card.metadata?.cardType}'`,
@@ -886,11 +919,14 @@ export class Validate {
       return validationErrors.join('\n');
     }
 
-    const workflow = this.getAndCacheResource(
-      project,
-      this.validatedWorkflows,
-      cardType.workflow,
-    );
+    let workflow;
+    try {
+      workflow = project.resources
+        .byType(cardType.workflow, 'workflows')
+        .show();
+    } catch {
+      workflow = undefined;
+    }
 
     if (!workflow) {
       validationErrors.push(
@@ -919,15 +955,16 @@ export class Validate {
 
   /**
    * Validates a single resource.
-   * @param resource Resource to validate
-   * @returns string containing all validation errors
+   * @param resourceName Resource to validate
+   * @param project Project instance to use.
+   * @returns validation errors, if any.
    */
   public async validateResource(
     resourceName: ResourceName,
     project: Project,
   ): Promise<string> {
     try {
-      const resource = Project.resourceObject(project, resourceName);
+      const resource = project.resources.byType(resourceName);
       await resource.validate();
       return '';
     } catch (error) {

package/src/containers/card-container.ts

@@ -19,17 +19,19 @@ import { writeFile } from 'node:fs/promises';
 import { CardCache } from './project/card-cache.js';
 import { cardPathParts } from '../utils/card-utils.js';
 import { deleteDir } from '../utils/file-utils.js';
+import { getChildLogger } from '../utils/log-utils.js';
 import { writeJsonFile } from '../utils/json.js';
 
 import type {
   CardAttachment,
   Card,
+  CardMetadata,
   FetchCardDetails,
 } from '../interfaces/project-interfaces.js';
 
 import asciidoctor from '@asciidoctor/core';
 
-import { ROOT } from '../utils/constants.js';
+import { isPredefinedField, ROOT } from '../utils/constants.js';
 
 /**
  * Card container base class. Used for both Project and Template.
@@ -38,17 +40,19 @@ import { ROOT } from '../utils/constants.js';
 export class CardContainer {
   public basePath: string;
   protected cardCache: CardCache;
-  protected containerName: string;
   protected prefix: string;
 
+  protected static get logger() {
+    return getChildLogger({ module: 'CardContainer' });
+  }
+
   static cardContentFile = 'index.adoc';
   static cardMetadataFile = 'index.json';
   static projectConfigFileName = 'cardsConfig.json';
   static schemaContentFile = '.schema';
 
-  constructor(path: string, prefix: string, name: string) {
+  constructor(path: string, prefix: string) {
     this.basePath = path;
-    this.containerName = name;
     this.prefix = prefix;
     this.cardCache = new CardCache(this.prefix);
   }
@@ -226,13 +230,41 @@ export class CardContainer {
     if (card.metadata != null) {
       const metadataFile = join(card.path, CardContainer.cardMetadataFile);
       card.metadata!.lastUpdated = new Date().toISOString();
-      await writeJsonFile(metadataFile, card.metadata);
+
+      const sanitizedMetadata = CardContainer.sanitizeMetadata(card);
+      await writeJsonFile(metadataFile, sanitizedMetadata);
       return this.cardCache.updateCardMetadata(card.key, card.metadata);
     }
     return false;
   }
 
   /**
+   * Removes non-metadata fields that should not be persisted.
+   *
+   * @param metadata The metadata object to sanitize
+   * @returns Clean metadata object with only valid metadata fields
+   */
+  private static sanitizeMetadata(card: Card): CardMetadata {
+    const sanitized: Record<string, unknown> = {};
+
+    if (card.metadata) {
+      for (const [key, value] of Object.entries(card.metadata)) {
+        // Keys are not filtered out if they are: predefined, or field types
+        if (isPredefinedField(key) || key.includes('/')) {
+          sanitized[key] = value;
+        } else {
+          this.logger.warn(
+            `Card ${card.key} had extra metadata key ${key} with value ${value}. Key was removed`,
+          );
+        }
+        // Everything else is filtered out
+      }
+    }
+
+    return sanitized as CardMetadata;
+  }
+
+  /*
    * Show root cards from a given path.
    * @param path The path to get cards from
    * @returns an array of root-level cards (each with their children populated).