@cyberismo/data-handler 0.0.7 → 0.0.8

package/src/commands/create.ts

@@ -72,15 +72,20 @@ export class Create {
     },
   ];
 
-  static gitIgnoreContent: string = `.calc\n
-        .asciidoctor\n
-        .vscode\n
-        *.html\n
-        *.pdf\n
-        *.puml\n
-        **/.DS_Store\n
-        *-debug.log\n
-        *-error.log\n`;
+  static gitIgnoreContent: string[] = [
+    '.calc',
+    '.asciidoctor',
+    '.vscode',
+    '*.html',
+    '*.pdf',
+    '*.puml',
+    '**/.DS_Store',
+    '*-debug.log',
+    '*-error.log',
+    '.temp',
+    '.logs',
+    '.cache',
+  ];
 
   /**
    * Adds new cards to a template.
@@ -510,7 +515,10 @@ export class Create {
     });
 
     try {
-      await writeFile(join(projectPath, '.gitignore'), this.gitIgnoreContent);
+      await writeFile(
+        join(projectPath, '.gitignore'),
+        this.gitIgnoreContent.join('\n') + '\n',
+      );
     } catch {
       console.error('Failed to create project');
     }

package/src/commands/import.ts

@@ -134,7 +134,10 @@ export class Import {
     destination?: string,
     options?: ModuleSettingOptions,
   ) {
-    const gitModule = source.startsWith('https');
+    const beforeImportValidateErrors = await Validate.getInstance().validate(
+      this.project.basePath,
+    );
+    const gitModule = source.startsWith('https') || source.startsWith('git@');
     const modulePrefix = gitModule
       ? await this.moduleManager.importGitModule(source, options)
       : await this.moduleManager.importFileModule(source, destination);
@@ -156,7 +159,17 @@ export class Import {
     await this.moduleManager.updateModule(moduleSettings, options?.credentials);
 
     // Add module as a dependency.
-    return this.project.importModule(moduleSettings);
+    await this.project.importModule(moduleSettings);
+
+    // Validate the project after module has been imported
+    const afterImportValidateErrors = await Validate.getInstance().validate(
+      this.project.basePath,
+    );
+    if (afterImportValidateErrors.length > beforeImportValidateErrors.length) {
+      console.error(
+        `There are new validations errors after importing the module. Check the project`,
+      );
+    }
   }
 
   /**

package/src/commands/remove.ts

@@ -17,6 +17,7 @@ import { join, sep } from 'node:path';
 import { ActionGuard } from '../permissions/action-guard.js';
 import type { Calculate } from './index.js';
 import { deleteDir, deleteFile } from '../utils/file-utils.js';
+import { ModuleManager } from '../module-manager.js';
 import { Project } from '../containers/project.js';
 import type { RemovableResourceTypes } from '../interfaces/project-interfaces.js';
 import { resourceName } from '../utils/resource-utils.js';
@@ -27,10 +28,13 @@ const MODULES_PATH = `${sep}modules${sep}`;
  * Remove command.
  */
 export class Remove {
+  private moduleManager: ModuleManager;
   constructor(
     private project: Project,
     private calculateCmd: Calculate,
-  ) {}
+  ) {
+    this.moduleManager = new ModuleManager(this.project);
+  }
 
   // True, if resource is a project resource
   private projectResource(type: RemovableResourceTypes): boolean {
@@ -182,16 +186,6 @@ export class Remove {
     await this.project.updateCardMetadataKey(sourceCardKey, 'links', newLinks);
   }
 
-  // Removes modules from project
-  private async removeModule(moduleName: string) {
-    const module = await this.project.module(moduleName);
-    if (!module) {
-      throw new Error(`Module '${moduleName}' not found`);
-    }
-    await this.project.removeModule(moduleName);
-    await deleteDir(module.path);
-  }
-
   /**
    * Removes either attachment, card, imported module, link or resource from project.
    * @param type Type of resource
@@ -236,7 +230,8 @@ export class Remove {
       else if (type == 'card') return this.removeCard(targetName);
       else if (type == 'link')
         return this.removeLink(targetName, rest[0], rest[1], rest.at(2));
-      else if (type == 'module') return this.removeModule(targetName);
+      else if (type == 'module')
+        return this.moduleManager.removeModule(targetName);
       else if (type == 'label') return this.removeLabel(targetName, rest[0]);
     }
     throw new Error(`Unknown resource type '${type}'`);

package/src/commands/validate.ts

@@ -704,41 +704,6 @@ export class Validate {
     return contentValidated && lengthValidated;
   }
 
-  /**
-   * Validate schema that matches schemaId from path.
-   * @param projectPath path to schema
-   * @param schemaId schema's id
-   * @returns string containing all validation errors
-   * @todo - unused; remove?
-   */
-  public async validateSchema(
-    projectPath: string,
-    schemaId: string,
-  ): Promise<string> {
-    const validationErrors: string[] = [];
-    if (!schemaId.startsWith('/')) {
-      schemaId = '/' + schemaId;
-    }
-    const activeJsonSchema = this.validator.schemas[schemaId];
-    if (activeJsonSchema === undefined) {
-      throw new Error(`Unknown schema '${schemaId}'`);
-    } else {
-      let contentFile = '';
-      try {
-        contentFile = await readJsonFile(projectPath);
-      } catch {
-        throw new Error(`Path is not valid ${projectPath}`);
-      }
-
-      const result = this.validator.validate(contentFile, activeJsonSchema);
-      for (const error of result.errors) {
-        const msg = `Schema '${schemaId}' validation Error: ${error.message}\n`;
-        validationErrors.push(msg);
-      }
-    }
-    return validationErrors.join('\n');
-  }
-
   /**
    * Validates that card's custom fields are according to schema and have correct data in them.
    * @param project currently used Project

package/src/containers/project/resource-collector.ts

@@ -55,17 +55,31 @@ class ResourceCollection {
    * @param type Resource array type to return.
    * @returns resource array of a give type.
    */
-  public resourceArray(type: ResourceFolderType): Resource[] {
-    if (type === 'calculations') return this.calculations;
-    if (type === 'cardTypes') return this.cardTypes;
-    if (type === 'fieldTypes') return this.fieldTypes;
-    if (type === 'graphViews') return this.graphViews;
-    if (type === 'graphModels') return this.graphModels;
-    if (type === 'linkTypes') return this.linkTypes;
-    if (type === 'reports') return this.reports;
-    if (type === 'templates') return this.templates;
-    if (type === 'workflows') return this.workflows;
-    throw new Error(`Unknown resource type '${type}'`);
+  public resourceArray(
+    type: ResourceFolderType,
+    moduleName?: string,
+  ): Resource[] {
+    let resources: Resource[] = [];
+
+    if (type === 'calculations') resources = this.calculations;
+    else if (type === 'cardTypes') resources = this.cardTypes;
+    else if (type === 'fieldTypes') resources = this.fieldTypes;
+    else if (type === 'graphViews') resources = this.graphViews;
+    else if (type === 'graphModels') resources = this.graphModels;
+    else if (type === 'linkTypes') resources = this.linkTypes;
+    else if (type === 'reports') resources = this.reports;
+    else if (type === 'templates') resources = this.templates;
+    else if (type === 'workflows') resources = this.workflows;
+    else throw new Error(`Unknown resource type '${type}'`);
+
+    if (moduleName) {
+      resources = resources.filter((item) => {
+        const { prefix } = resourceName(item.name);
+        return moduleName === prefix;
+      });
+    }
+
+    return resources;
   }
 }
 
@@ -162,6 +176,7 @@ export class ResourceCollector {
   // Adds a resource type from all modules.
   private async addResourcesFromModules(
     type: ResourceFolderType,
+    moduleName?: string,
   ): Promise<Resource[]> {
     try {
       // 'modules' is a bit special; it is collected separately from actual resources.
@@ -174,7 +189,7 @@ export class ResourceCollector {
       }
 
       await this.addModuleResources();
-      return this.modules.resourceArray(type);
+      return this.modules.resourceArray(type, moduleName);
     } catch {
       return [];
     }
@@ -261,10 +276,14 @@ export class ResourceCollector {
   /**
    * Collect specific resource from modules.
    * @param type Type of resource (e.g. 'templates').
+   * @param moduleName Name of the module to collect resources from
    * @returns array of collected items.
    */
-  public async collectResourcesFromModules(type: ResourceFolderType) {
-    return (await this.addResourcesFromModules(type)).map((item) =>
+  public async collectResourcesFromModules(
+    type: ResourceFolderType,
+    moduleName?: string,
+  ) {
+    return (await this.addResourcesFromModules(type, moduleName)).map((item) =>
       stripExtension(item.name),
     );
   }

package/src/containers/project.ts

@@ -660,31 +660,58 @@ export class Project extends CardContainer {
         path: modulePath,
         cardKeyPrefix: moduleConfig.cardKeyPrefix,
         calculations: [
-          ...(await this.resources.collectResourcesFromModules('calculations')),
+          ...(await this.resources.collectResourcesFromModules(
+            'calculations',
+            moduleName,
+          )),
         ],
         cardTypes: [
-          ...(await this.resources.collectResourcesFromModules('cardTypes')),
+          ...(await this.resources.collectResourcesFromModules(
+            'cardTypes',
+            moduleName,
+          )),
         ],
         fieldTypes: [
-          ...(await this.resources.collectResourcesFromModules('fieldTypes')),
+          ...(await this.resources.collectResourcesFromModules(
+            'fieldTypes',
+            moduleName,
+          )),
         ],
         graphModels: [
-          ...(await this.resources.collectResourcesFromModules('graphModels')),
+          ...(await this.resources.collectResourcesFromModules(
+            'graphModels',
+            moduleName,
+          )),
         ],
         graphViews: [
-          ...(await this.resources.collectResourcesFromModules('graphViews')),
+          ...(await this.resources.collectResourcesFromModules(
+            'graphViews',
+            moduleName,
+          )),
         ],
         linkTypes: [
-          ...(await this.resources.collectResourcesFromModules('linkTypes')),
+          ...(await this.resources.collectResourcesFromModules(
+            'linkTypes',
+            moduleName,
+          )),
         ],
         reports: [
-          ...(await this.resources.collectResourcesFromModules('reports')),
+          ...(await this.resources.collectResourcesFromModules(
+            'reports',
+            moduleName,
+          )),
         ],
         templates: [
-          ...(await this.resources.collectResourcesFromModules('templates')),
+          ...(await this.resources.collectResourcesFromModules(
+            'templates',
+            moduleName,
+          )),
         ],
         workflows: [
-          ...(await this.resources.collectResourcesFromModules('workflows')),
+          ...(await this.resources.collectResourcesFromModules(
+            'workflows',
+            moduleName,
+          )),
         ],
       };
     }
@@ -823,15 +850,6 @@ export class Project extends CardContainer {
     return prefixes;
   }
 
-  /**
-   * Removes a module from project.
-   * @param moduleName Name of the module
-   */
-  public async removeModule(moduleName: string) {
-    await this.configuration.removeModule(moduleName);
-    await this.collectModuleResources();
-  }
-
   /**
    * Array of reports in the project.
    * @param from Defines where resources are collected from.

package/src/exceptions/index.ts

@@ -27,3 +27,39 @@ export class SchemaNotFound extends Error {
     this.name = 'SchemaNotFound';
   }
 }
+/**
+ * Stores the context of a macro error that originated from another macro
+ */
+export interface MacroDependency {
+  macroName: string;
+  parameters: string;
+  output?: string;
+}
+/**
+ * Thrown when a macro fails to execute.
+ */
+export class MacroError extends Error {
+  public context: {
+    cardKey: string;
+    macroName: string;
+    parameters: string;
+    dependency?: MacroDependency;
+  };
+
+  constructor(
+    message: string,
+    cardKey: string,
+    macroName: string,
+    parameters: string,
+    dependency?: MacroDependency,
+  ) {
+    super(message);
+    this.name = 'MacroError';
+    this.context = {
+      cardKey,
+      macroName,
+      parameters,
+      dependency,
+    };
+  }
+}

package/src/interfaces/macros.ts

@@ -12,6 +12,7 @@
 
 import type { macroMetadata } from '../macros/common.js';
 import type { Project } from '../containers/project.js';
+import type { MacroError } from '../exceptions/index.js';
 
 type Mode = 'validate' | 'static' | 'inject';
 
@@ -44,6 +45,8 @@ export interface MacroTaskState {
   placeholder: string;
   promiseResult: string | null;
   macro: string;
+  parameters: string;
+  error: MacroError | null;
 }
 
 // Handlebars options is not documented

package/src/macros/base-macro.ts

@@ -17,13 +17,23 @@ import type {
   MacroTaskState,
 } from '../interfaces/macros.js';
 import { generateRandomString } from '../utils/random.js';
-import { handleMacroError } from './index.js';
+import { MacroError } from '../exceptions/index.js';
 import type TaskQueue from './task-queue.js';
+import { ClingoError } from '@cyberismo/node-clingo';
+import { getChildLogger } from '../utils/log-utils.js';
 
 abstract class BaseMacro {
   private globalId: string;
   private localCounter: number = 0;
 
+  // Macros share the same logger
+  protected get logger() {
+    return getChildLogger({
+      module: 'macro',
+      macro: this.macroMetadata.name,
+    });
+  }
+
   constructor(
     protected macroMetadata: MacroMetadata,
     private readonly tasks: TaskQueue,
@@ -64,7 +74,7 @@ abstract class BaseMacro {
       if (task) {
         dependencies.push(task);
       } else {
-        console.warn(
+        this.logger.warn(
           `Dependency not found for placeholder: ${placeholder} (globalId: ${globalId}, localId: ${localId})`,
         );
       }
@@ -81,6 +91,16 @@ abstract class BaseMacro {
     };
   }
 
+  private findTask(globalId: string, localId: number) {
+    const task = this.tasks.find(globalId, localId);
+    if (!task) {
+      this.logger.warn(
+        `Task not found for global id ${globalId}, local id ${localId}.`,
+      );
+    }
+    return task;
+  }
+
   /**
    * Function responsible for starting the promise and storing it along with its localId.
    */
@@ -111,20 +131,53 @@ abstract class BaseMacro {
     const dependencies = this.findDependencies(rawInput);
 
     // Create a promise to resolve dependencies, execute the macro, and handle the results
-    const promise = Promise.all(dependencies.map((dep) => dep.promise))
+    const promise = Promise.allSettled(dependencies.map((dep) => dep.promise))
       .then(() => {
         for (const dependency of dependencies) {
+          if (dependency.error) {
+            const task = this.findTask(this.globalId, localId);
+            if (task) {
+              // There could be a better way, but multi-nested macros are rare
+              task.error = new MacroError(
+                dependency.error.message,
+                context.cardKey,
+                this.metadata.name,
+                dependency.error.context.parameters,
+                {
+                  macroName: dependency.macro,
+                  parameters: dependency.parameters,
+                },
+              );
+            }
+            return;
+          }
           input = input.replace(
             dependency.placeholder,
             dependency.promiseResult || '',
           );
+          // parse json after each dep, so we know the exact macro which produced the error
+          try {
+            JSON.parse(input);
+          } catch {
+            const task = this.findTask(this.globalId, localId);
+            if (task) {
+              task.error = new MacroError(
+                'Invalid JSON produced by macro dependency',
+                context.cardKey,
+                this.metadata.name,
+                input,
+                {
+                  macroName: dependency.macro,
+                  parameters: dependency.parameters,
+                  output: input,
+                },
+              );
+            }
+            return;
+          }
         }
-        let parsed;
-        try {
-          parsed = JSON.parse(input);
-        } catch {
-          return 'Invalid JSON';
-        }
+        // This will never throw in practice, thus no need to catch
+        const parsed = JSON.parse(input);
 
         // Select the function to execute based on context mode
         const functionToCall =
@@ -134,28 +187,37 @@ abstract class BaseMacro {
         return functionToCall(context, parsed);
       })
       .then((result) => {
-        const task = this.tasks.find(this.globalId, localId);
+        // undefined is used to indicate that the macro did not run for some reason
+        if (result === undefined) {
+          return;
+        }
+        const task = this.findTask(this.globalId, localId);
         if (task) {
           task.promiseResult = result;
-        } else {
-          console.error(
-            `Task not found after execution: macro ${this.metadata.name}, local id ${localId}.`,
-          );
         }
       })
       .catch((err) => {
-        const task = this.tasks.find(this.globalId, localId);
+        if (!(err instanceof Error)) {
+          this.logger.error(err, 'Unknown error');
+          err = new Error('Unknown error');
+        }
+        const message =
+          err instanceof ClingoError
+            ? err.details.errors.join('\n')
+            : err.message;
+        const error =
+          err instanceof MacroError
+            ? err
+            : new MacroError(
+                message,
+                context.cardKey,
+                this.metadata.name,
+                input,
+              );
+
+        const task = this.findTask(this.globalId, localId);
         if (task) {
-          task.promiseResult = handleMacroError(
-            err,
-            this.metadata.name,
-            context,
-          );
-        } else {
-          console.error(
-            `Error handling task for macro ${this.metadata.name}, local id ${localId}:`,
-            err,
-          );
+          task.error = error;
         }
       });
 
@@ -167,6 +229,8 @@ abstract class BaseMacro {
       placeholder,
       promiseResult: null,
       macro: this.macroMetadata.name,
+      parameters: rawInput,
+      error: null,
     });
     // Return the placeholder
     return placeholder;

package/src/macros/graph/index.ts

@@ -17,7 +17,6 @@ import type { MacroOptions } from '../index.js';
 import { createImage, validateMacroContent } from '../index.js';
 import Handlebars from 'handlebars';
 import { join } from 'node:path';
-import { getChildLogger } from '../../utils/log-utils.js';
 import type { MacroGenerationContext } from '../../interfaces/macros.js';
 import macroMetadata from './metadata.js';
 import { pathExists } from '../../utils/file-utils.js';
@@ -26,6 +25,7 @@ import { resourceName } from '../../utils/resource-utils.js';
 import type { Schema } from 'jsonschema';
 import { validateJson } from '../../utils/validate.js';
 import type TaskQueue from '../task-queue.js';
+import { ClingoError } from '@cyberismo/node-clingo';
 
 export interface GraphOptions extends MacroOptions {
   model: string;
@@ -33,11 +33,6 @@ export interface GraphOptions extends MacroOptions {
 }
 
 class ReportMacro extends BaseMacro {
-  private get logger() {
-    return getChildLogger({
-      module: 'graphMacro',
-    });
-  }
   constructor(tasksQueue: TaskQueue) {
     super(macroMetadata, tasksQueue);
   }
@@ -117,7 +112,17 @@ class ReportMacro extends BaseMacro {
     const view = handlebars.compile(viewContent)(handlebarsContext);
 
     const modelContent = await readFile(modelLocation, { encoding: 'utf-8' });
-    const result = await calculate.runGraph(modelContent, view);
+    let result: string;
+    try {
+      result = await calculate.runGraph(modelContent, view);
+    } catch (error) {
+      if (error instanceof ClingoError) {
+        throw new Error(
+          `Error running graph in view '${options.view}' in model '${options.model}': ${error.details.errors.join('\n')}`,
+        );
+      }
+      throw error;
+    }
 
     if (typeof result !== 'string') {
       throw new Error(

package/src/macros/index.ts

@@ -18,7 +18,7 @@ import report from './report/index.js';
 import scoreCard from './scoreCard/index.js';
 
 import { validateJson } from '../utils/validate.js';
-import { DHValidationError } from '../exceptions/index.js';
+import { DHValidationError, MacroError } from '../exceptions/index.js';
 import type { AdmonitionType } from '../interfaces/adoc.js';
 import type { Validator } from 'jsonschema';
 import type {
@@ -29,6 +29,7 @@ import type {
 import type BaseMacro from './base-macro.js';
 import TaskQueue from './task-queue.js';
 import type { Calculate } from '../commands/index.js';
+import { ClingoError } from '@cyberismo/node-clingo';
 const CURLY_LEFT = '{';
 const CURLY_RIGHT = '}';
 
@@ -211,7 +212,14 @@ export function applyMacroResults(
   context: MacroGenerationContext,
 ) {
   for (const item of tasks) {
-    if (item.promiseResult === null) {
+    if (item.error) {
+      input = input.replace(
+        item.placeholder,
+        handleMacroError(item.error, item.macro, context),
+      );
+    } // It should not be possible that promiseResult is null if there never was an error
+    // Unless the function itself returns null / undefined
+    else if (item.promiseResult == null) {
       input = handleMacroError(
         new Error(
           `Tried to access result before it was resolved for ${item.placeholder}`,
@@ -240,7 +248,17 @@ export function handleMacroError(
   let message = `Error: ${error instanceof Error ? error.message : 'Unknown error'}`;
   if (error instanceof DHValidationError) {
     message = `Check json syntax of macro ${macro}: ${error.errors?.map((e) => e.message).join(', ')}`;
+  } else if (error instanceof MacroError) {
+    const { cardKey, macroName, dependency } = error.context;
+    message = `Macro error in card '${cardKey}' in macro '${macroName}':\n\n${error.message}.`;
+
+    if (dependency) {
+      message += `\n\nParameters:\n\n${context.mode === 'validate' ? dependency.parameters : createCodeBlock(dependency.parameters)}.\n\n${dependency.output ? `Output:\n\n${context.mode === 'validate' ? dependency.output : createCodeBlock(dependency.output)}` : ''}`;
+    }
+  } else if (error instanceof ClingoError) {
+    message = `Error running logic program in macro '${macro}':${error.details.errors.join('\n')}`;
   }
+
   if (
     typeof error === 'object' &&
     error != null &&
@@ -328,6 +346,15 @@ export function createAdmonition(
   return `[${type}]\n.${label}\n====\n${content}\n====\n\n`;
 }
 
+/**
+ * Creates a code block
+ * @param content - The content of the code block
+ * @returns The code block as a string
+ */
+export function createCodeBlock(content: string) {
+  return `\n\n----\n${content}\n----\n\n`;
+}
+
 /**
  * Helper function for including base64 encoded images for now
  * @param image base64 encoded image