@cyberismo/data-handler 0.0.6 → 0.0.8

package/src/containers/project.ts

@@ -64,6 +64,9 @@ import { ReportResource } from '../resources/report-resource.js';
 import { TemplateResource } from '../resources/template-resource.js';
 import { WorkflowResource } from '../resources/workflow-resource.js';
 
+import { ContentWatcher } from './project/project-content-watcher.js';
+import { pathToResourceName } from '../utils/resource-utils.js';
+
 // Re-export this, so that classes that use Project do not need to have separate import.
 export { ResourcesFrom };
 
@@ -75,12 +78,16 @@ export class Project extends CardContainer {
   private projectPaths: ProjectPaths;
   private settings: ProjectConfiguration;
   private validator: Validate;
+  private resourceWatcher: ContentWatcher | undefined;
 
   // Created resources are held in a cache.
   // In the cache, key is resource name, and data is resource metadata (as JSON).
   private createdResources = new Map<string, JSON>();
 
-  constructor(path: string) {
+  constructor(
+    path: string,
+    private watchResourceChanges?: boolean,
+  ) {
     super(path, '');
 
     this.settings = new ProjectConfiguration(
@@ -93,6 +100,48 @@ export class Project extends CardContainer {
     // todo: implement project validation
     this.validator = Validate.getInstance();
     this.resources.collectLocalResources();
+
+    const ignoreRenameFileChanges = true;
+
+    // Watch changes in .cards if there are multiple instances of Project being
+    // run concurrently.
+    if (this.watchResourceChanges) {
+      this.resourceWatcher = new ContentWatcher(
+        ignoreRenameFileChanges,
+        this.paths.resourcesFolder,
+        async (fileName: string) => {
+          let resource;
+          try {
+            resource = pathToResourceName(
+              this,
+              join(this.paths.resourcesFolder, fileName),
+            );
+            if (!resource) {
+              return;
+            }
+          } catch {
+            // it wasn't a resource that changed, so ignore the change
+            return;
+          }
+          const resourceName = `${resource.prefix}/${resource.type}/${resource.identifier}`;
+          await this.replaceCacheValue(resourceName);
+          this.resources.collectLocalResources();
+        },
+      );
+    }
+  }
+
+  // Removes current version of resource from cache.
+  // Then re-creates the resource with current data and caches the value again.
+  // If the value wasn't in the cache before, it will be added.
+  private async replaceCacheValue(resourceName: string) {
+    if (this.createdResources.has(resourceName)) {
+      // First, remove the old version from cache
+      this.createdResources.delete(resourceName);
+    }
+    const resourceData = await this.resource(resourceName);
+    if (!resourceData) return;
+    this.createdResources.set(resourceName, resourceData as JSON);
   }
 
   // Finds specific module.
@@ -318,6 +367,16 @@ export class Project extends CardContainer {
     return new TemplateResource(this, resourceName(template)).templateObject();
   }
 
+  /**
+   * Cleanups project when it is being closed.
+   */
+  public dispose() {
+    if (this.resourceWatcher) {
+      this.resourceWatcher.close();
+      this.resourceWatcher = undefined;
+    }
+  }
+
   /**
    * Returns an array of all the field types in the project.
    * @param from Defines where resources are collected from.
@@ -601,31 +660,58 @@ export class Project extends CardContainer {
         path: modulePath,
         cardKeyPrefix: moduleConfig.cardKeyPrefix,
         calculations: [
-          ...(await this.resources.collectResourcesFromModules('calculations')),
+          ...(await this.resources.collectResourcesFromModules(
+            'calculations',
+            moduleName,
+          )),
         ],
         cardTypes: [
-          ...(await this.resources.collectResourcesFromModules('cardTypes')),
+          ...(await this.resources.collectResourcesFromModules(
+            'cardTypes',
+            moduleName,
+          )),
         ],
         fieldTypes: [
-          ...(await this.resources.collectResourcesFromModules('fieldTypes')),
+          ...(await this.resources.collectResourcesFromModules(
+            'fieldTypes',
+            moduleName,
+          )),
         ],
         graphModels: [
-          ...(await this.resources.collectResourcesFromModules('graphModels')),
+          ...(await this.resources.collectResourcesFromModules(
+            'graphModels',
+            moduleName,
+          )),
         ],
         graphViews: [
-          ...(await this.resources.collectResourcesFromModules('graphViews')),
+          ...(await this.resources.collectResourcesFromModules(
+            'graphViews',
+            moduleName,
+          )),
         ],
         linkTypes: [
-          ...(await this.resources.collectResourcesFromModules('linkTypes')),
+          ...(await this.resources.collectResourcesFromModules(
+            'linkTypes',
+            moduleName,
+          )),
         ],
         reports: [
-          ...(await this.resources.collectResourcesFromModules('reports')),
+          ...(await this.resources.collectResourcesFromModules(
+            'reports',
+            moduleName,
+          )),
         ],
         templates: [
-          ...(await this.resources.collectResourcesFromModules('templates')),
+          ...(await this.resources.collectResourcesFromModules(
+            'templates',
+            moduleName,
+          )),
         ],
         workflows: [
-          ...(await this.resources.collectResourcesFromModules('workflows')),
+          ...(await this.resources.collectResourcesFromModules(
+            'workflows',
+            moduleName,
+          )),
         ],
       };
     }
@@ -764,15 +850,6 @@ export class Project extends CardContainer {
     return prefixes;
   }
 
-  /**
-   * Removes a module from project.
-   * @param moduleName Name of the module
-   */
-  public async removeModule(moduleName: string) {
-    await this.configuration.removeModule(moduleName);
-    await this.collectModuleResources();
-  }
-
   /**
    * Array of reports in the project.
    * @param from Defines where resources are collected from.

package/src/containers/template.ts

@@ -37,7 +37,7 @@ import {
   getRankAfter,
   sortItems,
 } from '../utils/lexorank.js';
-import { logger } from '../utils/log-utils.js';
+import { getChildLogger } from '../utils/log-utils.js';
 import { readJsonFile } from '../utils/json.js';
 import { Project } from './project.js';
 import { resourceName } from '../utils/resource-utils.js';
@@ -47,6 +47,11 @@ export class Template extends CardContainer {
   private templatePath: string;
   private templateCardsPath: string;
   private project: Project;
+  private get logger() {
+    return getChildLogger({
+      module: 'template',
+    });
+  }
 
   constructor(project: Project, template: Resource) {
     // Templates might come from modules. Remove module name from template name.
@@ -465,7 +470,7 @@ export class Template extends CardContainer {
     details?: FetchCardDetails,
   ): Promise<Card[]> {
     if (placeHolderPath) {
-      logger.warn('A non-used variable was used in the cards method');
+      this.logger.warn('A non-used variable was used in the cards method');
     }
     const cardDetails = details
       ? details

package/src/exceptions/index.ts

@@ -27,3 +27,39 @@ export class SchemaNotFound extends Error {
     this.name = 'SchemaNotFound';
   }
 }
+/**
+ * Stores the context of a macro error that originated from another macro
+ */
+export interface MacroDependency {
+  macroName: string;
+  parameters: string;
+  output?: string;
+}
+/**
+ * Thrown when a macro fails to execute.
+ */
+export class MacroError extends Error {
+  public context: {
+    cardKey: string;
+    macroName: string;
+    parameters: string;
+    dependency?: MacroDependency;
+  };
+
+  constructor(
+    message: string,
+    cardKey: string,
+    macroName: string,
+    parameters: string,
+    dependency?: MacroDependency,
+  ) {
+    super(message);
+    this.name = 'MacroError';
+    this.context = {
+      cardKey,
+      macroName,
+      parameters,
+      dependency,
+    };
+  }
+}

package/src/index.ts

@@ -17,6 +17,7 @@ import {
   CommandManager,
   ExportFormats,
 } from './command-handler.js';
+import type { Credentials } from './interfaces/project-interfaces.js';
 import { requestStatus } from './interfaces/request-status-interfaces.js';
 import { UpdateOperations } from './resources/resource-object.js';
 import { evaluateMacros } from './macros/index.js';
@@ -26,6 +27,7 @@ export {
   Cmd,
   CommandManager,
   Commands,
+  Credentials,
   ExportFormats,
   requestStatus,
   UpdateOperations,

package/src/interfaces/macros.ts

@@ -11,11 +11,13 @@
 */
 
 import type { macroMetadata } from '../macros/common.js';
+import type { Project } from '../containers/project.js';
+import type { MacroError } from '../exceptions/index.js';
 
 type Mode = 'validate' | 'static' | 'inject';
 
 export interface MacroGenerationContext {
-  projectPath: string;
+  project: Project;
   mode: Mode;
   cardKey: string;
 }
@@ -43,6 +45,8 @@ export interface MacroTaskState {
   placeholder: string;
   promiseResult: string | null;
   macro: string;
+  parameters: string;
+  error: MacroError | null;
 }
 
 // Handlebars options is not documented

package/src/interfaces/project-interfaces.ts

@@ -65,6 +65,12 @@ export interface CardMetadata extends PredefinedCardMetadata {
 // Content in project (apart from cards) is either .schema files or cardsConfig.json.
 type ContentType = DotSchemaContent | ProjectSettings;
 
+// Credentials for private modules.
+export interface Credentials {
+  username?: string;
+  token?: string;
+}
+
 // Single CSV row as read from a file.
 export type CSVRowRaw = {
   [key: string]: string;
@@ -147,9 +153,11 @@ export interface ModuleSetting extends ModuleSettingOptions {
   location: string;
 }
 
+// Additional options for module configuration.
 export interface ModuleSettingOptions {
   branch?: string;
   private?: boolean;
+  credentials?: Credentials;
 }
 
 // Resources that are possible to remove.

package/src/interfaces/resource-interfaces.ts

@@ -1,13 +1,14 @@
 /**
-    Cyberismo
-    Copyright © Cyberismo Ltd and contributors 2024
-
-    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
-
-    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public
-    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2024
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation.
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+  FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+  details. You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
 
 import type { Schema } from 'jsonschema';
@@ -30,6 +31,7 @@ export interface CardType extends ResourceBaseMetadata {
 }
 
 // Custom field
+// todo: merge with FieldType.
 export interface CustomField {
   name: string;
   description?: string;
@@ -59,8 +61,6 @@ export interface EnumDefinition {
 
 // Field type content.
 export interface FieldType extends ResourceBaseMetadata {
-  displayName?: string;
-  fieldDescription?: string;
   dataType: DataType;
   enumValues?: Array<EnumDefinition>;
 }
@@ -68,8 +68,6 @@ export interface FieldType extends ResourceBaseMetadata {
 // Graph model content.
 export interface GraphModelMetadata extends ResourceBaseMetadata {
   category?: string;
-  description?: string;
-  displayName: string;
 }
 
 export interface GraphModel extends GraphModelMetadata {
@@ -79,8 +77,6 @@ export interface GraphModel extends GraphModelMetadata {
 // Graph view content.
 export interface GraphViewMetadata extends ResourceBaseMetadata {
   category?: string;
-  description?: string;
-  displayName: string;
 }
 
 export interface GraphView extends GraphViewMetadata {
@@ -104,7 +100,7 @@ export interface LinkType extends ResourceBaseMetadata {
 }
 
 // Report resource.
-export interface Report {
+export interface Report extends ResourceBaseMetadata {
   name: string;
   metadata: ReportMetadata;
   contentTemplate: string;
@@ -114,14 +110,14 @@ export interface Report {
 
 // Metadata for report
 export interface ReportMetadata extends ResourceBaseMetadata {
-  displayName: string;
-  description: string;
   category: string;
 }
 
 // Base interface for all resources.
 export interface ResourceBaseMetadata {
   name: string;
+  description?: string;
+  displayName: string;
   usedIn?: string[];
 }
 
@@ -137,8 +133,7 @@ export type ResourceContent =
   | Workflow;
 
 // Template configuration details.
-export interface TemplateConfiguration {
-  name: string;
+export interface TemplateConfiguration extends ResourceBaseMetadata {
   path: string;
   numberOfCards: number;
   metadata: TemplateMetadata;
@@ -146,8 +141,6 @@ export interface TemplateConfiguration {
 
 // Template configuration content details.
 export interface TemplateMetadata extends ResourceBaseMetadata {
-  displayName?: string;
-  description?: string;
   category?: string;
 }
 

package/src/macros/base-macro.ts

@@ -17,13 +17,23 @@ import type {
   MacroTaskState,
 } from '../interfaces/macros.js';
 import { generateRandomString } from '../utils/random.js';
-import { handleMacroError } from './index.js';
+import { MacroError } from '../exceptions/index.js';
 import type TaskQueue from './task-queue.js';
+import { ClingoError } from '@cyberismo/node-clingo';
+import { getChildLogger } from '../utils/log-utils.js';
 
 abstract class BaseMacro {
   private globalId: string;
   private localCounter: number = 0;
 
+  // Macros share the same logger
+  protected get logger() {
+    return getChildLogger({
+      module: 'macro',
+      macro: this.macroMetadata.name,
+    });
+  }
+
   constructor(
     protected macroMetadata: MacroMetadata,
     private readonly tasks: TaskQueue,
@@ -64,7 +74,7 @@ abstract class BaseMacro {
       if (task) {
         dependencies.push(task);
       } else {
-        console.warn(
+        this.logger.warn(
           `Dependency not found for placeholder: ${placeholder} (globalId: ${globalId}, localId: ${localId})`,
         );
       }
@@ -81,6 +91,16 @@ abstract class BaseMacro {
     };
   }
 
+  private findTask(globalId: string, localId: number) {
+    const task = this.tasks.find(globalId, localId);
+    if (!task) {
+      this.logger.warn(
+        `Task not found for global id ${globalId}, local id ${localId}.`,
+      );
+    }
+    return task;
+  }
+
   /**
    * Function responsible for starting the promise and storing it along with its localId.
    */
@@ -111,20 +131,53 @@ abstract class BaseMacro {
     const dependencies = this.findDependencies(rawInput);
 
     // Create a promise to resolve dependencies, execute the macro, and handle the results
-    const promise = Promise.all(dependencies.map((dep) => dep.promise))
+    const promise = Promise.allSettled(dependencies.map((dep) => dep.promise))
       .then(() => {
         for (const dependency of dependencies) {
+          if (dependency.error) {
+            const task = this.findTask(this.globalId, localId);
+            if (task) {
+              // There could be a better way, but multi-nested macros are rare
+              task.error = new MacroError(
+                dependency.error.message,
+                context.cardKey,
+                this.metadata.name,
+                dependency.error.context.parameters,
+                {
+                  macroName: dependency.macro,
+                  parameters: dependency.parameters,
+                },
+              );
+            }
+            return;
+          }
           input = input.replace(
             dependency.placeholder,
             dependency.promiseResult || '',
           );
+          // parse json after each dep, so we know the exact macro which produced the error
+          try {
+            JSON.parse(input);
+          } catch {
+            const task = this.findTask(this.globalId, localId);
+            if (task) {
+              task.error = new MacroError(
+                'Invalid JSON produced by macro dependency',
+                context.cardKey,
+                this.metadata.name,
+                input,
+                {
+                  macroName: dependency.macro,
+                  parameters: dependency.parameters,
+                  output: input,
+                },
+              );
+            }
+            return;
+          }
         }
-        let parsed;
-        try {
-          parsed = JSON.parse(input);
-        } catch {
-          return 'Invalid JSON';
-        }
+        // This will never throw in practice, thus no need to catch
+        const parsed = JSON.parse(input);
 
         // Select the function to execute based on context mode
         const functionToCall =
@@ -134,28 +187,37 @@ abstract class BaseMacro {
         return functionToCall(context, parsed);
       })
       .then((result) => {
-        const task = this.tasks.find(this.globalId, localId);
+        // undefined is used to indicate that the macro did not run for some reason
+        if (result === undefined) {
+          return;
+        }
+        const task = this.findTask(this.globalId, localId);
         if (task) {
           task.promiseResult = result;
-        } else {
-          console.error(
-            `Task not found after execution: macro ${this.metadata.name}, local id ${localId}.`,
-          );
         }
       })
       .catch((err) => {
-        const task = this.tasks.find(this.globalId, localId);
+        if (!(err instanceof Error)) {
+          this.logger.error(err, 'Unknown error');
+          err = new Error('Unknown error');
+        }
+        const message =
+          err instanceof ClingoError
+            ? err.details.errors.join('\n')
+            : err.message;
+        const error =
+          err instanceof MacroError
+            ? err
+            : new MacroError(
+                message,
+                context.cardKey,
+                this.metadata.name,
+                input,
+              );
+
+        const task = this.findTask(this.globalId, localId);
         if (task) {
-          task.promiseResult = handleMacroError(
-            err,
-            this.metadata.name,
-            context,
-          );
-        } else {
-          console.error(
-            `Error handling task for macro ${this.metadata.name}, local id ${localId}:`,
-            err,
-          );
+          task.error = error;
         }
       });
 
@@ -167,6 +229,8 @@ abstract class BaseMacro {
       placeholder,
       promiseResult: null,
       macro: this.macroMetadata.name,
+      parameters: rawInput,
+      error: null,
     });
     // Return the placeholder
     return placeholder;

package/src/macros/graph/index.ts

@@ -1,7 +1,6 @@
 /**
   Cyberismo
   Copyright © Cyberismo Ltd and contributors 2024
-
   This program is free software: you can redistribute it and/or modify it under
   the terms of the GNU Affero General Public License version 3 as published by
   the Free Software Foundation. This program is distributed in the hope that it
@@ -18,16 +17,15 @@ import type { MacroOptions } from '../index.js';
 import { createImage, validateMacroContent } from '../index.js';
 import Handlebars from 'handlebars';
 import { join } from 'node:path';
-import { logger } from '../../utils/log-utils.js';
 import type { MacroGenerationContext } from '../../interfaces/macros.js';
 import macroMetadata from './metadata.js';
 import { pathExists } from '../../utils/file-utils.js';
-import { Project } from '../../containers/project.js';
 import { readFile } from 'node:fs/promises';
 import { resourceName } from '../../utils/resource-utils.js';
 import type { Schema } from 'jsonschema';
 import { validateJson } from '../../utils/validate.js';
 import type TaskQueue from '../task-queue.js';
+import { ClingoError } from '@cyberismo/node-clingo';
 
 export interface GraphOptions extends MacroOptions {
   model: string;
@@ -48,16 +46,20 @@ class ReportMacro extends BaseMacro {
   };
 
   handleInject = async (context: MacroGenerationContext, input: unknown) => {
-    const project = new Project(context.projectPath);
-    const calculate = new Calculate(project);
+    const calculate = new Calculate(context.project);
 
     const resourceNameToPath = (name: string, fileName: string) => {
       const { identifier, prefix, type } = resourceName(name);
-      if (prefix === project.projectPrefix) {
-        return join(project.paths.resourcesFolder, type, identifier, fileName);
+      if (prefix === context.project.projectPrefix) {
+        return join(
+          context.project.paths.resourcesFolder,
+          type,
+          identifier,
+          fileName,
+        );
       }
       return join(
-        project.paths.modulesFolder,
+        context.project.paths.modulesFolder,
         prefix,
         type,
         identifier,
@@ -76,7 +78,7 @@ class ReportMacro extends BaseMacro {
         ),
       );
     } catch (err) {
-      logger.trace(
+      this.logger.trace(
         err,
         'Graph schema not found or failed to read, skipping validation',
       );
@@ -110,9 +112,17 @@ class ReportMacro extends BaseMacro {
     const view = handlebars.compile(viewContent)(handlebarsContext);
 
     const modelContent = await readFile(modelLocation, { encoding: 'utf-8' });
-    const result = await calculate.runGraph({
-      query: view + '\n' + modelContent,
-    });
+    let result: string;
+    try {
+      result = await calculate.runGraph(modelContent, view);
+    } catch (error) {
+      if (error instanceof ClingoError) {
+        throw new Error(
+          `Error running graph in view '${options.view}' in model '${options.model}': ${error.details.errors.join('\n')}`,
+        );
+      }
+      throw error;
+    }
 
     if (typeof result !== 'string') {
       throw new Error(