@cyberismo/data-handler 0.0.5 → 0.0.7

package/src/commands/show.ts

@@ -29,6 +29,7 @@ import type {
   ProjectFetchCardDetails,
   ProjectMetadata,
   Resource,
+  CardLocation,
 } from '../interfaces/project-interfaces.js';
 import type {
   CardType,
@@ -43,6 +44,7 @@ import { UserPreferences } from '../utils/user-preferences.js';
 
 import ReportMacro from '../macros/report/index.js';
 import TaskQueue from '../macros/task-queue.js';
+import type { Calculate } from './calculate.js';
 
 /**
  * Show command.
@@ -52,7 +54,10 @@ export class Show {
     string,
     (from?: ResourcesFrom) => Promise<Resource[]>
   >;
-  constructor(private project: Project) {
+  constructor(
+    private project: Project,
+    private calculate: Calculate,
+  ) {
     this.resourceFunction = new Map([
       ['cardTypes', this.project.cardTypes.bind(this.project)],
       ['fieldTypes', this.project.fieldTypes.bind(this.project)],
@@ -250,10 +255,13 @@ export class Show {
 
   /**
    * Shows all cards (either template or project cards) from a project.
+   * @param cardsFrom - The location from which to look for cards. Either from the project, templates or both.
    * @returns cards list array
    */
-  public async showCards(): Promise<CardListContainer[]> {
-    return this.project.listCards();
+  public async showCards(
+    cardsFrom?: CardLocation,
+  ): Promise<CardListContainer[]> {
+    return this.project.listCards(cardsFrom);
   }
 
   /**
@@ -351,10 +359,10 @@ export class Show {
       throw new Error(`Report '${reportName}' does not exist`);
     }
 
-    const reportMacro = new ReportMacro(new TaskQueue());
+    const reportMacro = new ReportMacro(new TaskQueue(), this.calculate);
     const result = await reportMacro.handleInject(
       {
-        projectPath: this.project.basePath,
+        project: this.project,
         cardKey: cardKey,
         mode: 'static',
       },

package/src/commands/validate.ts

@@ -20,7 +20,7 @@ import { readdir } from 'node:fs/promises';
 // dependencies
 import { Validator as JSONValidator, type Schema } from 'jsonschema';
 import { Validator as DirectoryValidator } from 'directory-schema-validator';
-import { parentSchema, schemas } from '@cyberismo/resources';
+import { parentSchema, schemas } from '@cyberismo/assets';
 
 // data-handler
 import type {
@@ -52,7 +52,7 @@ const SHORT_TEXT_MAX_LENGTH = 80;
 
 import * as EmailValidator from 'email-validator';
 import { evaluateMacros } from '../macros/index.js';
-
+import { Calculate } from './calculate.js';
 const baseDir = dirname(fileURLToPath(import.meta.url));
 const subFoldersToValidate = ['.cards', 'cardRoot'];
 
@@ -111,7 +111,7 @@ export class Validate {
   // Loads child schemas to validator.
   private addChildSchemas() {
     schemas.forEach((schema) => {
-      this.validator.addSchema(schema, schema.$id);
+      this.validator.addSchema(schema as Schema, schema.$id);
     });
   }
 
@@ -355,7 +355,7 @@ export class Validate {
       }
       const result = this.validator.validate(content, schema);
       for (const error of result.errors) {
-        const msg = `Validation error from '${fullPath}': ${error.message}.`;
+        const msg = `Validation error from '${fullPath}': ${error.property} ${error.message}.`;
         messages.push(msg);
       }
     }
@@ -547,6 +547,7 @@ export class Validate {
       } else {
         const errorMsg: string[] = [];
         const project = new Project(projectPath);
+        const calculate = new Calculate(project);
 
         // Then, validate that each 'contentSchema' children as well.
         const result = await this.readAndValidateContentFiles(
@@ -590,11 +591,15 @@ export class Validate {
 
           // Validate macros in content
           if (card.content) {
-            await evaluateMacros(card.content, {
-              mode: 'validate',
-              projectPath,
-              cardKey: card.key,
-            });
+            await evaluateMacros(
+              card.content,
+              {
+                mode: 'validate',
+                project,
+                cardKey: card.key,
+              },
+              calculate,
+            );
           }
         }
         if (errorMsg.length) {

package/src/containers/project/project-content-watcher.ts

@@ -0,0 +1,65 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { watch } from 'node:fs';
+
+import { getChildLogger } from '../../utils/log-utils.js';
+
+/**
+ * Class that starts watching certain path for changes.
+ * Generally we are not interested for file renames, as they are handled
+ * through rename command, thus there is an option to ignore those.
+ */
+export class ContentWatcher {
+  private watcher;
+  private abortController = new AbortController();
+
+  constructor(
+    ignoreRenames: boolean,
+    private watchPath: string,
+    callback: (fileName: string) => void,
+  ) {
+    this.watcher = watch(
+      this.watchPath,
+      {
+        persistent: true,
+        recursive: true,
+        signal: this.abortController.signal,
+      },
+      (eventType, filename) => {
+        if ((ignoreRenames && eventType === 'rename') || !filename) {
+          return;
+        }
+        callback(filename);
+      },
+    ).on('error', (error) => {
+      ContentWatcher.logger.error(error, 'Watch error');
+      this.close();
+    });
+    this.watcher.unref();
+  }
+
+  // Returns instance of logger.
+  private static get logger() {
+    return getChildLogger({
+      module: 'contentWatcher',
+    });
+  }
+
+  /**
+   * Close content watcher. Removes watcher.
+   */
+  public close() {
+    this.abortController.abort();
+  }
+}

package/src/containers/project/project-paths.ts

@@ -81,7 +81,7 @@ export class ProjectPaths {
   }
 
   public get logPath(): string {
-    return join(this.resourcesFolder, 'dh.log');
+    return join(this.path, '.logs', 'cyberismo_data-handler.log');
   }
 
   public get modulesFolder(): string {

package/src/containers/project.ts

@@ -64,6 +64,9 @@ import { ReportResource } from '../resources/report-resource.js';
 import { TemplateResource } from '../resources/template-resource.js';
 import { WorkflowResource } from '../resources/workflow-resource.js';
 
+import { ContentWatcher } from './project/project-content-watcher.js';
+import { pathToResourceName } from '../utils/resource-utils.js';
+
 // Re-export this, so that classes that use Project do not need to have separate import.
 export { ResourcesFrom };
 
@@ -75,12 +78,16 @@ export class Project extends CardContainer {
   private projectPaths: ProjectPaths;
   private settings: ProjectConfiguration;
   private validator: Validate;
+  private resourceWatcher: ContentWatcher | undefined;
 
   // Created resources are held in a cache.
   // In the cache, key is resource name, and data is resource metadata (as JSON).
   private createdResources = new Map<string, JSON>();
 
-  constructor(path: string) {
+  constructor(
+    path: string,
+    private watchResourceChanges?: boolean,
+  ) {
     super(path, '');
 
     this.settings = new ProjectConfiguration(
@@ -93,6 +100,48 @@ export class Project extends CardContainer {
     // todo: implement project validation
     this.validator = Validate.getInstance();
     this.resources.collectLocalResources();
+
+    const ignoreRenameFileChanges = true;
+
+    // Watch changes in .cards if there are multiple instances of Project being
+    // run concurrently.
+    if (this.watchResourceChanges) {
+      this.resourceWatcher = new ContentWatcher(
+        ignoreRenameFileChanges,
+        this.paths.resourcesFolder,
+        async (fileName: string) => {
+          let resource;
+          try {
+            resource = pathToResourceName(
+              this,
+              join(this.paths.resourcesFolder, fileName),
+            );
+            if (!resource) {
+              return;
+            }
+          } catch {
+            // it wasn't a resource that changed, so ignore the change
+            return;
+          }
+          const resourceName = `${resource.prefix}/${resource.type}/${resource.identifier}`;
+          await this.replaceCacheValue(resourceName);
+          this.resources.collectLocalResources();
+        },
+      );
+    }
+  }
+
+  // Removes current version of resource from cache.
+  // Then re-creates the resource with current data and caches the value again.
+  // If the value wasn't in the cache before, it will be added.
+  private async replaceCacheValue(resourceName: string) {
+    if (this.createdResources.has(resourceName)) {
+      // First, remove the old version from cache
+      this.createdResources.delete(resourceName);
+    }
+    const resourceData = await this.resource(resourceName);
+    if (!resourceData) return;
+    this.createdResources.set(resourceName, resourceData as JSON);
   }
 
   // Finds specific module.
@@ -318,6 +367,16 @@ export class Project extends CardContainer {
     return new TemplateResource(this, resourceName(template)).templateObject();
   }
 
+  /**
+   * Cleanups project when it is being closed.
+   */
+  public dispose() {
+    if (this.resourceWatcher) {
+      this.resourceWatcher.close();
+      this.resourceWatcher = undefined;
+    }
+  }
+
   /**
    * Returns an array of all the field types in the project.
    * @param from Defines where resources are collected from.

package/src/containers/template.ts

@@ -37,7 +37,7 @@ import {
   getRankAfter,
   sortItems,
 } from '../utils/lexorank.js';
-import { logger } from '../utils/log-utils.js';
+import { getChildLogger } from '../utils/log-utils.js';
 import { readJsonFile } from '../utils/json.js';
 import { Project } from './project.js';
 import { resourceName } from '../utils/resource-utils.js';
@@ -47,6 +47,11 @@ export class Template extends CardContainer {
   private templatePath: string;
   private templateCardsPath: string;
   private project: Project;
+  private get logger() {
+    return getChildLogger({
+      module: 'template',
+    });
+  }
 
   constructor(project: Project, template: Resource) {
     // Templates might come from modules. Remove module name from template name.
@@ -465,7 +470,7 @@ export class Template extends CardContainer {
     details?: FetchCardDetails,
   ): Promise<Card[]> {
     if (placeHolderPath) {
-      logger.warn('A non-used variable was used in the cards method');
+      this.logger.warn('A non-used variable was used in the cards method');
     }
     const cardDetails = details
       ? details

package/src/index.ts

@@ -17,6 +17,7 @@ import {
   CommandManager,
   ExportFormats,
 } from './command-handler.js';
+import type { Credentials } from './interfaces/project-interfaces.js';
 import { requestStatus } from './interfaces/request-status-interfaces.js';
 import { UpdateOperations } from './resources/resource-object.js';
 import { evaluateMacros } from './macros/index.js';
@@ -26,6 +27,7 @@ export {
   Cmd,
   CommandManager,
   Commands,
+  Credentials,
   ExportFormats,
   requestStatus,
   UpdateOperations,

package/src/interfaces/macros.ts

@@ -11,11 +11,12 @@
 */
 
 import type { macroMetadata } from '../macros/common.js';
+import type { Project } from '../containers/project.js';
 
 type Mode = 'validate' | 'static' | 'inject';
 
 export interface MacroGenerationContext {
-  projectPath: string;
+  project: Project;
   mode: Mode;
   cardKey: string;
 }

package/src/interfaces/project-interfaces.ts

@@ -65,6 +65,12 @@ export interface CardMetadata extends PredefinedCardMetadata {
 // Content in project (apart from cards) is either .schema files or cardsConfig.json.
 type ContentType = DotSchemaContent | ProjectSettings;
 
+// Credentials for private modules.
+export interface Credentials {
+  username?: string;
+  token?: string;
+}
+
 // Single CSV row as read from a file.
 export type CSVRowRaw = {
   [key: string]: string;
@@ -147,9 +153,11 @@ export interface ModuleSetting extends ModuleSettingOptions {
   location: string;
 }
 
+// Additional options for module configuration.
 export interface ModuleSettingOptions {
   branch?: string;
   private?: boolean;
+  credentials?: Credentials;
 }
 
 // Resources that are possible to remove.

package/src/interfaces/resource-interfaces.ts

@@ -1,13 +1,14 @@
 /**
-    Cyberismo
-    Copyright © Cyberismo Ltd and contributors 2024
-
-    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
-
-    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public
-    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2024
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation.
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+  FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+  details. You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
 
 import type { Schema } from 'jsonschema';
@@ -30,6 +31,7 @@ export interface CardType extends ResourceBaseMetadata {
 }
 
 // Custom field
+// todo: merge with FieldType.
 export interface CustomField {
   name: string;
   description?: string;
@@ -59,8 +61,6 @@ export interface EnumDefinition {
 
 // Field type content.
 export interface FieldType extends ResourceBaseMetadata {
-  displayName?: string;
-  fieldDescription?: string;
   dataType: DataType;
   enumValues?: Array<EnumDefinition>;
 }
@@ -68,8 +68,6 @@ export interface FieldType extends ResourceBaseMetadata {
 // Graph model content.
 export interface GraphModelMetadata extends ResourceBaseMetadata {
   category?: string;
-  description?: string;
-  displayName: string;
 }
 
 export interface GraphModel extends GraphModelMetadata {
@@ -79,8 +77,6 @@ export interface GraphModel extends GraphModelMetadata {
 // Graph view content.
 export interface GraphViewMetadata extends ResourceBaseMetadata {
   category?: string;
-  description?: string;
-  displayName: string;
 }
 
 export interface GraphView extends GraphViewMetadata {
@@ -104,7 +100,7 @@ export interface LinkType extends ResourceBaseMetadata {
 }
 
 // Report resource.
-export interface Report {
+export interface Report extends ResourceBaseMetadata {
   name: string;
   metadata: ReportMetadata;
   contentTemplate: string;
@@ -114,14 +110,14 @@ export interface Report {
 
 // Metadata for report
 export interface ReportMetadata extends ResourceBaseMetadata {
-  displayName: string;
-  description: string;
   category: string;
 }
 
 // Base interface for all resources.
 export interface ResourceBaseMetadata {
   name: string;
+  description?: string;
+  displayName: string;
   usedIn?: string[];
 }
 
@@ -137,8 +133,7 @@ export type ResourceContent =
   | Workflow;
 
 // Template configuration details.
-export interface TemplateConfiguration {
-  name: string;
+export interface TemplateConfiguration extends ResourceBaseMetadata {
   path: string;
   numberOfCards: number;
   metadata: TemplateMetadata;
@@ -146,8 +141,6 @@ export interface TemplateConfiguration {
 
 // Template configuration content details.
 export interface TemplateMetadata extends ResourceBaseMetadata {
-  displayName?: string;
-  description?: string;
   category?: string;
 }
 

package/src/macros/graph/index.ts

@@ -1,7 +1,6 @@
 /**
   Cyberismo
   Copyright © Cyberismo Ltd and contributors 2024
-
   This program is free software: you can redistribute it and/or modify it under
   the terms of the GNU Affero General Public License version 3 as published by
   the Free Software Foundation. This program is distributed in the hope that it
@@ -18,11 +17,10 @@ import type { MacroOptions } from '../index.js';
 import { createImage, validateMacroContent } from '../index.js';
 import Handlebars from 'handlebars';
 import { join } from 'node:path';
-import { logger } from '../../utils/log-utils.js';
+import { getChildLogger } from '../../utils/log-utils.js';
 import type { MacroGenerationContext } from '../../interfaces/macros.js';
 import macroMetadata from './metadata.js';
 import { pathExists } from '../../utils/file-utils.js';
-import { Project } from '../../containers/project.js';
 import { readFile } from 'node:fs/promises';
 import { resourceName } from '../../utils/resource-utils.js';
 import type { Schema } from 'jsonschema';
@@ -35,6 +33,11 @@ export interface GraphOptions extends MacroOptions {
 }
 
 class ReportMacro extends BaseMacro {
+  private get logger() {
+    return getChildLogger({
+      module: 'graphMacro',
+    });
+  }
   constructor(tasksQueue: TaskQueue) {
     super(macroMetadata, tasksQueue);
   }
@@ -48,16 +51,20 @@ class ReportMacro extends BaseMacro {
   };
 
   handleInject = async (context: MacroGenerationContext, input: unknown) => {
-    const project = new Project(context.projectPath);
-    const calculate = new Calculate(project);
+    const calculate = new Calculate(context.project);
 
     const resourceNameToPath = (name: string, fileName: string) => {
       const { identifier, prefix, type } = resourceName(name);
-      if (prefix === project.projectPrefix) {
-        return join(project.paths.resourcesFolder, type, identifier, fileName);
+      if (prefix === context.project.projectPrefix) {
+        return join(
+          context.project.paths.resourcesFolder,
+          type,
+          identifier,
+          fileName,
+        );
       }
       return join(
-        project.paths.modulesFolder,
+        context.project.paths.modulesFolder,
         prefix,
         type,
         identifier,
@@ -76,7 +83,7 @@ class ReportMacro extends BaseMacro {
         ),
       );
     } catch (err) {
-      logger.trace(
+      this.logger.trace(
         err,
         'Graph schema not found or failed to read, skipping validation',
       );
@@ -110,9 +117,7 @@ class ReportMacro extends BaseMacro {
     const view = handlebars.compile(viewContent)(handlebarsContext);
 
     const modelContent = await readFile(modelLocation, { encoding: 'utf-8' });
-    const result = await calculate.runGraph({
-      query: view + '\n' + modelContent,
-    });
+    const result = await calculate.runGraph(modelContent, view);
 
     if (typeof result !== 'string') {
       throw new Error(

package/src/macros/index.ts

@@ -28,21 +28,45 @@ import type {
 } from '../interfaces/macros.js';
 import type BaseMacro from './base-macro.js';
 import TaskQueue from './task-queue.js';
-
+import type { Calculate } from '../commands/index.js';
 const CURLY_LEFT = '{';
 const CURLY_RIGHT = '}';
 
-export interface MacroConstructor {
-  new (tasks: TaskQueue): BaseMacro; // Constructor signature
+/**
+ * Constructor for all macros except report macros
+ */
+export interface SimpleMacroConstructor {
+  new (tasks: TaskQueue): BaseMacro;
+}
+
+/**
+ * Constructor for report macros
+ */
+export interface ReportMacroConstructor {
+  new (tasks: TaskQueue, calculate: Calculate): BaseMacro;
 }
 
-export const macros: { [K in MacroName]: MacroConstructor } = {
+/**
+ * Constructor for all macros
+ */
+export type MacroConstructor = SimpleMacroConstructor | ReportMacroConstructor;
+
+export const macros: {
+  [K in MacroName]: MacroConstructor;
+} = {
   createCards,
   graph,
   report,
   scoreCard,
 };
 
+/**
+ * Validates the content inside a macro
+ * @param macro - The macro to validate the content of
+ * @param data - The data to validate
+ * @param validator - The validator to use
+ * @returns The validated data
+ */
 export function validateMacroContent<T>(
   macro: MacroMetadata,
   data: unknown,
@@ -74,11 +98,12 @@ export function registerMacros(
   instance: typeof Handlebars,
   context: MacroGenerationContext,
   tasks: TaskQueue,
+  calculate: Calculate,
 ) {
   const macroInstances: BaseMacro[] = [];
   for (const macro of Object.keys(macros) as MacroName[]) {
     const MacroClass = macros[macro];
-    const macroInstance = new MacroClass(tasks);
+    const macroInstance = new MacroClass(tasks, calculate);
     instance.registerHelper(macro, function (this: unknown, options) {
       if (
         this != null &&
@@ -134,11 +159,12 @@ export function registerEmptyMacros(instance: typeof Handlebars) {
 export async function evaluateMacros(
   content: string,
   context: MacroGenerationContext,
+  calculate: Calculate,
   maxTries: number = 10,
 ) {
   const handlebars = Handlebars.create();
   const tasks = new TaskQueue();
-  registerMacros(handlebars, context, tasks);
+  registerMacros(handlebars, context, tasks, calculate);
   let result = content;
   while (maxTries-- > 0) {
     tasks.reset();