@cyberismo/data-handler 0.0.14 → 0.0.16

package/src/macros/percentage/types.ts

@@ -0,0 +1,32 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+export interface PercentageOptions {
+  /**
+   * Title of percentage macro
+   */
+  title: string;
+  /**
+   * Percentage value
+   */
+  value: number;
+  /**
+   * Legend text below the macro
+   */
+  legend: string;
+  /**
+   * Color of the bar the represents the percentage
+   */
+  colour?: 'blue' | 'green' | 'yellow' | 'red' | 'orange' | 'purple';
+}

package/src/macros/report/index.ts

@@ -18,14 +18,9 @@ import macroMetadata from './metadata.js';
 import BaseMacro from '../base-macro.js';
 import { validateJson } from '../../utils/validate.js';
 import type TaskQueue from '../task-queue.js';
-import { ReportResource } from '../../resources/report-resource.js';
-import { resourceName } from '../../utils/resource-utils.js';
 import { generateReportContent } from '../../utils/report.js';
 import { ClingoError } from '@cyberismo/node-clingo';
-
-export interface ReportOptions {
-  name: string;
-}
+import type { ReportOptions } from './types.js';
 
 class ReportMacro extends BaseMacro {
   constructor(tasks: TaskQueue) {
@@ -37,13 +32,9 @@ class ReportMacro extends BaseMacro {
 
   handleStatic = async (context: MacroGenerationContext, data: unknown) => {
     const options = this.validate(data);
-    const resource = new ReportResource(
-      context.project,
-      resourceName(options.name),
-    );
-    const report = await resource.show();
-
-    if (!report) throw new Error(`Report ${options.name} does not exist`);
+    const report = context.project.resources
+      .byType(options.name, 'reports')
+      .show();
 
     if (report.content.schema) {
       validateJson(options, {

package/src/macros/report/types.ts

@@ -0,0 +1,20 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+export interface ReportOptions {
+  /**
+   * Name of the report
+   */
+  name: string;
+}

package/src/macros/scoreCard/index.ts

@@ -19,13 +19,7 @@ import macroMetadata from './metadata.js';
 import BaseMacro from '../base-macro.js';
 import type TaskQueue from '../task-queue.js';
 import { scoreCard } from '../../svg/index.js';
-
-export interface ScoreCardOptions {
-  value: number;
-  legend?: string;
-  title?: string;
-  unit?: string;
-}
+import type { ScoreCardOptions } from './types.js';
 
 class ScoreCardMacro extends BaseMacro {
   constructor(tasksQueue: TaskQueue) {

package/src/macros/scoreCard/types.ts

@@ -0,0 +1,32 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+export interface ScoreCardOptions {
+  /**
+   * Value shown in score card
+   */
+  value: number;
+  /**
+   * Optional legend text
+   */
+  legend?: string;
+  /**
+   * Option title text
+   */
+  title?: string;
+  /**
+   * Option unit that is next to the value
+   */
+  unit?: string;
+}

package/src/macros/types.ts

@@ -0,0 +1,48 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+import { CreateCardsOptions } from './createCards/types.js';
+import { GraphOptions } from './graph/types.js';
+import { ImageMacroOptions } from './image/types.js';
+import { IncludeMacroOptions } from './include/types.js';
+import { PercentageOptions } from './percentage/types.js';
+import { ReportOptions } from './report/types.js';
+import { ScoreCardOptions } from './scoreCard/types.js';
+import { VegaMacroInput } from './vega/types.js';
+import { VegaLiteMacroInput } from './vegalite/types.js';
+import { XrefMacroOptions } from './xref/types.js';
+
+export type AnyMacroOption =
+  | CreateCardsOptions
+  | GraphOptions
+  | ImageMacroOptions
+  | IncludeMacroOptions
+  | PercentageOptions
+  | ReportOptions
+  | ScoreCardOptions
+  | VegaMacroInput
+  | VegaLiteMacroInput
+  | XrefMacroOptions;
+
+export {
+  CreateCardsOptions,
+  GraphOptions,
+  ImageMacroOptions,
+  IncludeMacroOptions,
+  PercentageOptions,
+  ReportOptions,
+  ScoreCardOptions,
+  VegaMacroInput,
+  VegaLiteMacroInput,
+  XrefMacroOptions,
+};

package/src/macros/vega/index.ts

@@ -21,10 +21,7 @@ import {
   validateMacroContent,
 } from '../index.js';
 import * as vega from 'vega';
-
-export interface VegaMacroInput {
-  spec: vega.Spec;
-}
+import type { VegaMacroInput } from './types.js';
 
 class VegaMacro extends BaseMacro {
   constructor(tasksQueue: TaskQueue) {

package/src/macros/vega/types.ts

@@ -0,0 +1,21 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+import type { Spec } from 'vega';
+
+export interface VegaMacroInput {
+  /**
+   * Defines spec for vega
+   */
+  spec: Spec;
+}

package/src/macros/vegalite/index.ts

@@ -16,10 +16,7 @@ import macroMetadata from './metadata.js';
 import type TaskQueue from '../task-queue.js';
 import * as vegaLite from 'vega-lite';
 import { createMacro, validateMacroContent } from '../index.js';
-
-export interface VegaLiteMacroInput {
-  spec: vegaLite.TopLevelSpec;
-}
+import type { VegaLiteMacroInput } from './types.js';
 
 class VegaLiteMacro extends BaseMacro {
   constructor(tasksQueue: TaskQueue) {

package/src/macros/vegalite/types.ts

@@ -0,0 +1,22 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import type { TopLevelSpec } from 'vega-lite';
+
+export interface VegaLiteMacroInput {
+  /**
+   * Defines spec for vega-lite
+   */
+  spec: TopLevelSpec;
+}

package/src/macros/xref/index.ts

@@ -17,10 +17,7 @@ import type { MacroGenerationContext } from '../../interfaces/macros.js';
 import macroMetadata from './metadata.js';
 import BaseMacro from '../base-macro.js';
 import type TaskQueue from '../task-queue.js';
-
-export interface XrefMacroOptions {
-  cardKey: string;
-}
+import type { XrefMacroOptions } from './types.js';
 
 export default class XrefMacro extends BaseMacro {
   constructor(tasksQueue: TaskQueue) {
@@ -36,7 +33,7 @@ export default class XrefMacro extends BaseMacro {
     input: unknown,
   ): Promise<string> => {
     const options = this.validate(input);
-    const card = await this.getCard(options.cardKey, context);
+    const card = this.getCard(options.cardKey, context);
 
     if (!card || !card.metadata) {
       throw new Error(`Card key ${options.cardKey} not found`);
@@ -48,23 +45,12 @@ export default class XrefMacro extends BaseMacro {
 
   handleInject = async (context: MacroGenerationContext, input: unknown) => {
     const options = this.validate(input);
-    const card = await this.getCard(options.cardKey, context);
-
-    if (!card || !card.metadata) {
-      throw new Error(`Card key ${options.cardKey} not found`);
-    }
-
-    return `xref:${options.cardKey}.adoc[${card.metadata.title}]`;
+    const card = this.getCard(options.cardKey, context);
+    return `xref:${options.cardKey}.adoc[${card?.metadata?.title}]`;
   };
 
-  private async getCard(cardKey: string, context: MacroGenerationContext) {
-    const card = await context.project.cardDetailsById(cardKey, {
-      metadata: true,
-    });
-    if (!card || !card.metadata) {
-      throw new Error(`Card key ${cardKey} not found`);
-    }
-    return card;
+  private getCard(cardKey: string, context: MacroGenerationContext) {
+    return context.project.findCard(cardKey);
   }
 
   private validate(input: unknown): XrefMacroOptions {

package/src/macros/xref/types.ts

@@ -0,0 +1,20 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+export interface XrefMacroOptions {
+  /**
+   * Card of the card being referenced
+   */
+  cardKey: string;
+}

package/src/module-manager.ts

@@ -26,7 +26,7 @@ import { Project } from './containers/project.js';
 import type { ProjectConfiguration } from './project-settings.js';
 import { ProjectPaths } from './containers/project/project-paths.js';
 import { readJsonFile } from './utils/json.js';
-import { Validate } from './commands/index.js';
+import { Validate } from './commands/validate.js';
 
 const FILE_PROTOCOL = 'file:';
 const HTTPS_PROTOCOL = 'https:';
@@ -54,7 +54,7 @@ export class ModuleManager {
     await copyDir(sourcePath, destinationPath);
 
     // Update the resources.
-    await this.project.collectModuleResources();
+    this.project.resources.changedModules();
   }
 
   // Creates a map of what dependencies each module depend from.
@@ -213,13 +213,11 @@ export class ModuleManager {
 
   // Fetches direct dependencies of a module.
   private async dependencies(moduleName: string): Promise<Set<string>> {
-    const allModules = await this.project.modules();
-    if (!allModules) return new Set();
-    const module = allModules.find((m) => m.name === moduleName);
+    const module = await this.project.module(moduleName);
     if (!module) {
       throw new Error(`Module '${moduleName}' not found`);
     }
-    const modulePath = join(module.path, module.name, 'cardsConfig.json');
+    const modulePath = join(module.path, 'cardsConfig.json');
     const moduleConfiguration = (await readJsonFile(
       modulePath,
     )) as ProjectConfiguration;
@@ -287,7 +285,7 @@ export class ModuleManager {
 
   // Imports from a given folder. Is used both for .temp/<module name> and file locations.
   private async importFromFolder(path: string, name: string) {
-    await this.importFileModule(path);
+    await this.importFileModule(path, undefined, true); // Skip validation during updates
     console.log(
       `... Imported module '${name}' to '${this.project.configuration.name}'`,
     );
@@ -309,10 +307,10 @@ export class ModuleManager {
 
   // Collect modules that could be removed from .cards/modules when
   // 'moduleName' is removed.
-  private async orphanedModules(
+  private orphanedModules(
     dependencies: DependencyGraph,
     moduleName: string,
-  ): Promise<string[]> {
+  ): string[] {
     const projectModules = this.project.configuration.modules;
     const removableTransientModules: string[] = [];
     if (dependencies.has(moduleName)) {
@@ -361,6 +359,9 @@ export class ModuleManager {
     try {
       await this.removeModuleFiles(module.name);
       console.log(`... Removed imported module '${module.name}'`);
+
+      // Refresh module resources in cache after filesystem removal to avoid stale prefixes
+      this.project.resources.changedModules();
     } catch (error) {
       if (error instanceof Error)
         console.error(
@@ -479,7 +480,11 @@ export class ModuleManager {
   }
 
   // Updates modules in the project.
-  private async update(module?: ModuleSetting, credentials?: Credentials) {
+  private async update(
+    module?: ModuleSetting,
+    credentials?: Credentials,
+    skipModules?: Set<string>,
+  ) {
     // Prints dots every half second so that user knows that something is ongoing
     function start() {
       console.log('... Collecting unique modules. This takes a moment.');
@@ -514,22 +519,46 @@ export class ModuleManager {
         uniqueModules.map((item) => item.name),
       );
 
+      // Filter out modules that are already imported
+      const modulesToImport = skipModules
+        ? uniqueModules.filter((module) => !skipModules.has(module.name))
+        : uniqueModules;
+
+      if (
+        skipModules &&
+        skipModules.size > 0 &&
+        modulesToImport.length < uniqueModules.length
+      ) {
+        const skippedModules = uniqueModules
+          .filter((module) => skipModules.has(module.name))
+          .map((m) => m.name)
+          .join(', ');
+        console.log(
+          `... Skipping already imported module(s): ${skippedModules}`,
+        );
+      }
+
       // Update modules parallel.
       const promises: Promise<void>[] = [];
-      uniqueModules.forEach((module) =>
+      modulesToImport.forEach((module) =>
         promises.push(this.handleModule(module)),
       );
       await Promise.all(promises);
       await deleteDir(this.tempModulesDir);
-      await this.project.collectModuleResources();
+      this.project.resources.changedModules();
     }
   }
 
   // Checks that module prefix is not in use in the project
-  private async validatePrefix(modulePrefix: string) {
+  // Optionally skip check for modules that are already imported (during updates)
+  private validatePrefix(modulePrefix: string, skipIfExists = false) {
     // Do not allow modules with same prefixes.
-    const currentlyUsedPrefixes = await this.project.projectPrefixes();
+    const currentlyUsedPrefixes = this.project.projectPrefixes();
     if (currentlyUsedPrefixes.includes(modulePrefix)) {
+      // If skipIfExists is true, allow re-importing modules that are already present
+      if (skipIfExists) {
+        return;
+      }
       throw new Error(
         `Imported project has a prefix '${modulePrefix}' that is already used in the project. Cannot import from module.`,
       );
@@ -540,8 +569,14 @@ export class ModuleManager {
    * Imports module from local file path.
    * @param source Path to import from.
    * @param destination is this really needed???
+   * @param skipValidation Skip prefix validation (used during updates)
+   * @returns Module prefix of the imported module.
    */
-  public async importFileModule(source: string, destination?: string) {
+  public async importFileModule(
+    source: string,
+    destination?: string,
+    skipValidation = false,
+  ) {
     if (!Validate.validateFolder(source)) {
       throw new Error(
         `Input validation error: folder name is invalid '${source}'`,
@@ -558,6 +593,7 @@ export class ModuleManager {
       );
     }
     const sourceProject = new Project(source);
+    await sourceProject.populateCaches();
     const modulePrefix = sourceProject.projectPrefix;
     const destinationPath = join(
       this.project.paths.modulesFolder,
@@ -565,7 +601,7 @@ export class ModuleManager {
     );
     const sourcePath = sourceProject.paths.resourcesFolder;
 
-    await this.validatePrefix(modulePrefix);
+    this.validatePrefix(modulePrefix, skipValidation);
 
     // Copy files.
     await this.addFileContents(sourcePath, destinationPath);
@@ -577,12 +613,14 @@ export class ModuleManager {
    * @param source Git URL to import from.
    * @param options Modules setting options.
    * @param credentials Credentials for private repositories.
+   * @param skipValidation Skip prefix validation (used during updates)
    * @returns module prefix as defined in its CardsConfig.json
    */
   public async importGitModule(
     source: string,
     options?: ModuleSettingOptions,
     credentials?: Credentials,
+    skipValidation = false,
   ) {
     const clonedName = await this.clone(
       {
@@ -595,7 +633,7 @@ export class ModuleManager {
     );
     const clonePath = join(this.tempModulesDir, clonedName);
     const modulePrefix = (await this.configuration(clonePath)).cardKeyPrefix;
-    await this.validatePrefix(modulePrefix);
+    this.validatePrefix(modulePrefix, skipValidation);
 
     const sourcePath = new ProjectPaths(clonePath).resourcesFolder;
     const destinationPath = join(
@@ -611,6 +649,7 @@ export class ModuleManager {
    * If module is not used by any other modules, then will remove the module from disk as well.
    * Otherwise, only updates project configuration.
    * @param moduleName Name of the module to remove
+   * @throws If module was not found.
    */
   public async removeModule(moduleName: string) {
     const projectModules = this.project.configuration.modules;
@@ -625,7 +664,7 @@ export class ModuleManager {
     // but modules under .cards/modules must be checked not to be used by
     // other modules.
     if (this.canBeRemoved(dependencies, moduleName)) {
-      const orphans = await this.orphanedModules(dependencies, moduleName);
+      const orphans = this.orphanedModules(dependencies, moduleName);
       await deleteDir(module.path);
       for (const moduleToDelete of orphans) {
         const modulePath = join(
@@ -634,19 +673,37 @@ export class ModuleManager {
         );
         await deleteDir(modulePath);
       }
-      await this.project.collectModuleResources();
     }
-    await this.project.configuration.removeModule(moduleName);
+    await this.project.removeModule(moduleName);
+  }
+
+  /**
+   * Updates dependencies for a module without re-importing the module itself.
+   * Used during module import to fetch dependencies after the main module is already imported.
+   * @param module Module whose dependencies should be updated.
+   * @param credentials Optional credentials for private repositories.
+   * @returns Module prefix as defined in its CardsConfig.json
+   */
+  public async updateDependencies(
+    module: ModuleSetting,
+    credentials?: Credentials,
+  ) {
+    return this.update(module, credentials, new Set([module.name]));
   }
 
   /**
    * Imports module from a local file path or a git URL.
    * @param module Module to update. If not provided, updates all modules.
    * @param credentials Optional credentials for private repositories.
+   * @param skipModules Optional set of module names to skip during import.
    * @returns Module prefix as defined in its CardsConfig.json
    */
-  public async updateModule(module: ModuleSetting, credentials?: Credentials) {
-    return this.update(module, credentials);
+  public async updateModule(
+    module: ModuleSetting,
+    credentials?: Credentials,
+    skipModules?: Set<string>,
+  ) {
+    return this.update(module, credentials, skipModules);
   }
 
   /**