@cyberismo/data-handler 0.0.5 → 0.0.7

package/src/command-handler.ts

@@ -20,6 +20,7 @@ import type {
   Card,
   CardAttachment,
   CardListContainer,
+  Credentials,
   FileContentType,
   ModuleContent,
   ProjectMetadata,
@@ -43,13 +44,17 @@ import { errorFunction } from './utils/log-utils.js';
 import { readJsonFile } from './utils/json.js';
 import { resourceName } from './utils/resource-utils.js';
 
+import { type Level } from 'pino';
+
 // Generic options interface
 export interface CardsOptions {
   details?: boolean;
   forceStart?: boolean;
+  watchResourceChanges?: boolean;
   projectPath?: string;
   repeat?: number;
   showUse?: boolean;
+  logLevel?: Level;
 }
 
 // Commands that this class supports.
@@ -90,6 +95,7 @@ export class Commands {
   private commands?: CommandManager;
   private projectPath: string;
   private validateCmd: Validate;
+  private level?: Level;
 
   constructor() {
     this.projectPath = '';
@@ -112,19 +118,20 @@ export class Commands {
     command: Cmd,
     args: string[],
     options: CardsOptions,
+    credentials?: Credentials,
   ): Promise<requestStatus> {
     // Set project path and validate it.
     const creatingNewProject = command === Cmd.create && args[0] === 'project';
     if (!creatingNewProject) {
       try {
-        await this.doSetProject(options.projectPath || '');
+        await this.doSetProject(options);
       } catch (error) {
         return { statusCode: 400, message: errorFunction(error) };
       }
     } else {
       this.projectPath = options.projectPath || '';
     }
-    return await this.doHandleCommand(command, args, options);
+    return await this.doHandleCommand(command, args, options, credentials);
   }
 
   // Returns command 'target'.
@@ -143,7 +150,8 @@ export class Commands {
   }
 
   // Handles initializing the project so that it can be used in the class.
-  private async doSetProject(path: string) {
+  private async doSetProject(options: CardsOptions) {
+    const path = options.projectPath || '';
     this.projectPath = resolveTilde(await this.setProjectPath(path));
     if (!Validate.validateFolder(this.projectPath)) {
       let errorMessage = '';
@@ -159,7 +167,10 @@ export class Commands {
       throw new Error(`Input validation error: cannot find project '${path}'`);
     }
 
-    this.commands = await CommandManager.getInstance(this.projectPath);
+    this.commands = await CommandManager.getInstance(this.projectPath, {
+      logLevel: options.logLevel,
+      watchResourceChanges: options.watchResourceChanges,
+    });
     if (!this.commands) {
       throw new Error('Cannot get instance of CommandManager');
     }
@@ -171,6 +182,7 @@ export class Commands {
     command: Cmd,
     args: string[],
     options: CardsOptions,
+    credentials?: Credentials,
   ) {
     try {
       if (command === Cmd.add) {
@@ -263,6 +275,7 @@ export class Commands {
             source,
             branch,
             useCredentials && useCredentials === 'true' ? true : false,
+            credentials,
           );
         }
         if (target === 'csv') {
@@ -334,7 +347,7 @@ export class Commands {
           newValue ? JSON.parse(newValue) : undefined,
         );
       } else if (command === Cmd.updateModules) {
-        await this.commands?.importCmd.updateAllModules();
+        await this.commands?.importCmd.updateAllModules(credentials);
       } else if (command === Cmd.validate) {
         return this.validate();
       } else {
@@ -449,11 +462,6 @@ export class Commands {
         destination,
         parentCardKey,
       );
-    } else if (format === 'site') {
-      message = await this.commands?.exportSiteCmd.exportToSite(
-        destination,
-        parentCardKey,
-      );
     }
     process.env.EXPORT_FORMAT = '';
     return { statusCode: 200, message: message };
@@ -474,10 +482,12 @@ export class Commands {
     source: string,
     branch?: string,
     useCredentials?: boolean,
+    credentials?: Credentials,
   ) {
     return this.commands?.importCmd.importModule(source, this.projectPath, {
       branch: branch,
       private: useCredentials,
+      credentials,
     });
   }
 
@@ -509,9 +519,9 @@ export class Commands {
   // Runs a given logic program along with the query-language
   private async runLogicProgram(filePath: string): Promise<requestStatus> {
     try {
-      const res = await this.commands?.calculateCmd.runLogicProgram({
-        query: readFileSync(filePath, 'utf-8'),
-      });
+      const res = await this.commands?.calculateCmd.runLogicProgram(
+        readFileSync(filePath, 'utf-8'),
+      );
       return {
         statusCode: 200,
         payload: res,

package/src/command-manager.ts

@@ -15,7 +15,6 @@ import {
   Create,
   Edit,
   Export,
-  ExportSite,
   Import,
   Move,
   Remove,
@@ -26,6 +25,14 @@ import {
   Validate,
 } from './commands/index.js';
 import { Project } from './containers/project.js';
+import { ProjectPaths } from './containers/project/project-paths.js';
+import pino, { type Level, type TransportTargetOptions } from 'pino';
+import { setLogger } from './utils/log-utils.js';
+
+export interface CommandManagerOptions {
+  watchResourceChanges?: boolean;
+  logLevel?: Level;
+}
 
 // Handles commands and ensures that no extra instances are created.
 export class CommandManager {
@@ -35,7 +42,6 @@ export class CommandManager {
   public calculateCmd: Calculate;
   public createCmd: Create;
   public editCmd: Edit;
-  public exportSiteCmd: ExportSite;
   public exportCmd: Export;
   public importCmd: Import;
   public moveCmd: Move;
@@ -46,26 +52,24 @@ export class CommandManager {
   public updateCmd: Update;
   public validateCmd: Validate;
 
-  constructor(path: string) {
-    this.project = new Project(path);
+  private pathHandler: ProjectPaths;
+
+  constructor(path: string, options?: CommandManagerOptions) {
+    this.project = new Project(path, options?.watchResourceChanges);
     this.validateCmd = Validate.getInstance();
 
-    this.showCmd = new Show(this.project);
     this.calculateCmd = new Calculate(this.project);
+    this.showCmd = new Show(this.project, this.calculateCmd);
     this.createCmd = new Create(this.project, this.calculateCmd);
     this.editCmd = new Edit(this.project, this.calculateCmd);
     this.exportCmd = new Export(this.project, this.calculateCmd, this.showCmd);
-    this.exportSiteCmd = new ExportSite(
-      this.project,
-      this.calculateCmd,
-      this.showCmd,
-    );
     this.importCmd = new Import(this.project, this.createCmd);
     this.moveCmd = new Move(this.project, this.calculateCmd);
     this.removeCmd = new Remove(this.project, this.calculateCmd);
     this.renameCmd = new Rename(this.project, this.calculateCmd);
     this.transitionCmd = new Transition(this.project, this.calculateCmd);
     this.updateCmd = new Update(this.project);
+    this.pathHandler = new ProjectPaths(path);
   }
 
   /**
@@ -76,24 +80,61 @@ export class CommandManager {
     await this.project.collectModuleResources();
   }
 
+  /**
+   * Sets the logger for the command manager.
+   * @param level Log level.
+   */
+  public async setLogger(level: Level) {
+    const all: TransportTargetOptions[] = [
+      {
+        target: 'pino/file',
+        level: 'trace',
+        options: { destination: this.pathHandler.logPath, mkdir: true },
+      },
+      {
+        target: 'pino/file',
+        level: level,
+        options: { destination: 1 }, // stdout
+      },
+    ];
+
+    setLogger(
+      pino.default({
+        level: 'trace',
+        transport: {
+          targets: all,
+        },
+      }),
+    );
+  }
+
   /**
    * Either creates a new instance, or passes the current one.
    * New instance is created, if path differs, or there is no previous instance.
    * @param path Project path.
+   * @param watchResourceChanges Optional. If true, file changes are watched.
    * @returns Instance of this class.
    */
-  public static async getInstance(path: string): Promise<CommandManager> {
+  public static async getInstance(
+    path: string,
+    options?: CommandManagerOptions,
+  ): Promise<CommandManager> {
     if (
       CommandManager.instance &&
       CommandManager.instance.project.basePath !== path
     ) {
-      CommandManager.instance = new CommandManager(path);
+      CommandManager.instance.project.dispose();
+      CommandManager.instance = new CommandManager(path, options);
       await CommandManager.instance.initialize();
     }
     if (!CommandManager.instance) {
-      CommandManager.instance = new CommandManager(path);
+      CommandManager.instance = new CommandManager(path, options);
       await CommandManager.instance.initialize();
     }
+
+    if (options?.logLevel) {
+      await CommandManager.instance.setLogger(options?.logLevel);
+    }
     return CommandManager.instance;
   }
 }

package/src/commands/calculate.ts

@@ -13,9 +13,9 @@
 
 // node
 import { basename, join, resolve } from 'node:path';
-import { readFile, rm } from 'node:fs/promises';
-import { spawnSync } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
 import { sanitizeSvgBase64 } from '../utils/sanitize-svg.js';
+import { instance } from '@viz-js/viz';
 
 import type {
   BaseResult,
@@ -30,7 +30,7 @@ import { Mutex } from 'async-mutex';
 import Handlebars from 'handlebars';
 import { type Project, ResourcesFrom } from '../containers/project.js';
 import { flattenCardArray } from '../utils/card-utils.js';
-import { logger } from '../utils/log-utils.js';
+import { getChildLogger } from '../utils/log-utils.js';
 import {
   createCardFacts,
   createCardTypeFacts,
@@ -43,7 +43,6 @@ import {
   createWorkflowFacts,
 } from '../utils/clingo-facts.js';
 import { CardMetadataUpdater } from '../card-metadata-updater.js';
-import { generateRandomString } from '../utils/random.js';
 import type {
   CardType,
   FieldType,
@@ -53,7 +52,8 @@ import type {
   Workflow,
 } from '../interfaces/resource-interfaces.js';
 import { solve, setBaseProgram } from '@cyberismo/node-clingo';
-import { lpFiles } from '@cyberismo/resources';
+import { generateReportContent } from '../utils/report.js';
+import { lpFiles, graphvizReport } from '@cyberismo/assets';
 
 // Define names for the base programs
 const BASE_PROGRAM_KEY = 'base';
@@ -61,9 +61,13 @@ const QUERY_LANGUAGE_KEY = 'queryLanguage';
 
 // Class that calculates with logic program card / project level calculations.
 export class Calculate {
+  private get logger() {
+    return getChildLogger({
+      module: 'calculate',
+    });
+  }
   private static mutex = new Mutex();
 
-  private pythonBinary: string = 'python';
   constructor(private project: Project) {}
 
   // Storage for in-memory program content
@@ -79,7 +83,7 @@ export class Calculate {
       const modules = await this.generateModules();
       this.modules = modules; // Store initial modules in logicProgram
     } catch (error) {
-      logger.error(`Failed to initialize modules: ${error}`);
+      this.logger.error(error, 'Failed to initialize modules');
     }
   }
 
@@ -201,8 +205,9 @@ export class Calculate {
             content += `${moduleContent}\n`;
             content += `% SECTION: MODULE_${calculationFile.name}_END\n\n`;
           } catch (error) {
-            logger.warn(
-              `Failed to read module ${calculationFile.name}: ${error}`,
+            this.logger.warn(
+              error,
+              `Failed to read module ${calculationFile.name}`,
             );
           }
         }
@@ -292,43 +297,37 @@ export class Calculate {
     return parser.parseInput(data.join('\n'));
   }
 
-  //
-  private async run(
-    data: {
-      query?: string;
-    },
-    argMode: 'graph' | 'query' = 'query',
-  ): Promise<string[]> {
-    if (!data.query) {
-      throw new Error('Must provide query to run a clingo program');
-    }
-
-    const res = await Calculate.mutex.runExclusive(async () => {
-      // For queries, use both base and queryLanguage
-      const basePrograms =
-        argMode === 'query'
-          ? [BASE_PROGRAM_KEY, QUERY_LANGUAGE_KEY]
-          : BASE_PROGRAM_KEY;
-
-      // Then solve with the program - need to pass the program as parameter
-      return solve(data.query as string, basePrograms);
-    });
-
-    logger.trace(
-      {
-        query: data.query,
-      },
-      `Ran Clingo solve command`,
-    );
+  private async run(query: string): Promise<string[]> {
+    try {
+      const res = await Calculate.mutex.runExclusive(async () => {
+        // For queries, use both base and queryLanguage
+        const basePrograms = [BASE_PROGRAM_KEY, QUERY_LANGUAGE_KEY];
+
+        this.logger.trace(
+          {
+            clingo: true,
+          },
+          'Solving',
+        );
 
-    if (res && res.answers && res.answers.length > 0) {
-      logger.trace({
-        result: res.answers,
+        // Then solve with the program - need to pass the program as parameter
+        return solve(query, basePrograms);
       });
-      return res.answers;
-    }
 
-    throw new Error('Failed to run Clingo solve. No answers returned.');
+      if (res && res.answers && res.answers.length > 0) {
+        return res.answers;
+      }
+      throw new Error('Failed to run Clingo solve. No answers returned.');
+    } catch (error) {
+      this.logger.error(
+        {
+          error,
+          query,
+        },
+        'Clingo solve failed',
+      );
+      throw error;
+    }
   }
 
   /**
@@ -337,6 +336,14 @@ export class Calculate {
    */
   public async generate(cardKey?: string) {
     await Calculate.mutex.runExclusive(async () => {
+      this.logger.trace(
+        {
+          clingo: true,
+          cardKey,
+        },
+        'Generating logic program',
+      );
+
       if (!this.modulesInitialized) {
         await this.initializeModules();
         this.modulesInitialized = true;
@@ -397,6 +404,13 @@ export class Calculate {
 
       // Also store the base program (without query language) for updates
       this.logicProgram = baseProgram;
+      this.logger.trace(
+        {
+          clingo: true,
+          cardKey,
+        },
+        'Logic program set',
+      );
     });
   }
 
@@ -489,70 +503,40 @@ export class Calculate {
    * @param timeout Maximum amount of milliseconds clingraph is allowed to run
    * @returns a base64 encoded image as a string
    */
-  public async runGraph(
-    data: { query?: string; file?: string },
-    timeout?: number,
-  ) {
-    const clingoOutput = await this.run(data, 'graph');
-
-    // unlikely we ever get a collision
-    const randomId = generateRandomString(36, 20);
-
-    const clingGraphArgs = [
-      '--out=render',
-      '--format=svg',
-      '--type=digraph',
-      `--name-format=${randomId}`,
-      `--dir=${this.project.paths.tempFolder}`,
-    ];
-
-    // python is only used for windows
-    const pythonArgs = [
-      '-c',
-      `from clingraph import main; import sys; sys.argv = ["sys.argv[0]", ${clingGraphArgs
-        .map((arg) => `'${arg.replace(/\\/g, '\\\\')}'`)
-        .join(',')}]; sys.exit(main())`,
-    ];
-
-    const clingraph = spawnSync(this.pythonBinary, pythonArgs, {
-      encoding: 'utf8',
-      // below looks like it could be replaced by just joining dots, but
-      // it breaks it because elements contain newlines, which must be
-      // replaced by dots
-      input: clingoOutput.join('\n').replaceAll('\n', '.') + '.',
-      timeout,
-      maxBuffer: 1024 * 1024 * 100,
-    });
-
-    if (clingraph.status !== 0) {
-      throw new Error(`Graph: Failed to run clingraph ${clingraph.stderr}`);
-    }
-
-    const filePath = join(this.project.paths.tempFolder, randomId);
-
-    let fileData;
-    try {
-      fileData = await readFile(filePath + '.svg');
-    } catch (e) {
-      throw new Error(
-        `Graph: Failed to read image file after generating graph: ${e}`,
-      );
-    } finally {
-      await rm(filePath, { force: true });
-      await rm(filePath + '.svg', { force: true });
-    }
+  public async runGraph(model: string, view: string) {
+    this.logger.trace(
+      {
+        model,
+        view,
+      },
+      'Running graph',
+    );
 
-    // Sanitizing SVG before returning it
-    return sanitizeSvgBase64(fileData);
+    // Let's run the clingraph query
+    const result = await generateReportContent({
+      calculate: this,
+      contentTemplate: graphvizReport.content,
+      queryTemplate: graphvizReport.query,
+      options: {
+        model: model,
+        view: view,
+      },
+      graph: true,
+    });
+    return sanitizeSvgBase64(
+      (await instance()).renderString(result, {
+        format: 'svg',
+      }),
+    );
   }
 
   /**
    * Runs a logic program using clingo.
-   * @param filePath Path to a query file to be run in relation to current working directory
+   * @param query Logic program to be run
    * @returns parsed program output
    */
-  public async runLogicProgram(data: { query?: string; file?: string }) {
-    const clingoOutput = await this.run(data);
+  public async runLogicProgram(query: string) {
+    const clingoOutput = await this.run(query);
 
     return this.parseClingoResult(clingoOutput);
   }
@@ -574,12 +558,12 @@ export class Calculate {
       content = compiled(options);
     }
 
-    const clingoOutput = await this.run(
-      {
-        query: content,
-      },
-      'query',
-    );
+    if (!content) {
+      throw new Error(`Query file ${queryName} not found`);
+    }
+
+    this.logger.trace({ queryName }, 'Running query');
+    const clingoOutput = await this.run(content);
 
     const result = await this.parseClingoResult(clingoOutput);
 

package/src/commands/export.ts

@@ -196,16 +196,17 @@ export class Export {
     );
 
     let asciiDocContent = '';
-    const projectPath = this.project.basePath;
+    const project = this.project;
     try {
       const { evaluateMacros } = await import('../macros/index.js');
       asciiDocContent = await evaluateMacros(
         cardDetailsResponse.content || '',
         {
           mode: 'static',
-          projectPath,
+          project,
           cardKey: card.key,
         },
+        this.calculateCmd,
       );
     } catch (error) {
       asciiDocContent = `Macro error: ${error instanceof Error ? error.message : 'Unknown error'}\n\n${asciiDocContent}`;

package/src/commands/import.ts

@@ -14,7 +14,10 @@
 import type { CardType } from '../interfaces/resource-interfaces.js';
 import { type Create, Validate } from './index.js';
 import { ModuleManager } from '../module-manager.js';
-import type { ModuleSettingOptions } from '../interfaces/project-interfaces.js';
+import type {
+  Credentials,
+  ModuleSettingOptions,
+} from '../interfaces/project-interfaces.js';
 import type { Project } from '../containers/project.js';
 import { readCsvFile } from '../utils/csv.js';
 import { resourceName } from '../utils/resource-utils.js';
@@ -26,7 +29,7 @@ export class Import {
     private project: Project,
     private createCmd: Create,
   ) {
-    this.moduleManager = new ModuleManager(this.project, this);
+    this.moduleManager = new ModuleManager(this.project);
   }
 
   /**
@@ -142,28 +145,25 @@ export class Import {
       );
     }
 
-    // Add module as a dependency.
-    return this.project.importModule({
+    const moduleSettings = {
       name: modulePrefix,
       branch: options ? options.branch : undefined,
       private: options ? options.private : undefined,
       location: gitModule ? source : `file:${source}`,
-    });
-  }
+    };
 
-  /**
-   * Updates all imported modules.
-   */
-  public async updateAllModules() {
-    return this.moduleManager.update();
+    // Fetch module dependencies.
+    await this.moduleManager.updateModule(moduleSettings, options?.credentials);
+
+    // Add module as a dependency.
+    return this.project.importModule(moduleSettings);
   }
 
   /**
-   * Updates 'moduleName' module from its source.
-   * Modules using gitUrl, are first copied to .temp
-   * @param moduleName module name (prefix) to update
+   * Updates all imported modules.
+   * @param credentials Optional credentials for private modules.
    */
-  public async updateExistingModule(moduleName: string) {
-    await this.moduleManager.importFileModule(moduleName);
+  public async updateAllModules(credentials?: Credentials) {
+    return this.moduleManager.updateModules(credentials);
   }
 }

package/src/commands/index.ts

@@ -15,7 +15,6 @@ import { Calculate } from './calculate.js';
 import { Create } from './create.js';
 import { Edit } from './edit.js';
 import { Export } from './export.js';
-import { ExportSite } from './export-site.js';
 import { Import } from './import.js';
 import { Move } from './move.js';
 import { Remove } from './remove.js';
@@ -30,7 +29,6 @@ export {
   Create,
   Edit,
   Export,
-  ExportSite,
   Import,
   Move,
   Remove,

package/src/commands/remove.ts

@@ -188,8 +188,8 @@ export class Remove {
     if (!module) {
       throw new Error(`Module '${moduleName}' not found`);
     }
-    await deleteDir(module.path);
     await this.project.removeModule(moduleName);
+    await deleteDir(module.path);
   }
 
   /**