@cyberismo/data-handler 0.0.5 → 0.0.7

package/src/macros/report/index.ts

@@ -1,35 +1,39 @@
 /**
-    Cyberismo
-    Copyright © Cyberismo Ltd and contributors 2024
-
-    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
-
-    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public
-    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2024
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { registerEmptyMacros, validateMacroContent } from '../index.js';
+import { validateMacroContent } from '../index.js';
 import type { MacroOptions } from '../index.js';
+
 import type { MacroGenerationContext } from '../../interfaces/macros.js';
 import macroMetadata from './metadata.js';
-import { Project } from '../../containers/project.js';
-import { Calculate } from '../../commands/index.js';
-import Handlebars from 'handlebars';
+import type { Calculate } from '../../commands/index.js';
 import BaseMacro from '../base-macro.js';
 import { validateJson } from '../../utils/validate.js';
 import type TaskQueue from '../task-queue.js';
 import { ReportResource } from '../../resources/report-resource.js';
 import { resourceName } from '../../utils/resource-utils.js';
+import { generateReportContent } from '../../utils/report.js';
 
 export interface ReportOptions extends MacroOptions {
   name: string;
 }
 
 class ReportMacro extends BaseMacro {
-  constructor(tasksQueue: TaskQueue) {
-    super(macroMetadata, tasksQueue);
+  constructor(
+    tasks: TaskQueue,
+    private readonly calculate: Calculate,
+  ) {
+    super(macroMetadata, tasks);
   }
   handleValidate = (input: unknown) => {
     this.validate(input);
@@ -41,11 +45,13 @@ class ReportMacro extends BaseMacro {
 
   handleInject = async (context: MacroGenerationContext, data: unknown) => {
     const options = this.validate(data);
-    const project = new Project(context.projectPath);
-    const resource = new ReportResource(project, resourceName(options.name));
+    const resource = new ReportResource(
+      context.project,
+      resourceName(options.name),
+    );
     const report = await resource.show();
 
-    if (!report) throw new Error(`Report ${options} does not exist`);
+    if (!report) throw new Error(`Report ${options.name} does not exist`);
 
     if (report.schema) {
       validateJson(options, {
@@ -53,30 +59,14 @@ class ReportMacro extends BaseMacro {
       });
     }
 
-    const handlebarsContext = {
-      cardKey: context.cardKey,
-      ...options,
-    };
-
-    const handlebars = Handlebars.create();
-
-    const template = handlebars.compile(report.queryTemplate, {
-      strict: true,
-    });
-
-    const calculate = new Calculate(project);
-    const result = await calculate.runLogicProgram({
-      query: template(handlebarsContext),
-    });
-    if (result.error) {
-      throw new Error(result.error);
-    }
-    // register empty macros so that other macros aren't touched yet
-    registerEmptyMacros(handlebars);
-
-    return handlebars.compile(report.contentTemplate)({
-      ...handlebarsContext,
-      ...result,
+    return generateReportContent({
+      calculate: this.calculate,
+      contentTemplate: report.contentTemplate,
+      queryTemplate: report.queryTemplate,
+      options: {
+        cardKey: context.cardKey,
+        ...options,
+      },
     });
   };
 

package/src/module-manager.ts

@@ -11,16 +11,14 @@
   License along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
 
-import fs from 'node:fs';
 import { join } from 'node:path';
 import { mkdir, readdir, rm } from 'node:fs/promises';
 
-import git from 'isomorphic-git';
-import http from 'isomorphic-git/http/node/index.js';
+import { simpleGit, type SimpleGit } from 'simple-git';
 
 import { copyDir, deleteDir, pathExists } from './utils/file-utils.js';
-import type { Import } from './commands/index.js';
 import type {
+  Credentials,
   ModuleSetting,
   ModuleSettingOptions,
 } from './interfaces/project-interfaces.js';
@@ -33,6 +31,8 @@ import { Validate } from './commands/index.js';
 const FILE_PROTOCOL = 'file:';
 // todo: add support for git's default branch.
 const MAIN_BRANCH = 'main';
+// timeout in milliseconds for git client (no stdout / stderr activity)
+const DEFAULT_TIMEOUT = 10000;
 
 /**
  * Class that handles module updates and imports.
@@ -40,10 +40,7 @@ const MAIN_BRANCH = 'main';
 export class ModuleManager {
   private modules: ModuleSetting[] = [];
   private tempModulesDir: string = '';
-  constructor(
-    private project: Project,
-    private importCmd: Import,
-  ) {
+  constructor(private project: Project) {
     this.tempModulesDir = join(this.project.paths.tempFolder, 'modules');
   }
 
@@ -56,87 +53,82 @@ export class ModuleManager {
     await this.project.collectModuleResources();
   }
 
-  // Handles a branch of a repository.
-  private async branch(module: ModuleSetting) {
-    if (module.branch === MAIN_BRANCH || module.branch === '' || !module.branch)
-      return;
-
-    await git.checkout({
-      fs,
-      dir: join(this.tempModulesDir, module.name),
-      ref: module.branch,
-    });
-    console.error(
-      `... Switched to '${module.branch}' branch for module '${module.name}'`,
-    );
-  }
-
   // Handles cloning of a repository.
   private async clone(
     module: ModuleSetting,
     verbose: boolean = true,
+    credentials?: Credentials,
   ): Promise<string> {
     if (!module.name || module.name === '') {
       module.name = this.repositoryName(module.location);
     }
 
-    let repoUrl: URL;
-    try {
-      repoUrl = new URL(module.location);
-    } catch {
-      throw new Error(`Invalid repository URL: ${module.location}`);
+    const destinationPath = join(this.tempModulesDir, module.name);
+
+    let remote = module.location;
+    if (module.private) {
+      if (credentials && credentials?.username && credentials?.token) {
+        if (verbose) {
+          console.log(
+            `... Using credentials '${credentials?.username}' for cloning '${module.name}'`,
+          );
+        }
+        try {
+          const repoUrl = new URL(module.location);
+          const user = credentials?.username;
+          const pass = credentials?.token;
+          const host = repoUrl.host;
+          const path = repoUrl.pathname;
+          remote = `https://${user}:${pass}@${host}${path}`;
+        } catch {
+          throw new Error(`Invalid repository URL: ${module.location}`);
+        }
+      } else {
+        if (verbose) {
+          console.log(`... Not using credentials for cloning '${module.name}'`);
+        }
+      }
     }
 
-    if (
-      process.env.CYBERISMO_GIT_USER &&
-      process.env.CYBERISMO_GIT_TOKEN &&
-      module.private
-    ) {
+    try {
+      await mkdir(this.tempModulesDir, { recursive: true });
+      const cloneOptions = this.setCloneOptions(module);
+      await rm(destinationPath, { recursive: true, force: true });
+
+      const git: SimpleGit = simpleGit({
+        timeout: {
+          block: DEFAULT_TIMEOUT,
+        },
+      });
+
+      await git
+        // turn off git prompts
+        .env({ GIT_TERMINAL_PROMPT: 0, GCM_INTERACTIVE: 'never' })
+        .clone(remote, destinationPath, cloneOptions);
+
       if (verbose) {
-        console.log(
-          `... Using credentials '${process.env.CYBERISMO_GIT_USER}' for cloning '${module.name}'`,
-        );
+        console.log(`... Cloned '${module.name}' to a temporary folder`);
       }
-      repoUrl.username = process.env.CYBERISMO_GIT_USER;
-      repoUrl.password = process.env.CYBERISMO_GIT_TOKEN;
+    } catch (error) {
+      console.error(error);
+      if (error instanceof Error)
+        throw new Error(
+          `Failed to clone module '${module.name}': ${error.message}`,
+        );
     }
-    await git.clone({
-      fs,
-      http,
-      dir: join(this.tempModulesDir, module.name),
-      url: repoUrl.toString(),
-      depth: 1,
-      onAuth: () => {
-        // Turn credentials 'off' when they are not available
-        if (
-          !process.env.CYBERISMO_GIT_USER ||
-          !process.env.CYBERISMO_GIT_TOKEN
-        ) {
-          return undefined;
-        }
-        // Turn credentials 'off' for public repos
-        if (!module.private) {
-          return undefined;
-        }
-        return {
-          username: process.env.CYBERISMO_GIT_USER,
-          password: process.env.CYBERISMO_GIT_TOKEN,
-        };
-      },
-    });
 
-    if (verbose) {
-      console.log(`... Cloned '${module.name}' to a temporary folder`);
-    }
     return module.name;
   }
 
   // Collects all module prefixes from module hierarchy into 'this.modules'.
   // Note that collected result can contain duplicates.
-  private async collectModulePrefixes(modules: ModuleSetting[]) {
+  private async collectModulePrefixes(
+    modules: ModuleSetting[],
+    credentials?: Credentials,
+  ) {
     if (modules) {
       for (const module of modules) {
-        await this.doCollectModulePrefix(module);
+        await this.doCollectModulePrefix(module, credentials);
       }
     }
   }
@@ -153,21 +145,24 @@ export class ModuleManager {
 
   // Collects one module's dependency prefixes to 'this.modules'.
   // Note that there can be duplicate entries.
-  private async doCollectModulePrefix(module: ModuleSetting) {
+  private async doCollectModulePrefix(
+    module: ModuleSetting,
+    credentials?: Credentials,
+  ) {
     let moduleRoot = '';
     if (this.isFileModule(module)) {
       const urlStart = FILE_PROTOCOL.length;
       // Remove 'file:' from location
       moduleRoot = module.location.substring(urlStart, module.location.length);
     } else {
-      await this.clone(module, false);
+      await this.clone(module, false, credentials);
       moduleRoot = join(this.tempModulesDir, module.name);
     }
 
     this.modules.push(module);
 
     const configuration = await this.configuration(moduleRoot);
-    await this.collectModulePrefixes(configuration.modules);
+    await this.collectModulePrefixes(configuration.modules, credentials);
   }
 
   // Updates one module that is read from local file system.
@@ -180,21 +175,20 @@ export class ModuleManager {
   // Updates one module that is received from Git.
   private async handleGitModule(module: ModuleSetting) {
     await this.clone(module);
-    await this.branch(module);
     await this.remove(module);
     await this.importFromTemp(module);
   }
 
   // Updates one module.
   private async handleModule(module: ModuleSetting) {
-    return this.isFileModule(module)
-      ? this.handleFileModule(module)
-      : this.handleGitModule(module);
+    return this.isGitModule(module)
+      ? this.handleGitModule(module)
+      : this.handleFileModule(module);
   }
 
   // Handles importing a module from module settings 'location'
   private async importFromFolder(module: ModuleSetting) {
-    await this.importCmd.updateExistingModule(module.location);
+    await this.importFileModule(module.location);
     console.log(
       `... Imported module '${module.name}' to '${this.project.configuration.name}'`,
     );
@@ -202,9 +196,7 @@ export class ModuleManager {
 
   // Handles importing a module from '.temp' folder
   private async importFromTemp(module: ModuleSetting) {
-    await this.importCmd.updateExistingModule(
-      join(this.tempModulesDir, module.name),
-    );
+    await this.importFileModule(join(this.tempModulesDir, module.name));
     console.log(
       `... Imported module '${module.name}' to '${this.project.configuration.name}'`,
     );
@@ -216,14 +208,26 @@ export class ModuleManager {
     return module.location.startsWith('file:');
   }
 
+  // Returns true if module is imported from git.
+  private isGitModule(module: ModuleSetting): boolean {
+    if (!module.location) return false;
+    return module.location.startsWith('https:');
+  }
+
   // Prepares '.temp/modules' for cloning
   private async prepare() {
-    await mkdir(this.tempModulesDir, { recursive: true });
-    for (const file of await readdir(this.tempModulesDir)) {
-      await rm(join(this.tempModulesDir, file), {
-        force: true,
-        recursive: true,
-      });
+    try {
+      await mkdir(this.tempModulesDir, { recursive: true });
+      const files = await readdir(this.tempModulesDir);
+      for (const file of files) {
+        const filePath = join(this.tempModulesDir, file);
+        await rm(filePath, {
+          force: true,
+          recursive: true,
+        });
+      }
+    } catch (error) {
+      throw new Error(`Failed to prepare temporary directory: ${error}`);
     }
   }
 
@@ -281,11 +285,14 @@ export class ModuleManager {
     for (const module of modules) {
       const existingModule = moduleMap.get(module.name);
       if (existingModule) {
-        if (existingModule.private !== module.private) {
+        if (
+          (existingModule.private && !module.private) ||
+          (!existingModule.private && module.private)
+        ) {
           throw new Error(
             `Module conflict: '${module.name}' has different access:\n` +
-              `  - ${existingModule.private || 'undefined'}\n` +
-              `  - ${module.private || 'undefined'}`,
+              `  - ${existingModule.private === true ? 'true' : 'false'}\n` +
+              `  - ${module.private === true ? 'true' : 'false'}`,
           );
         }
         if (existingModule.location !== module.location) {
@@ -319,6 +326,66 @@ export class ModuleManager {
     return repoName;
   }
 
+  // Sets cloning options.
+  private setCloneOptions(module: ModuleSetting) {
+    const cloneOptions = ['--depth', '1'];
+    if (
+      module.branch &&
+      module.branch !== '' &&
+      module.branch !== MAIN_BRANCH
+    ) {
+      cloneOptions.push('--branch', module.branch);
+    }
+    return cloneOptions;
+  }
+
+  // Updates modules in the project.
+  private async update(module?: ModuleSetting, credentials?: Credentials) {
+    // Prints dots every half second so that user knows that something is ongoing
+    function start() {
+      console.log('... Collecting unique modules. This takes a moment.');
+      return setInterval(() => process.stdout.write(`.`), 500);
+    }
+
+    // Stops the above, and shows results
+    function finished(interval: NodeJS.Timeout, modules: string[]) {
+      clearInterval(interval);
+      console.log(`\n... Found modules: ${modules.join(', ')}`);
+    }
+
+    await this.prepare();
+
+    let modules = module ? [module] : this.project.configuration.modules;
+    if (modules.length === 0) {
+      throw new Error(`No modules in the project!`);
+    }
+    modules = modules.filter((module) => this.isGitModule(module));
+
+    const dotInterval = start();
+
+    let uniqueModules: ModuleSetting[] = [];
+    try {
+      await this.collectModulePrefixes(modules, credentials);
+
+      uniqueModules = this.removeDuplicates(this.modules);
+    } finally {
+      finished(
+        dotInterval,
+        uniqueModules.map((item) => item.name),
+      );
+
+      // Update modules parallel.
+      const promises: Promise<void>[] = [];
+      uniqueModules.forEach((module) =>
+        promises.push(this.handleModule(module)),
+      );
+      await Promise.all(promises);
+
+      await deleteDir(this.tempModulesDir);
+      await this.project.collectModuleResources();
+    }
+  }
+
   // Checks that module prefix is not in use in the project
   private async validatePrefix(modulePrefix: string) {
     // Do not allow modules with same prefixes.
@@ -370,16 +437,24 @@ export class ModuleManager {
    * Imports module from gitUrl.
    * @param source Git URL to import from.
    * @param options Modules setting options.
+   * @param credentials Credentials for private repositories.
    * @returns module prefix as defined in its CardsConfig.json
    */
-  public async importGitModule(source: string, options?: ModuleSettingOptions) {
-    const repoName = await this.clone({
-      name: '',
-      location: source,
-      ...options,
-    });
-    await this.branch({ name: repoName, location: source, ...options });
-    const clonePath = join(this.project.paths.tempFolder, 'modules', repoName);
+  public async importGitModule(
+    source: string,
+    options?: ModuleSettingOptions,
+    credentials?: Credentials,
+  ) {
+    const clonedName = await this.clone(
+      {
+        name: this.repositoryName(source),
+        location: source,
+        ...options,
+      },
+      undefined,
+      credentials,
+    );
+    const clonePath = join(this.tempModulesDir, clonedName);
     const modulePrefix = (await this.configuration(clonePath)).cardKeyPrefix;
     await this.validatePrefix(modulePrefix);
 
@@ -393,51 +468,20 @@ export class ModuleManager {
   }
 
   /**
-   * Updates all imported modules.
+   * Imports module from a local file path or a git URL.
+   * @param module Module to update. If not provided, updates all modules.
+   * @param credentials Optional credentials for private repositories.
+   * @returns Module prefix as defined in its CardsConfig.json
    */
-  public async update() {
-    // Prints dots every half second so that user knows that something is ongoing
-    function start() {
-      console.log('... Collecting unique modules. This takes a moment.');
-      return setInterval(() => process.stdout.write(`.`), 500);
-    }
-
-    // Stops the above, and shows results
-    function finished(interval: NodeJS.Timeout, modules: string[]) {
-      clearInterval(interval);
-      console.log(`\n... Found modules: ${modules.join(', ')}`);
-    }
-
-    await this.prepare();
-
-    const modules = this.project.configuration.modules;
-    if (modules.length === 0) {
-      throw new Error(`No modules in the project!`);
-    }
-
-    const dotInterval = start();
-
-    // Collect prefixes from project's dependency modules.
-    await this.collectModulePrefixes(modules);
-
-    let uniqueModules: ModuleSetting[] = [];
-    try {
-      uniqueModules = this.removeDuplicates(this.modules);
-    } finally {
-      finished(
-        dotInterval,
-        uniqueModules.map((item) => item.name),
-      );
-
-      // Update modules parallel.
-      const promises: Promise<void>[] = [];
-      uniqueModules.forEach((module) =>
-        promises.push(this.handleModule(module)),
-      );
-      await Promise.all(promises);
+  public async updateModule(module: ModuleSetting, credentials?: Credentials) {
+    return this.update(module, credentials);
+  }
 
-      await deleteDir(this.tempModulesDir);
-      await this.project.collectModuleResources();
-    }
+  /**
+   * Updates all imported modules.
+   * @param credentials Optional credentials for private modules.
+   */
+  public async updateModules(credentials?: Credentials) {
+    return this.update(undefined, credentials);
   }
 }

package/src/project-settings.ts

@@ -13,6 +13,8 @@
 
 import { writeJsonFile as atomicWrite } from 'write-json-file';
 
+import { resolve } from 'path';
+
 import type {
   ModuleSetting,
   ProjectSettings,
@@ -103,6 +105,11 @@ export class ProjectConfiguration implements ProjectSettings {
     if (exists) {
       throw new Error(`Module '${module.name}' already imported`);
     }
+    // Ensure that module file location is absolute
+    if (module.location && module.location.startsWith('file:')) {
+      const filePath = module.location.substring(5, module.location.length);
+      module.location = `file:${resolve(filePath)}`;
+    }
     this.modules.push(module);
     return this.save();
   }

package/src/resources/array-handler.ts

@@ -83,15 +83,20 @@ export class ArrayHandler<T> {
     if (targetIndex === -1) {
       throw new Error(`Item '${JSON.stringify(target)}' not found`);
     }
+    const actualTarget = array[targetIndex];
     const parsedTo = this.tryParseJSON(to);
 
     if (typeof to === 'string' && (to.startsWith('[') || to.startsWith('{'))) {
       return parsedTo as T[];
     }
 
-    return array.map((item) =>
-      deepCompare(item as object, target as object) ? parsedTo : item,
+    const updatedArray = array.map((item) =>
+      deepCompare(item as object, actualTarget as object) ? parsedTo : item,
     );
+    if (deepCompare(updatedArray, array)) {
+      throw new Error('Cannot change value. Target was not found.');
+    }
+    return updatedArray;
   }
 
   private handleRank(operation: RankOperation<T>, array: T[]): T[] {