@cyberismo/data-handler 0.0.2

package/src/utils/csv.ts

@@ -0,0 +1,53 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { readFile } from 'fs/promises';
+import { parse } from 'csv-parse/sync';
+import type { CSVRowRaw } from '../interfaces/project-interfaces.js';
+
+/**
+ * Reads a CSV file and returns its content as an array of objects.
+ * @param file Path to the CSV file.
+ * @returns Array of objects. Each object represents a row in the CSV file.
+ */
+export async function readCsvFile(file: string): Promise<CSVRowRaw[]> {
+  const content = await readFile(file, {
+    encoding: 'utf-8',
+  });
+  const records = parse(content, {
+    bom: true,
+  });
+
+  if (!Array.isArray(records) || records.length < 2) {
+    throw new Error('CSV file must have headers');
+  }
+
+  const [headers, ...data] = records;
+
+  if (
+    !Array.isArray(headers) ||
+    new Set(headers).size !== headers.length ||
+    headers.length === 0
+  ) {
+    throw new Error('Error parsing header');
+  }
+
+  return data.map((row) => {
+    if (!Array.isArray(row)) {
+      throw new Error('Row is not an array');
+    }
+    return headers.reduce((acc, header, index) => {
+      acc[header] = row[index];
+      return acc;
+    }, {});
+  });
+}

package/src/utils/file-utils.ts

@@ -0,0 +1,182 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2024
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import {
+  copyFile,
+  mkdir,
+  readdir,
+  rm,
+  unlink,
+  writeFile,
+} from 'node:fs/promises';
+import { existsSync, readdirSync } from 'node:fs';
+import { dirname, join, sep } from 'node:path';
+import { homedir } from 'node:os';
+
+/**
+ * Works like the writeFile method, but ensures that the directory exists
+ * There is only one difference: This method only supports a string as the filePath
+ */
+export async function writeFileSafe(
+  filePath: string,
+  data: Parameters<typeof writeFile>[1],
+  options?: Parameters<typeof writeFile>[2],
+) {
+  const dir = dirname(filePath);
+  await mkdir(dir, {
+    recursive: true,
+  });
+  return writeFile(filePath, data, options);
+}
+
+/**
+ * Copies directory content (subdirectories and files) to destination.
+ * Note that it won't create 'source', but copies all that is inside of 'source'.
+ * @param source path to start from
+ * @param destination path where to copy to
+ */
+export async function copyDir(source: string, destination: string) {
+  const entries = await readdir(source, { withFileTypes: true });
+  for (const entry of entries) {
+    const sourcePath = join(source, entry.name);
+    const destinationPath = join(destination, entry.name);
+    if (entry.isDirectory()) {
+      await mkdir(destinationPath, { recursive: true });
+      await copyDir(sourcePath, destinationPath);
+    } else {
+      await mkdir(destination, { recursive: true });
+      await copyFile(sourcePath, destinationPath);
+    }
+  }
+}
+
+/**
+ * Delete directory.
+ * @param path path to be deleted
+ */
+export async function deleteDir(path: string) {
+  await rm(resolveTilde(path), { recursive: true, force: true });
+}
+
+/**
+ * Delete file.
+ * @param path path to file to be deleted
+ * @returns true, if file was deleted; false otherwise.
+ */
+export async function deleteFile(path: string): Promise<boolean> {
+  if (!path) {
+    return false;
+  }
+  try {
+    await unlink(path);
+  } catch {
+    console.error(`Cannot delete file '${path}'`);
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Removes extension from filename.
+ * @param filename Filename
+ * @returns filename without extension. If there was no extension, returns the original filename.
+ */
+export function stripExtension(filename: string) {
+  // First handle special cases. Just return the filename in all of these cases.
+  // 1) If the filename ends to "."" or ".." (e.g. ".cards/local/..")
+  const parts = filename.split(sep);
+  if (
+    parts.at(parts.length - 1) === '..' ||
+    parts.at(parts.length - 1) === '.'
+  ) {
+    return filename;
+  }
+  const dotLocation = filename.lastIndexOf('.');
+  const sepLocation = filename.lastIndexOf(sep);
+  // 2) If there is a dot in the filename before sep (e.g. ".cards/local")
+  if (dotLocation < sepLocation) {
+    return filename;
+  }
+  // 3) If there is a dot in filename but it is not an actual extension (e.g. "test/.filename", or ".filename")
+  if (dotLocation === 0 || filename.at(dotLocation - 1) === sep) {
+    return filename;
+  }
+
+  const noExtension = filename.split('.').slice(0, -1).join('.');
+  // if there was no extension at all, return the original file name.
+  return noExtension ? noExtension : filename;
+}
+
+/**
+ * Lists all files from a folder.
+ * @param path path to folder
+ * @param pathPrefix relative adjustment to 'path', if any; optional; by default empty.
+ * @param files currently collected files; optional; by default empty.
+ * @returns array of filenames that are in the folder or in one of its subfolders.
+ * @note that 'pathPrefix' and 'files' are generally only used in internal recursion.
+ *       When calling this from code, do not pass the parameters.
+ */
+export function getFilesSync(
+  path: string,
+  pathPrefix: string = '',
+  files: string[] = [],
+): string[] {
+  try {
+    for (const entry of readdirSync(path, { withFileTypes: true })) {
+      const relativePath = pathPrefix
+        ? join(pathPrefix, entry.name)
+        : entry.name;
+
+      if (entry.isFile()) {
+        files.push(relativePath);
+      } else if (entry.isDirectory()) {
+        getFilesSync(join(path, entry.name), relativePath, files);
+      }
+    }
+  } catch {
+    // do nothing, wrong path, or no permissions to read the files
+  }
+  return files;
+}
+
+/**
+ * Checks if file or folder exists.
+ * @param path file or folder path
+ * @returns true if file exists, otherwise false
+ */
+export function pathExists(path: string): boolean {
+  path = resolveTilde(path);
+  return existsSync(path);
+}
+
+/**
+ * Handles tilde from a path (ie. appends user home to its place).
+ * @param filePath Path to handle tilde from.
+ * @returns Path with tilde resolved.
+ */
+export function resolveTilde(filePath: string): string {
+  if (!filePath || typeof filePath !== 'string') {
+    return '';
+  }
+  // '~/folder/path' or '~' not '~alias/folder/path'
+  if (filePath.startsWith('~/') || filePath === '~') {
+    return filePath.replace('~', homedir());
+  }
+  return filePath;
+}
+
+/**
+ * Path separator RE.
+ */
+export const sepRegex = /[/\\]/;

package/src/utils/json.ts

@@ -0,0 +1,118 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2024
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { readFileSync } from 'node:fs';
+import { type FileHandle, readFile, writeFile } from 'node:fs/promises';
+
+/**
+ * Handles reading of a JSON file.
+ * @param file file name (and path) to read.
+ * @returns Parsed JSON content.
+ * @throws if file is not found, or file is not a JSON file.
+ */
+export function readJsonFileSync(file: string) {
+  try {
+    const raw = readFileSync(file, { flag: 'rs', encoding: 'utf-8' });
+    const returnValue = JSON.parse(raw);
+    return returnValue;
+  } catch (error) {
+    if (error instanceof Error) {
+      throw new Error(
+        `Error while handling JSON file '${file}' : ${error.message}`,
+      );
+    }
+  }
+}
+
+/**
+ * Handles reading of a JSON file.
+ * @param file file name (and path) to read.
+ * @returns Parsed JSON content.
+ * @throws if file is not found, or file is not a JSON file.
+ */
+export async function readJsonFile(file: string) {
+  try {
+    const raw = await readFile(file, { encoding: 'utf-8' });
+    return JSON.parse(raw);
+  } catch (error) {
+    if (error instanceof Error) {
+      throw new Error(
+        `Error while handling JSON file '${file}' : ${error.message}`,
+      );
+    }
+  }
+}
+
+/**
+ * Reads ADOC file.
+ * @param file file name (and path) to read.
+ * @returns ADOC file content.
+ * @throws if file is not found.
+ */
+export function readADocFileSync(file: string) {
+  try {
+    const raw = readFileSync(file, { encoding: 'utf-8' });
+    return raw;
+  } catch {
+    throw new Error(`Adoc file '${file}' not found`);
+  }
+}
+
+/**
+ * Removes leading or trailing whitespace
+ * Can be passed to JSON.stringify method
+ * @param _ First parameter is key of object, but it is not used
+ * @param value Value of the json object
+ * @returns JSON object with values trimmed for leading or trailing white space.
+ */
+export function trimReplacer(_: string, value: unknown) {
+  if (typeof value === 'string') {
+    return value.trim();
+  }
+  return value;
+}
+
+/**
+ * Format an object with JSON.stringify
+ *
+ * The purpose of this function is to format the JSON output in a centralised function
+ * so that the format can be controlled in a single location.
+ *
+ * @param json JSON object to format.
+ * @returns Formatted JSON string
+ */
+export function formatJson(json: object) {
+  return JSON.stringify(json, trimReplacer, 4);
+}
+
+/**
+ * Writes and formats a JSON file.
+ * @param filename file name (and path) to write.
+ * @param json JSON object to format.
+ * @param options Optional, write options
+ * @return true if write succeeded, false otherwise.
+ */
+export async function writeJsonFile(
+  filename: string | FileHandle,
+  json: object,
+  options?: object,
+) {
+  try {
+    await writeFile(filename, formatJson(json), options);
+    return true;
+  } catch {
+    // do nothing, file didn't exist, or no permissions to write
+  }
+  return false;
+}

package/src/utils/lexorank.ts

@@ -0,0 +1,180 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+const ALPHABET = 'abcdefghijklmnopqrstuvwxyz';
+const ALPHABET_SIZE = ALPHABET.length;
+
+const PREFIX = '0|'; // prefix to allow easy addition of buckets later if required
+
+export const FIRST_RANK = PREFIX + ALPHABET[0];
+export const LAST_RANK = PREFIX + ALPHABET[ALPHABET_SIZE - 1];
+
+export const EMPTY_RANK = `1|${ALPHABET[0]}`; // rank to be used when no rank is available
+
+type Rank = string; // Rank is a string that represents a number in base 26
+
+/**
+ * Convert a number to base 26
+ * @param num number to convert to base 26
+ * @returns the string representation of the number in base 26
+ */
+export function enbase(num: number): string {
+  if (num === 0) return ALPHABET[0];
+
+  let result = '';
+  while (num > 0) {
+    result =
+      String.fromCharCode(ALPHABET.charCodeAt(0) + (num % ALPHABET_SIZE)) +
+      result;
+    num = Math.floor(num / ALPHABET_SIZE);
+  }
+  return result;
+}
+
+/**
+ * Convert a rank to a number
+ * @param str rank to convert to a number
+ * @returns the number representation of the rank
+ */
+export function debase(str: string): number {
+  let result = 0;
+  for (let i = 0; i < str.length; i++) {
+    result +=
+      (str.charCodeAt(i) - ALPHABET.charCodeAt(0)) *
+      Math.pow(ALPHABET_SIZE, str.length - i - 1);
+  }
+  return result;
+}
+
+/**
+ * Returns the next available rank after the given rank
+ * @param rank rank to get the next rank after
+ * @returns the next available rank after the given rank
+ */
+export function getRankAfter(rank: Rank): Rank {
+  rank = rank.replace(PREFIX, '');
+  let num = debase(rank);
+  num++;
+  const newRank = enbase(num);
+  if (newRank.length > rank.length) {
+    return PREFIX + rank + ALPHABET[ALPHABET_SIZE / 2];
+  }
+  return PREFIX + newRank;
+}
+/**
+ * Returns the previous available rank before the given rank
+ * if the rank is already the first rank, it throws an error
+ * @param rank rank to get the previous rank before
+ * @returns the previous available rank before the given rank
+ */
+export function getRankBefore(rank: Rank): Rank {
+  rank = rank.replace(PREFIX, '');
+  let num = debase(rank);
+  num--;
+  if (num < 0) {
+    throw new Error('Rank cannot be negative');
+  }
+  return PREFIX + enbase(num);
+}
+
+/**
+ * Get the rank between two ranks
+ * Rank is a string that represents a number in base 26
+ * Rank must be a string of lowercase letters
+ * @param rank1 first rank
+ * @param rank2 second rank
+ * @returns the rank between the two ranks
+ */
+export function getRankBetween(rank1: string, rank2: string): string {
+  // must be padded for the comparison
+
+  rank1 = rank1.replace(PREFIX, '');
+  rank2 = rank2.replace(PREFIX, '');
+
+  const length = Math.max(rank1.length, rank2.length);
+
+  // padding with 'a' to make the ranks equal length
+  // since comparison is done lexicographically
+  // it does not effect the comparison
+
+  const paddedRank1 = rank1.padEnd(length, ALPHABET[0]);
+
+  const paddedRank2 = rank2.padEnd(length, ALPHABET[0]);
+
+  const num1 = debase(paddedRank1);
+  const num2 = debase(paddedRank2);
+
+  if (num1 >= num2) {
+    throw new Error('Rank1 must be smaller than rank2');
+  }
+
+  const res = enbase(Math.floor((num1 + num2) / 2)).padStart(
+    length,
+    ALPHABET[0],
+  );
+  if (res !== paddedRank1 && res !== paddedRank2) {
+    return PREFIX + res;
+  }
+  return PREFIX + res + ALPHABET[ALPHABET_SIZE / 2];
+}
+
+/**
+ * Rebalance the ranks so that the distance between each rank is equal
+ * @param rankAmount number of ranks to rebalance
+ * @returns rebalanced ranks
+ */
+export function rebalanceRanks(rankAmount: number): string[] {
+  if (rankAmount === 0) return [];
+  if (rankAmount === 1) return [FIRST_RANK];
+  const requiredLevels = Math.ceil(
+    Math.log(rankAmount) / Math.log(ALPHABET_SIZE),
+  );
+  // set minumum rank consisting of all 'a's and maximum rank consisting of all 'z's
+  const minRank = ALPHABET[0].repeat(requiredLevels);
+  const maxRank = ALPHABET[ALPHABET_SIZE - 1].repeat(requiredLevels);
+
+  // step size determines the distance between each rank
+  const stepSize = Math.floor(debase(maxRank) / (rankAmount - 1));
+
+  const ranks = [];
+  for (let i = 0; i < rankAmount; i++) {
+    if (i === 0) {
+      ranks.push(PREFIX + minRank);
+    } else if (i === rankAmount - 1) {
+      ranks.push(PREFIX + maxRank);
+    } else {
+      const rank = debase(minRank) + stepSize * i;
+      ranks.push(PREFIX + enbase(rank));
+    }
+  }
+  return ranks;
+}
+
+/**
+ * Sort items based on lexorank
+ * @param items items to sort
+ * @param rankGetter should return the lexorank of the item
+ * @returns sorted items
+ */
+export function sortItems<T>(items: T[], rankGetter: (item: T) => string): T[] {
+  if (items.length === 0) return items;
+  return items.sort((a, b) => compare(rankGetter(a), rankGetter(b)));
+}
+
+/**
+ * default compare function
+ * @param a first string
+ * @param b second string
+ * @returns -1 if a < b, 0 if a === b, 1 if a > b
+ */
+export function compare(a: string, b: string): number {
+  return a.localeCompare(b);
+}

package/src/utils/log-utils.ts

@@ -0,0 +1,127 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { join } from 'path';
+import pino, { type TransportTargetOptions } from 'pino';
+import { fileURLToPath } from 'url';
+const LOG_FILE_LOCATION: string = join(
+  fileURLToPath(import.meta.url),
+  '../../../../../logs/data-handler.trace',
+);
+
+function getLogger() {
+  const all: TransportTargetOptions[] = [
+    {
+      target: 'pino/file',
+      options: { destination: LOG_FILE_LOCATION, mkdir: true }, // trace file
+      level: 'trace',
+    },
+    {
+      target: 'pino-pretty',
+      level: process.env.LOG_LEVEL || 'debug',
+    },
+  ];
+
+  const [file, stdout] = all;
+
+  if (process.env.NODE_ENV === 'test') {
+    // While testing, we simply use stdout and info level by default
+    return pino.default({
+      level: process.env.LOG_LEVEL || 'info',
+      transport: stdout,
+    });
+  }
+  if (process.env.NODE_ENV === 'development') {
+    // In dev mode, only stdout and debug logs by default
+    return pino.default({
+      level: 'trace',
+      transport: {
+        targets: all,
+      },
+    });
+  }
+  return pino.default({
+    level: 'trace',
+    transport: file,
+  });
+}
+
+export const logger = getLogger();
+
+/**
+ * Returns error message string from an Error object.
+ * @param error Error object
+ * @returns Error message.
+ */
+export function errorFunction(error: unknown): string {
+  if (error instanceof Error) {
+    const err: Error = error;
+    return errorMessage(`${err.message}`);
+  } else if (typeof error === 'string') {
+    return errorMessage(`${error}`);
+  } else {
+    return `${logError.name} called without an error object. Actual object is ${JSON.stringify(error)}`;
+  }
+}
+
+/**
+ * Same as 'errorFunction' but can do automatic replacement fof the error message string.
+ * @param message Error message
+ * @param toReplace replacement substring
+ * @param replaceWith string that 'toReplace' is replaced with.
+ * @returns Modified error message.
+ */
+export function errorMessage(
+  message: string,
+  toReplace?: string,
+  replaceWith?: string,
+): string {
+  let errorMessage = message;
+  if (toReplace && replaceWith) {
+    errorMessage = message.replace(toReplace, replaceWith);
+  }
+  return `${errorMessage}`;
+}
+
+/**
+ * Logs error from Error object.
+ * @param error potentially an Error object. When exceptions are raised, they are typically Error objects.
+ */
+export function logError(error: unknown) {
+  if (error instanceof Error) {
+    const err: Error = error;
+    logErrorMessage(`${err.message}`);
+  } else {
+    console.error(
+      `${logError.name} called without an error object. Actual object is ${JSON.stringify(error)}`,
+    );
+  }
+}
+
+/**
+ * Log error message in RED. Certain parts of messages can be replaced.
+ * This is useful, if including a message from external sources, and want to reduce the verbosity of the message.
+ * @param message Error message to log.
+ * @param toReplace String to look for.
+ * @param replaceWith Replace 'toReplace' with this. Only replaces first instance of 'toReplace'.
+ */
+export function logErrorMessage(
+  message: string,
+  toReplace?: string,
+  replaceWith?: string,
+) {
+  let errorMessage = message;
+  if (toReplace && replaceWith) {
+    errorMessage = message.replace(toReplace, replaceWith);
+  }
+  console.error(`${errorMessage}`);
+}

package/src/utils/random.ts

@@ -0,0 +1,37 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { randomInt } from 'node:crypto';
+
+// Maximum base for randomInt.
+const MAX_VALUE = Math.pow(2, 48) - 1;
+
+/**
+ * Returns a random string in given base with given length.
+ * @param base which base to be used, e.g. 36 for base-36, must be an integer
+ * @param length length of string, must be an integer
+ * @returns a random string
+ * @throws if parameters are not integer numbers
+ */
+export function generateRandomString(base: number, length: number): string {
+  if (!Number.isInteger(base) || !Number.isInteger(length)) {
+    throw new Error('parameters must be integers');
+  }
+
+  // Generate a random number between 0 and given base max number - 1
+  const maxBaseNumber = Math.min(MAX_VALUE, Math.pow(base, length));
+  let generatedString = '';
+  while (generatedString.length < length) {
+    generatedString += randomInt(maxBaseNumber).toString(base);
+  }
+  return generatedString.substring(0, length);
+}