@cyberismo/data-handler 0.0.2

package/src/macros/graph/index.ts

@@ -0,0 +1,130 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2024
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import BaseMacro from '../base-macro.js';
+import { Calculate } from '../../commands/index.js';
+import type { MacroOptions } from '../index.js';
+import { createImage, validateMacroContent } from '../index.js';
+import Handlebars from 'handlebars';
+import { join } from 'node:path';
+import { logger } from '../../utils/log-utils.js';
+import type { MacroGenerationContext } from '../../interfaces/macros.js';
+import macroMetadata from './metadata.js';
+import { pathExists } from '../../utils/file-utils.js';
+import { Project } from '../../containers/project.js';
+import { readFile } from 'node:fs/promises';
+import { resourceName } from '../../utils/resource-utils.js';
+import type { Schema } from 'jsonschema';
+import { validateJson } from '../../utils/validate.js';
+import type TaskQueue from '../task-queue.js';
+
+export interface GraphOptions extends MacroOptions {
+  model: string;
+  view: string;
+}
+
+class ReportMacro extends BaseMacro {
+  constructor(tasksQueue: TaskQueue) {
+    super(macroMetadata, tasksQueue);
+  }
+
+  handleValidate = (input: unknown) => {
+    this.parseOptions(input);
+  };
+
+  handleStatic = async (context: MacroGenerationContext, input: unknown) => {
+    return this.handleInject(context, input);
+  };
+
+  handleInject = async (context: MacroGenerationContext, input: unknown) => {
+    const project = new Project(context.projectPath);
+    const calculate = new Calculate(project);
+
+    const resourceNameToPath = (name: string, fileName: string) => {
+      const { identifier, prefix, type } = resourceName(name);
+      if (prefix === project.projectPrefix) {
+        return join(project.paths.resourcesFolder, type, identifier, fileName);
+      }
+      return join(
+        project.paths.modulesFolder,
+        prefix,
+        type,
+        identifier,
+        fileName,
+      );
+    };
+
+    const options = this.parseOptions(input);
+
+    let schema: Schema | null = null;
+    try {
+      schema = JSON.parse(
+        await readFile(
+          resourceNameToPath(options.view, 'parameterSchema.json'),
+          { encoding: 'utf-8' },
+        ),
+      );
+    } catch (err) {
+      logger.trace(
+        err,
+        'Graph schema not found or failed to read, skipping validation',
+      );
+    }
+
+    if (schema) {
+      validateJson(options, {
+        schema,
+      });
+    }
+
+    const modelLocation = resourceNameToPath(options.model, 'model.lp');
+    const viewLocation = resourceNameToPath(options.view, 'view.lp.hbs');
+
+    if (!pathExists(modelLocation)) {
+      throw new Error(`Graph: Model ${options.model} does not exist`);
+    }
+
+    let viewContent = '';
+    try {
+      viewContent = await readFile(viewLocation, { encoding: 'utf-8' });
+    } catch {
+      throw new Error(`Graph: View ${options.view} does not exist`);
+    }
+    const handlebarsContext = {
+      cardKey: context.cardKey,
+      ...options,
+    };
+
+    const handlebars = Handlebars.create();
+    const view = handlebars.compile(viewContent)(handlebarsContext);
+
+    const modelContent = await readFile(modelLocation, { encoding: 'utf-8' });
+    const result = await calculate.runGraph({
+      query: view + '\n' + modelContent,
+    });
+
+    if (typeof result !== 'string') {
+      throw new Error(
+        'Graph macro expected a string from clingo, but received an object',
+      );
+    }
+    return createImage(result);
+  };
+
+  private parseOptions(input: unknown): GraphOptions {
+    return validateMacroContent<GraphOptions>(this.metadata, input);
+  }
+}
+
+export default ReportMacro;

package/src/macros/graph/metadata.ts

@@ -0,0 +1,21 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import type { MacroMetadata } from '../../interfaces/macros.js';
+
+const macroMetadata: MacroMetadata = {
+  name: 'graph',
+  tagName: 'graph',
+  schema: 'graphMacroBaseSchema',
+};
+
+export default macroMetadata;

package/src/macros/index.ts

@@ -0,0 +1,321 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import Handlebars from 'handlebars';
+
+import createCards from './createCards/index.js';
+import graph from './graph/index.js';
+import report from './report/index.js';
+import scoreCard from './scoreCard/index.js';
+
+import { validateJson } from '../utils/validate.js';
+import { DHValidationError } from '../exceptions/index.js';
+import type { AdmonitionType } from '../interfaces/adoc.js';
+import type { Validator } from 'jsonschema';
+import type {
+  MacroGenerationContext,
+  MacroMetadata,
+  MacroName,
+} from '../interfaces/macros.js';
+import type BaseMacro from './base-macro.js';
+import TaskQueue from './task-queue.js';
+
+const CURLY_LEFT = '&#123;';
+const CURLY_RIGHT = '&#125;';
+
+export interface MacroConstructor {
+  new (tasks: TaskQueue): BaseMacro; // Constructor signature
+}
+
+export const macros: { [K in MacroName]: MacroConstructor } = {
+  createCards,
+  graph,
+  report,
+  scoreCard,
+};
+
+export function validateMacroContent<T>(
+  macro: MacroMetadata,
+  data: unknown,
+  validator?: Validator,
+): T {
+  if (!macro.schema) {
+    throw new Error(`Macro ${macro.name} does not have a schema`);
+  }
+
+  try {
+    return validateJson<T>(data, {
+      schemaId: macro.schema,
+      validator,
+    });
+  } catch (error) {
+    let message = `Error: ${error instanceof Error ? error.message : 'Unknown error'}`;
+    if (error instanceof DHValidationError) {
+      message = `${error.errors?.map((e) => e.message).join(', ')}`;
+    }
+    throw new Error(`${macro.name} macro JSON validation error: ${message}`);
+  }
+}
+
+/**
+ * Registers the macros with Handlebars
+ * @param {Mode} mode - The mode to register the macros in
+ */
+export function registerMacros(
+  instance: typeof Handlebars,
+  context: MacroGenerationContext,
+  tasks: TaskQueue,
+) {
+  const macroInstances: BaseMacro[] = [];
+  for (const macro of Object.keys(macros) as MacroName[]) {
+    const MacroClass = macros[macro];
+    const macroInstance = new MacroClass(tasks);
+    instance.registerHelper(macro, function (this: unknown, options) {
+      if (
+        this != null &&
+        typeof this === 'object' &&
+        '__isRaw' in this &&
+        this.__isRaw
+      ) {
+        // we use escaped chars so that they will not be re-run
+        return `${CURLY_LEFT}${CURLY_LEFT}#${macro}${CURLY_RIGHT}${CURLY_RIGHT}${options.fn(this)}${CURLY_LEFT}${CURLY_LEFT}/${macro}${CURLY_RIGHT}${CURLY_RIGHT}`;
+      }
+      return macroInstance.invokeMacro(context, options);
+    });
+    macroInstances.push(macroInstance);
+  }
+
+  instance.registerHelper('raw', function (options) {
+    return options.fn({
+      __isRaw: true,
+    });
+  });
+
+  return macroInstances;
+}
+/**
+ * Calculate amount of handlebars templates inside a string
+ * @param input
+ */
+export function macroCount(input: string): number {
+  const regex = /{{#[\w\s-]+?}}/g;
+  const match = input.match(regex);
+  return match ? match.length : 0;
+}
+
+/**
+ * Registers macros as handlebars helpers, which just leaves the macros in the content
+ * Can be used when your macro uses handlebars and handles content that might contain macros
+ * The handleMacros function will handle recursive macros so do not execute them inside your macro
+ * @param instance handlebars instance
+ */
+export function registerEmptyMacros(instance: typeof Handlebars) {
+  for (const macro of Object.keys(macros) as MacroName[]) {
+    instance.registerHelper(macro, function (this: unknown, options) {
+      return `{{#${macro}}}${options.fn(this)}{{/${macro}}}`;
+    });
+  }
+}
+
+/**
+ * Handle the macros in the content
+ * @param content - The content to handle the macros in
+ * @param mode - The mode to handle the macros in. Inject mode will generate injectable placeholders for the macros while static mode will generate valid adoc. Validate mode will only validate the macros syntax and throw errors in case of issues.
+ */
+export async function evaluateMacros(
+  content: string,
+  context: MacroGenerationContext,
+  maxTries: number = 10,
+) {
+  const handlebars = Handlebars.create();
+  const tasks = new TaskQueue();
+  registerMacros(handlebars, context, tasks);
+  let result = content;
+  while (maxTries-- > 0) {
+    tasks.reset();
+    const compiled = handlebars.compile(result, {
+      strict: true,
+    });
+    try {
+      result = compiled({});
+
+      await tasks.waitAll();
+
+      result = applyMacroResults(result, tasks, context);
+
+      if (macroCount(result) === 0) {
+        break;
+      }
+    } catch (err) {
+      // This will produce a warning in the output in inject/static modes and throw an error in validate mode
+      return handleMacroError(err, '', context);
+    }
+  }
+  if (macroCount(result) !== 0) {
+    return handleMacroError(
+      new Error(`Too many recursive macro evaluations.`),
+      '',
+      context,
+    );
+  }
+  return result.replaceAll(CURLY_LEFT, '{').replaceAll(CURLY_RIGHT, '}');
+}
+
+/**
+ * This function assumes that tasks, which were started by macros, are complete.
+ * It replaces the placeholders of the tasks with the actual results
+ * @param input Adoc content, which might have macro placeholders
+ * @param tasks The taskqueue, which was used to run the macros
+ * @param context General context for the macro evaluation process
+ * @returns Adoc, where placeholders have been replaced with the results.
+ * Note that the results might include new macros.
+ */
+export function applyMacroResults(
+  input: string,
+  tasks: TaskQueue,
+  context: MacroGenerationContext,
+) {
+  for (const item of tasks) {
+    if (item.promiseResult === null) {
+      input = handleMacroError(
+        new Error(
+          `Tried to access result before it was resolved for ${item.placeholder}`,
+        ),
+        item.macro,
+        context,
+      );
+    } else {
+      input = input.replace(item.placeholder, item.promiseResult);
+    }
+  }
+  return input;
+}
+
+/**
+ * Handles errors that come when handling macros
+ * @param error - The error that was thrown
+ * @param macro - The macro that caused the error
+ * @returns The error message that is valid adoc
+ */
+export function handleMacroError(
+  error: unknown,
+  macro: string,
+  context: MacroGenerationContext,
+): string {
+  let message = `Error: ${error instanceof Error ? error.message : 'Unknown error'}`;
+  if (error instanceof DHValidationError) {
+    message = `Check json syntax of macro ${macro}: ${error.errors?.map((e) => e.message).join(', ')}`;
+  }
+  if (
+    typeof error === 'object' &&
+    error != null &&
+    'lineNumber' in error &&
+    typeof error.lineNumber === 'number'
+  ) {
+    message += ` at line ${error.lineNumber}`;
+  }
+  if (context.mode === 'validate') {
+    throw new Error(message);
+  } else {
+    return createAdmonition('WARNING', 'Macro Error', message);
+  }
+}
+
+// This is used to generate unique keys for macros
+// There might be a better way to do this
+let macroCounter = 0;
+
+type Value = string | number | boolean | undefined;
+
+/**
+ * Macro options can be a flat object or a nested object
+ * The nested object will be flattened into dot notation attributes
+ */
+export type MacroOptions = {
+  [key: string]: Value | MacroOptions;
+};
+/**
+ * Creates an injectable placeholder for a macro
+ * @param macro - The macro to create the placeholder for
+ * @param options - Options will be passed to the html element as attributes
+ */
+export function createHtmlPlaceholder(
+  macro: MacroMetadata,
+  options: MacroOptions,
+) {
+  // Flatten nested objects into dot notation attributes
+  const flattenedOptions: Record<string, Value> = {};
+
+  // Helper function to flatten nested objects
+  const flatten = (obj: MacroOptions, prefix = ''): void => {
+    Object.entries(obj).forEach(([key, value]) => {
+      const newKey = prefix ? `${prefix}.${key}` : key;
+
+      if (
+        value !== null &&
+        typeof value === 'object' &&
+        !Array.isArray(value)
+      ) {
+        // Recursively flatten nested objects
+        flatten(value, newKey);
+      } else {
+        // Add leaf values to flattened options
+        flattenedOptions[newKey] = value as Value;
+      }
+    });
+  };
+
+  flatten(options);
+
+  // Convert flattened options to attribute strings
+  const attributeStrings = Object.entries(flattenedOptions)
+    .filter(([, value]) => value !== undefined)
+    .map(([key, value]) => `${key}="${value}"`);
+
+  const optionString = attributeStrings.join(' ');
+
+  // start with a line change to ensure that inline passthrough +++ is on its own line
+  return `\n+++\n<${macro.tagName}${optionString ? ` ${optionString}` : ''} key="macro-${macroCounter++}"></${macro.tagName}>\n+++\n`;
+}
+
+/**
+ * Creates an adoc admonition
+ * @param type - The type of admonition
+ * @param label - The label of the admonition
+ * @param content - The content of the admonition
+ * @returns The adoc admonition as a string
+ */
+export function createAdmonition(
+  type: AdmonitionType,
+  label: string,
+  content: string,
+) {
+  return `[${type}]\n.${label}\n====\n${content}\n====\n\n`;
+}
+
+/**
+ * Helper function for including base64 encoded images for now
+ * @param image base64 encoded image
+ * @returns valid asciidoc with the image
+ */
+export function createImage(image: string) {
+  if (process.env.EXPORT_FORMAT) {
+    return `image::data:image/svg+xml;base64,${image}[]\n`;
+  } else {
+    return `++++
+<div class="cyberismo-svg-wrapper" data-type="cyberismo-svg-wrapper">
+${Buffer.from(image, 'base64').toString('utf-8')}
+</div>
+++++
+`;
+  }
+}

package/src/macros/report/index.ts

@@ -0,0 +1,88 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { registerEmptyMacros, validateMacroContent } from '../index.js';
+import type { MacroOptions } from '../index.js';
+import type { MacroGenerationContext } from '../../interfaces/macros.js';
+import macroMetadata from './metadata.js';
+import { Project } from '../../containers/project.js';
+import { Calculate } from '../../commands/index.js';
+import Handlebars from 'handlebars';
+import BaseMacro from '../base-macro.js';
+import { validateJson } from '../../utils/validate.js';
+import type TaskQueue from '../task-queue.js';
+import { ReportResource } from '../../resources/report-resource.js';
+import { resourceName } from '../../utils/resource-utils.js';
+
+export interface ReportOptions extends MacroOptions {
+  name: string;
+}
+
+class ReportMacro extends BaseMacro {
+  constructor(tasksQueue: TaskQueue) {
+    super(macroMetadata, tasksQueue);
+  }
+  handleValidate = (input: unknown) => {
+    this.validate(input);
+  };
+
+  handleStatic = async (context: MacroGenerationContext, data: unknown) => {
+    return this.handleInject(context, data);
+  };
+
+  handleInject = async (context: MacroGenerationContext, data: unknown) => {
+    const options = this.validate(data);
+    const project = new Project(context.projectPath);
+    const resource = new ReportResource(project, resourceName(options.name));
+    const report = await resource.show();
+
+    if (!report) throw new Error(`Report ${options} does not exist`);
+
+    if (report.schema) {
+      validateJson(options, {
+        schema: report.schema,
+      });
+    }
+
+    const handlebarsContext = {
+      cardKey: context.cardKey,
+      ...options,
+    };
+
+    const handlebars = Handlebars.create();
+
+    const template = handlebars.compile(report.queryTemplate, {
+      strict: true,
+    });
+
+    const calculate = new Calculate(project);
+    const result = await calculate.runLogicProgram({
+      query: template(handlebarsContext),
+    });
+    if (result.error) {
+      throw new Error(result.error);
+    }
+    // register empty macros so that other macros aren't touched yet
+    registerEmptyMacros(handlebars);
+
+    return handlebars.compile(report.contentTemplate)({
+      ...handlebarsContext,
+      ...result,
+    });
+  };
+
+  private validate(data: unknown): ReportOptions {
+    return validateMacroContent<ReportOptions>(this.metadata, data);
+  }
+}
+
+export default ReportMacro;

package/src/macros/report/metadata.ts

@@ -0,0 +1,21 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import type { MacroMetadata } from '../../interfaces/macros.js';
+
+const macroMetadata: MacroMetadata = {
+  name: 'report',
+  tagName: 'report',
+  schema: 'reportMacroBaseSchema',
+};
+
+export default macroMetadata;

package/src/macros/scoreCard/index.ts

@@ -0,0 +1,55 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import type { MacroOptions } from '../index.js';
+import { createHtmlPlaceholder, validateMacroContent } from '../index.js';
+
+import type { MacroGenerationContext } from '../../interfaces/macros.js';
+import macroMetadata from './metadata.js';
+import BaseMacro from '../base-macro.js';
+import type TaskQueue from '../task-queue.js';
+
+export interface ScoreCardOptions extends MacroOptions {
+  title?: string;
+  value: number;
+  unit?: string;
+  legend?: string;
+}
+
+class ScoreCardMacro extends BaseMacro {
+  constructor(tasksQueue: TaskQueue) {
+    super(macroMetadata, tasksQueue);
+  }
+  handleValidate = (data: string) => {
+    this.validate(data);
+  };
+
+  handleStatic = async (context: MacroGenerationContext, input: unknown) => {
+    const options = this.validate(input);
+    return this.createAsciidocElement(options);
+  };
+
+  handleInject = async (_: MacroGenerationContext, input: unknown) => {
+    const options = this.validate(input);
+    return createHtmlPlaceholder(this.metadata, options);
+  };
+
+  private validate(input: unknown): ScoreCardOptions {
+    return validateMacroContent<ScoreCardOptions>(this.metadata, input);
+  }
+
+  private createAsciidocElement(options: ScoreCardOptions) {
+    return `\n----\n${options.title}: ${options.value} ${options.unit ?? ''} ${options.legend ?? ''}\n----\n`;
+  }
+}
+
+export default ScoreCardMacro;

package/src/macros/scoreCard/metadata.ts

@@ -0,0 +1,21 @@
+/**
+    Cyberismo
+    Copyright © Cyberismo Ltd and contributors 2024
+
+    This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License version 3 as published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public
+    License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import type { MacroMetadata } from '../../interfaces/macros.js';
+
+const macroMetadata: MacroMetadata = {
+  name: 'scoreCard',
+  tagName: 'score-card',
+  schema: 'scoreCardMacroSchema',
+};
+
+export default macroMetadata;