@cyberismo/data-handler 0.0.2

package/src/commands/export-site.ts

@@ -0,0 +1,356 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2024
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+// node
+import fs from 'node:fs';
+import { platform, tmpdir } from 'node:os';
+import { appendFile, copyFile, mkdir, writeFile } from 'node:fs/promises';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+import git from 'isomorphic-git';
+import { dump } from 'js-yaml';
+
+import { type Calculate, type Show } from './index.js';
+import { Export } from './export.js';
+import type { Card } from '../interfaces/project-interfaces.js';
+import type { CardType } from '../interfaces/resource-interfaces.js';
+import { errorFunction } from '../utils/log-utils.js';
+import type { Project } from '../containers/project.js';
+import { sortItems } from '../utils/lexorank.js';
+import { getStaticDirectoryPath } from '@cyberismo/resources';
+
+interface ExportOptions {
+  silent: boolean;
+}
+
+export class ExportSite extends Export {
+  private tmpDir: string = '';
+  private moduleDir: string = '';
+  private pagesDir: string = '';
+  private imagesDir: string = '';
+  private playbookDir: string = '';
+  private playbookFile: string = '';
+  private navFile: string = '';
+  private options: ExportOptions | undefined;
+
+  constructor(project: Project, calculateCmd: Calculate, showCmd: Show) {
+    super(project, calculateCmd, showCmd);
+  }
+
+  // Writes index.html file into cards folder that just redirects to the first card.
+  private createCardsIndex(destination: string, cardKey: string) {
+    const indexFile = join(destination, 'cards', 'index.html');
+    const indexContent = `<meta http-equiv="refresh" content="0; url=${cardKey}.html">`;
+    try {
+      fs.writeFileSync(indexFile, indexContent);
+    } catch (error) {
+      throw new Error(errorFunction(error));
+    }
+  }
+
+  // todo: change this so that split temp folder creation to its own method.
+  // then parallelize this and export() as much as you can.
+  private async initDirectories() {
+    // Create temporary adoc output directory
+    try {
+      this.tmpDir = mkdtempSync(join(tmpdir(), 'cards-'));
+    } catch (error) {
+      throw new Error(errorFunction(error));
+    }
+
+    // Antora requires the content directory to be a Git repository
+    await this.initRepo();
+
+    // Create the pages and images directories
+    this.moduleDir = join(this.tmpDir, 'modules', 'ROOT');
+    this.pagesDir = join(this.moduleDir, 'pages');
+    this.imagesDir = join(this.moduleDir, 'assets', 'images');
+    this.navFile = join(this.moduleDir, 'nav.adoc');
+
+    const promiseContainer: Promise<string | undefined>[] = [];
+    promiseContainer.push(mkdir(this.pagesDir, { recursive: true }));
+    promiseContainer.push(mkdir(this.imagesDir, { recursive: true }));
+    await Promise.all(promiseContainer);
+
+    // Create the playbook directory
+    this.playbookDir = mkdtempSync(join(tmpdir(), 'cards-playbook-'));
+  }
+
+  // Generate the site from the source files using Antora.
+  private generate(outputPath: string) {
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+
+    const absoluteOutputPath = resolve(process.cwd(), outputPath);
+
+    const additionalArguments = [
+      '--to-dir',
+      absoluteOutputPath,
+      this.playbookFile,
+    ];
+    if (this.options && this.options?.silent) {
+      additionalArguments.unshift('--silent');
+    }
+    // Use spawnsync to npx execute the program "antora"
+    try {
+      // Allthough we use pnpm as the package manager, we rather use npx at runtime, because it is shipped with node
+      spawnSync('npx', ['antora', ...additionalArguments], {
+        stdio: 'inherit',
+        shell: platform() === 'win32',
+        cwd: __dirname,
+      });
+    } catch (error) {
+      throw new Error(errorFunction(error));
+    }
+  }
+
+  // Create the Antora playbook.
+  private async createPlaybook(cards: Card[]) {
+    let startPage: string = '';
+
+    if (cards[0]) {
+      startPage = cards[0].key + '.adoc';
+    } else {
+      throw new Error('Cannot create a playbook for an empty card set');
+    }
+
+    const playbook = {
+      site: {
+        title: this.project.configuration.name,
+        start_page: `cards:ROOT:${startPage}`,
+      },
+      content: {
+        sources: [
+          {
+            url: this.tmpDir,
+            branches: 'HEAD',
+          },
+        ],
+      },
+      urls: {
+        html_extension_style: 'default',
+      },
+      ui: {
+        bundle: {
+          url: join(await getStaticDirectoryPath(), 'ui-bundle'),
+          snapshot: true,
+        },
+      },
+    };
+
+    this.playbookFile = join(this.playbookDir, 'antora-playbook.yml');
+    writeFileSync(this.playbookFile, dump(playbook));
+  }
+
+  // Create the Antora site descriptor.
+  private createDescriptor() {
+    const projectName = this.project.configuration.name;
+    const descriptor = {
+      name: 'cards',
+      title: projectName,
+      version: null,
+      nav: ['modules/ROOT/nav.adoc'],
+    };
+
+    const descriptorPath = join(this.tmpDir, 'antora.yml');
+    try {
+      writeFileSync(descriptorPath, dump(descriptor));
+    } catch (error) {
+      throw new Error(errorFunction(error));
+    }
+  }
+
+  // Initialise the temporary directory as a temporary Git repository.
+  private async initRepo() {
+    try {
+      await git.init({ fs, dir: this.tmpDir });
+      writeFileSync(join(this.tmpDir, '.gitkeep'), '');
+      await git.add({ fs, dir: this.tmpDir, filepath: '.gitkeep' });
+    } catch (error) {
+      throw new Error(errorFunction(error));
+    }
+
+    try {
+      await git.commit({
+        fs,
+        dir: this.tmpDir,
+        author: {
+          name: 'Cyberismo Cards',
+          email: 'info@cyberismo.com',
+        },
+        message: 'Add .gitkeep',
+      });
+    } catch (error) {
+      throw new Error(errorFunction(error));
+    }
+  }
+
+  // Write cards as Antora-compatible AsciiDoc to the given location
+  // @param path Directory where the cards should be written
+  // @param cards Array of Cards
+  // @param depth Navigation depth - this is used in recursion to format the hierarchical menu
+  private async toAdocDirectoryAsContent(
+    path: string,
+    cards: Card[],
+    depth: number,
+  ) {
+    depth++;
+
+    // Sort the cards by rank
+    cards = sortItems(cards, function (card) {
+      return card.metadata?.rank || '1|z';
+    });
+
+    // Ensure the target path exists
+    await mkdir(path, { recursive: true });
+    for (const card of cards) {
+      // Construct path for individual card file
+      const cardPath = join(path, card.key + '.adoc');
+      const cardXRef = cardPath.slice(this.pagesDir.length);
+      let navFileContent = '';
+      if (card.metadata?.progress) {
+        navFileContent =
+          '*'.repeat(depth) +
+          ` xref:${cardXRef}[${card.metadata?.title} (${card.metadata?.progress}%)]\n`;
+      } else {
+        navFileContent =
+          '*'.repeat(depth) + ` xref:${cardXRef}[${card.metadata?.title}]\n`;
+      }
+
+      await appendFile(this.navFile, navFileContent);
+
+      let tempContent: string = '';
+      if (card.metadata) {
+        const cardTypeForCard = await this.project.resource<CardType>(
+          card.metadata?.cardType,
+        );
+        tempContent = '\n= ';
+        tempContent += card.metadata?.title
+          ? `${card.metadata.title}\n\n`
+          : 'Untitled\n\n';
+        tempContent += super.metaToAdoc(card, cardTypeForCard);
+      }
+
+      if (card.content) {
+        tempContent += '\n' + card.content;
+      }
+      if (tempContent !== '') {
+        await writeFile(cardPath, tempContent);
+      }
+
+      if (card.children && card.children.length > 0) {
+        // Recurse into the child cards
+        await this.toAdocDirectoryAsContent(path, card.children, depth);
+      }
+
+      if (card.attachments) {
+        const promiseContainer = [];
+        for (const attachment of card.attachments) {
+          const source = join(attachment.path, attachment.fileName);
+          const destination = join(this.imagesDir, `${attachment.fileName}`);
+          promiseContainer.push(copyFile(source, destination));
+        }
+        await Promise.all(promiseContainer);
+      }
+    }
+
+    --depth;
+  }
+
+  // Export the cards to the temporary directory as a full HTML site generated by Antora.
+  private async export(destination: string, cards: Card[]) {
+    await this.initDirectories();
+    this.createDescriptor();
+    await this.toAdocDirectoryAsContent(this.pagesDir, cards, 0);
+    await this.createPlaybook(cards);
+    this.generate(destination);
+  }
+
+  /**
+   * Export the card tree as an Antora site
+   * @param destination Path where the site is generated
+   * @param cardKey Optional; If defined exports the card tree from underneath this card.
+   */
+  public async exportToSite(
+    destination: string,
+    cardKey?: string,
+    options?: ExportOptions,
+  ): Promise<string> {
+    this.options = options;
+    const sourcePath: string = cardKey
+      ? join(
+          this.project.paths.cardRootFolder,
+          this.project.pathToCard(cardKey),
+        )
+      : this.project.paths.cardRootFolder;
+    let cards: Card[] = [];
+
+    // If doing a partial tree export, put the parent information as it would have already been gathered.
+    if (cardKey) {
+      const card = await this.project.findSpecificCard(cardKey);
+      if (!card) {
+        throw new Error(
+          `Input validation error: cannot find card '${cardKey}'`,
+        );
+      }
+      cards.push({
+        key: cardKey,
+        path: sourcePath,
+        children: [],
+        attachments: [],
+      });
+    }
+
+    await this.calculateCmd.generate();
+    const tree = await this.calculateCmd.runQuery('tree');
+
+    if (cardKey) {
+      const targetCard = this.findCardInTree(tree, cardKey);
+      if (!targetCard) {
+        throw new Error(`Cannot find card '${cardKey}' in the tree hierarchy`);
+      }
+      cards = [await this.treeQueryResultToCard(targetCard)];
+    } else {
+      for (const treeQueryResult of tree) {
+        cards.push(await this.treeQueryResultToCard(treeQueryResult));
+      }
+    }
+
+    if (!cards.length) {
+      throw new Error('No cards found');
+    }
+    if (cards.length > 3000) {
+      throw new Error(
+        `There are ${cards.length} cards in the project. Exporting to a site only supports maximum of 3000 cards.`,
+      );
+    }
+    if (cards.length > 1000 && cards.length < 3000) {
+      console.warn(
+        `Warning: There are ${cards.length} cards in the project. There is a hard limit of 3000 cards that can be exported as a site.`,
+      );
+    }
+
+    await this.export(destination, cards);
+    if (cards.length > 0 && cards.at(0)) {
+      const firstCard = cards.at(0);
+      if (firstCard && firstCard.key) {
+        this.createCardsIndex(destination, firstCard.key);
+      }
+    }
+    return '';
+  }
+}

package/src/commands/export.ts

@@ -0,0 +1,315 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2024
+
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation. This program is distributed in the hope that it
+  will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+  See the GNU Affero General Public License for more details.
+  You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+// node
+import { appendFile, copyFile, mkdir, truncate } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+
+// asciidoctor
+import Processor from '@asciidoctor/core';
+import type { Calculate, Show } from './index.js';
+import type {
+  Card,
+  FetchCardDetails,
+} from '../interfaces/project-interfaces.js';
+import type { CardType } from '../interfaces/resource-interfaces.js';
+import { Project } from '../containers/project.js';
+import type { QueryResult } from '../types/queries.js';
+import { sortItems } from '../utils/lexorank.js';
+
+const attachmentFolder: string = 'a';
+
+export class Export {
+  constructor(
+    protected project: Project,
+    protected calculateCmd: Calculate,
+    protected showCmd: Show,
+  ) {}
+
+  // This file should set the top level items to the adoc.
+  private async toAdocFile(path: string, cards: Card[]) {
+    await appendFile(path, `:imagesdir: ./${attachmentFolder}/\n`);
+    await this.toAdocFileAsContent(path, cards);
+  }
+
+  // Format card metadata to an AsciiDoc table.
+  protected metaToAdoc(card: Card, cardType: CardType | undefined): string {
+    let content = '';
+    if (card.metadata) {
+      content += `[.cyberismo-meta-wrapper]\n`;
+      content += '--\n';
+      content += `[.cyberismo-meta]\n`;
+      content += '[cols="1,1"]\n';
+      content += '[frame=none]\n';
+      content += '[grid=none]\n';
+      content += '|===\n';
+      content += `|Card key|${card.key}\n`;
+      content += `|Status|${card.metadata.workflowState}\n`;
+      content += `|Card type|${card.metadata.cardType}\n`;
+      content += `|Labels|${card.metadata.labels?.join(', ') || ''}`;
+
+      for (const [key, value] of Object.entries(card.metadata)) {
+        if (
+          cardType?.alwaysVisibleFields.includes(key) ||
+          cardType?.optionallyVisibleFields?.includes(key)
+        ) {
+          const displayName = cardType?.customFields.find(
+            (item) => item.name === key,
+          )?.displayName;
+          let nameToShow = displayName
+            ? displayName
+            : key[0].toUpperCase() + key.slice(1);
+          if (nameToShow === 'WorkflowState') {
+            nameToShow = 'Workflow state';
+          } else if (nameToShow === 'Cardtype') {
+            nameToShow = 'Card type';
+          }
+
+          // Escape pipe character in cell values
+          let escapedValue = 'N/A';
+
+          if (value) {
+            escapedValue = value.toString().replaceAll('|', '\\|');
+          }
+
+          content += `|${nameToShow}|${escapedValue}\n`;
+        }
+      }
+      content += '|===\n';
+      content += '--\n';
+    }
+    return content;
+  }
+
+  private async toAdocFileAsContent(path: string, cards: Card[]) {
+    for (const card of cards) {
+      let fileContent = '';
+
+      if (card.metadata?.title) {
+        fileContent += `== ${card.metadata.title}\n\n`;
+      } else {
+        fileContent += `== ${card.key}\n\n`;
+      }
+
+      if (card.metadata) {
+        const cardTypeForCard = await this.project.resource<CardType>(
+          card.metadata?.cardType,
+        );
+        const metaDataContent = this.metaToAdoc(card, cardTypeForCard);
+        fileContent += metaDataContent;
+      }
+
+      if (card.content) {
+        fileContent += card.content;
+      }
+
+      if (card.attachments) {
+        const promiseContainer = [];
+        for (const attachment of card.attachments) {
+          const destination = join(
+            dirname(path),
+            attachmentFolder,
+            attachment.fileName,
+          );
+          const source = join(attachment.path, attachment.fileName);
+          promiseContainer.push(copyFile(source, destination));
+        }
+        await Promise.all(promiseContainer);
+      }
+
+      // Add separator between cards
+      fileContent += '\n\n';
+
+      if (fileContent) {
+        await appendFile(path, fileContent);
+      }
+
+      if (card.children) {
+        await this.toAdocFileAsContent(path, card.children);
+      }
+    }
+  }
+
+  /**
+   * Recursively searches for a card with the specified key in the tree hierarchy.
+   * @param treeItems Array of tree query results to search through
+   * @param targetKey The key of the card to find
+   * @returns The found tree item or null if not found
+   */
+  protected findCardInTree(
+    treeItems: QueryResult<'tree'>[],
+    targetKey: string,
+  ): QueryResult<'tree'> | null {
+    for (const item of treeItems) {
+      if (item.key === targetKey) {
+        return item;
+      }
+      if (item.children && item.children.length > 0) {
+        const foundInChildren = this.findCardInTree(item.children, targetKey);
+        if (foundInChildren) {
+          return foundInChildren;
+        }
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Convert treeQueryResult object into a Card object and add content, metadata & attachments
+   * Handles card children recursively
+   * @param treeQueryResult tree query result object
+   */
+  protected async treeQueryResultToCard(
+    treeQueryResult: QueryResult<'tree'>,
+  ): Promise<Card> {
+    const card: Card = {
+      key: treeQueryResult.key,
+      path: '',
+      children: [],
+      attachments: [],
+    };
+
+    // Get content and attachments separately, not included in queries
+    const fetchCardDetails: FetchCardDetails = {
+      attachments: true,
+      children: false,
+      content: true,
+      contentType: 'adoc',
+      metadata: true,
+      parent: false,
+    };
+
+    const cardDetailsResponse = await this.showCmd.showCardDetails(
+      fetchCardDetails,
+      card.key,
+    );
+
+    let asciiDocContent = '';
+    const projectPath = this.project.basePath;
+    try {
+      const { evaluateMacros } = await import('../macros/index.js');
+      asciiDocContent = await evaluateMacros(
+        cardDetailsResponse.content || '',
+        {
+          mode: 'static',
+          projectPath,
+          cardKey: card.key,
+        },
+      );
+    } catch (error) {
+      asciiDocContent = `Macro error: ${error instanceof Error ? error.message : 'Unknown error'}\n\n${asciiDocContent}`;
+    }
+
+    card.path = cardDetailsResponse.path;
+    card.metadata = cardDetailsResponse.metadata;
+    card.metadata!.progress = treeQueryResult.progress;
+    card.content = asciiDocContent;
+    card.attachments = cardDetailsResponse.attachments;
+
+    for (const result of treeQueryResult.children ?? []) {
+      card.children!.push(await this.treeQueryResultToCard(result));
+    }
+
+    return card;
+  }
+
+  /**
+   * Exports the card(s) to ascii doc.
+   * @param destination Path to where the resulting file(s) will be created.
+   * @param cardKey If not exporting the whole card tree, card key of parent card.
+   * @returns status message
+   */
+  public async exportToADoc(
+    destination: string,
+    cardKey?: string,
+  ): Promise<string> {
+    const sourcePath: string = this.project.paths.cardRootFolder;
+    let cards: Card[] = [];
+
+    // If doing a partial tree export, put the parent information as it would have already been gathered.
+    if (cardKey) {
+      const card = await this.project.findSpecificCard(cardKey);
+      if (!card) {
+        throw new Error(
+          `Input validation error: cannot find card '${cardKey}'`,
+        );
+      }
+      cards.push({
+        key: cardKey,
+        path: sourcePath,
+        children: [],
+        attachments: [],
+      });
+    }
+
+    await this.calculateCmd.generate();
+    const tree = await this.calculateCmd.runQuery('tree');
+
+    if (cardKey) {
+      const targetCard = this.findCardInTree(tree, cardKey);
+      if (!targetCard) {
+        throw new Error(`Cannot find card '${cardKey}' in the tree hierarchy`);
+      }
+      cards = [await this.treeQueryResultToCard(targetCard)];
+    } else {
+      for (const treeQueryResult of tree) {
+        cards.push(await this.treeQueryResultToCard(treeQueryResult));
+      }
+    }
+
+    // Sort the cards by rank
+    cards = sortItems(cards, function (card) {
+      return card.metadata?.rank || '1|z';
+    });
+
+    await mkdir(join(destination, attachmentFolder), { recursive: true });
+    const resultDocumentPath: string = join(
+      destination,
+      Project.cardContentFile,
+    );
+    let message = '';
+    try {
+      await truncate(resultDocumentPath, 0);
+      message = `Using existing output file '${resultDocumentPath}'`;
+    } catch {
+      message = `Creating output file '${resultDocumentPath}'`;
+    }
+
+    await this.toAdocFile(resultDocumentPath, cards);
+    return message;
+  }
+
+  /**
+   * Exports the card(s) to HTML and opens the browser.
+   * @param destination Path to where the resulting file(s) will be created.
+   * @param cardKey Optional; If not exporting the whole card tree, card key of parent card.
+   */
+  public async exportToHTML(
+    destination: string,
+    cardKey?: string,
+  ): Promise<string> {
+    let message = '';
+    await this.exportToADoc(destination, cardKey).then((msg) => {
+      message = msg;
+      const asciiDocProcessor = Processor();
+      const adocFile = join(destination, Project.cardContentFile);
+      asciiDocProcessor.convertFile(adocFile, {
+        safe: 'safe',
+        base_dir: '/',
+        standalone: true,
+      });
+    });
+    return message;
+  }
+}