@cyberismo/data-handler 0.0.18 → 0.0.19

package/src/migrations/migration-worker.ts

@@ -0,0 +1,190 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation.
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+  FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+  details. You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { parentPort } from 'node:worker_threads';
+import { pathToFileURL } from 'node:url';
+
+import type {
+  Migration,
+  MigrationContext,
+  MigrationStepResult,
+} from '@cyberismo/migrations';
+import type { WorkerMessage, WorkerResponse } from './migration-executor.js';
+
+let currentMigration: Migration | undefined;
+let abortController: AbortController | undefined;
+
+/**
+ * Execute a specific step of a migration.
+ *
+ * Runs one of the migration steps (before, backup, migrate, after).
+ * Checks for cancellation before and after execution.
+ *
+ * @param migration The loaded migration object
+ * @param stepName Which step to execute ('before', 'backup', 'migrate', or 'after')
+ * @param context Migration context with paths and version information
+ * @returns Migration step result
+ */
+async function executeStep(
+  migration: Migration,
+  stepName: string,
+  context: MigrationContext,
+): Promise<MigrationStepResult> {
+  if (abortController?.signal.aborted) {
+    return {
+      success: false,
+      error: new Error('Migration cancelled'),
+    };
+  }
+
+  try {
+    let result: MigrationStepResult;
+
+    switch (stepName) {
+      case 'before':
+        if (!migration.before) {
+          return { success: true, message: `No 'before' step` };
+        }
+        result = await migration.before(context);
+        break;
+
+      case 'backup':
+        if (!migration.backup) {
+          return { success: true, message: `No 'backup' step` };
+        }
+        result = await migration.backup(context);
+        break;
+
+      case 'migrate':
+        result = await migration.migrate(context);
+        break;
+
+      case 'after':
+        if (!migration.after) {
+          return { success: true, message: `No 'after' step` };
+        }
+        result = await migration.after(context);
+        break;
+
+      default:
+        return {
+          success: false,
+          error: new Error(`Unknown migration step: ${stepName}`),
+        };
+    }
+
+    if (abortController?.signal.aborted) {
+      return {
+        success: false,
+        error: new Error('Migration cancelled'),
+      };
+    }
+
+    return result;
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error : new Error(String(error)),
+    };
+  }
+}
+
+/**
+ * Handle a cancellation request from the main thread.
+ */
+async function handleCancel(): Promise<void> {
+  abortController?.abort();
+
+  if (currentMigration?.cancel) {
+    try {
+      await currentMigration.cancel();
+    } catch {
+      // Ignore errors during cancellation - we're cancelling anyway
+    }
+  }
+}
+
+/**
+ * Worker thread message handler.
+ *
+ * Listens for messages from the main thread:
+ * - 'execute': Run a migration step
+ * - 'cancel': Cancel the currently running migration
+ *
+ * Responds with:
+ * - 'result': The migration step result
+ * - 'error': An error message when execution failed
+ */
+if (parentPort) {
+  parentPort.on('message', (message: WorkerMessage) => {
+    if (!parentPort) return;
+
+    void (async () => {
+      try {
+        if (message.type === 'cancel') {
+          await handleCancel();
+          const response: WorkerResponse = {
+            type: 'result',
+            result: {
+              success: false,
+              error: new Error('Migration cancelled'),
+            },
+          };
+          parentPort.postMessage(response);
+          return;
+        }
+
+        if (message.type === 'execute') {
+          if (!message.migrationPath || !message.stepName || !message.context) {
+            throw new Error('Missing required parameters for "execute"');
+          }
+
+          abortController = new AbortController();
+
+          const migrationUrl = pathToFileURL(message.migrationPath).href;
+          const migrationModule = await import(migrationUrl);
+          currentMigration = migrationModule.default || migrationModule;
+
+          if (typeof currentMigration?.migrate !== 'function') {
+            throw new Error(
+              `Migration does not implement migrate() function: ${message.migrationPath}`,
+            );
+          }
+
+          const contextWithSignal: MigrationContext = {
+            ...message.context,
+            signal: abortController.signal,
+          };
+
+          const result = await executeStep(
+            currentMigration,
+            message.stepName,
+            contextWithSignal,
+          );
+
+          const response: WorkerResponse = {
+            type: 'result',
+            result,
+          };
+          parentPort.postMessage(response);
+        }
+      } catch (error) {
+        const response: WorkerResponse = {
+          type: 'error',
+          error: error instanceof Error ? error.message : String(error),
+        };
+        parentPort.postMessage(response);
+      }
+    })();
+  });
+}

package/src/migrations/worker-executor.ts

@@ -0,0 +1,185 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation.
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+  FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+  details. You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { Worker } from 'node:worker_threads';
+
+import { getChildLogger } from '../utils/log-utils.js';
+import type {
+  MigrationContext,
+  MigrationStepResult,
+} from '@cyberismo/migrations';
+import type { WorkerMessage, WorkerResponse } from './migration-executor.js';
+
+const CANCEL_PERIOD_MS = 100;
+const logger = getChildLogger({ module: 'WorkerExecutor' });
+
+/**
+ * Execute a migration step in a separate worker thread.
+ * The worker loads the migration program dynamically and executes the specified step.
+ *
+ * @param migrationPath Absolute path to the migration's 'index.js' file
+ * @param stepName The migration step to execute
+ * @param context Migration context
+ * @param timeoutMilliSeconds Timeout in milliseconds to wait for the step to complete
+ * @returns Migration step result
+ */
+export async function executeStep(
+  migrationPath: string,
+  stepName: string,
+  context: MigrationContext,
+  timeoutMilliSeconds: number,
+): Promise<MigrationStepResult> {
+  // Always uses the compiled .js version from the dist directory.
+  function _workerPath() {
+    const currentFilePath = fileURLToPath(import.meta.url);
+    const currentDir = dirname(currentFilePath);
+    const srcMigrationsSegment = join('src', 'migrations');
+    const distMigrationsSegment = join('dist', 'migrations');
+    const distDir = currentDir.replace(
+      srcMigrationsSegment,
+      distMigrationsSegment,
+    );
+    return join(distDir, 'migration-worker.js');
+  }
+
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(_workerPath(), {
+      execArgv: process.execArgv,
+    });
+    let timeoutId: NodeJS.Timeout | undefined;
+    let isResolved = false;
+
+    const cleanup = () => {
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+        timeoutId = undefined;
+      }
+    };
+
+    const terminate = async (sendCancel: boolean = false): Promise<void> => {
+      if (sendCancel) {
+        try {
+          const cancelMessage: WorkerMessage = { type: 'cancel' };
+          worker.postMessage(cancelMessage);
+          await new Promise((_resolve) =>
+            setTimeout(_resolve, CANCEL_PERIOD_MS),
+          );
+        } catch {
+          // Ignore errors when sending cancel message
+        }
+      }
+
+      try {
+        await worker.terminate();
+      } catch (error) {
+        logger.debug({ error }, 'Error terminating worker');
+      }
+    };
+
+    timeoutId = setTimeout(() => {
+      if (!isResolved) {
+        isResolved = true;
+        cleanup();
+
+        void (async () => {
+          await terminate(true);
+          resolve({
+            success: false,
+            error: new Error(`Migration step '${stepName}' timeout`),
+          });
+        })();
+      }
+    }, timeoutMilliSeconds);
+
+    worker.on('message', (response: WorkerResponse) => {
+      if (isResolved) return;
+
+      isResolved = true;
+      cleanup();
+
+      void (async () => {
+        if (response.type === 'error') {
+          await terminate(false);
+          resolve({
+            success: false,
+            error: new Error(response.error || 'Unknown worker error'),
+          });
+        } else if (response.type === 'result' && response.result) {
+          await terminate(false);
+          resolve(response.result);
+        } else {
+          await terminate(false);
+          resolve({
+            success: false,
+            error: new Error('Invalid worker response'),
+          });
+        }
+      })();
+    });
+
+    worker.on('error', (error) => {
+      if (isResolved) return;
+
+      isResolved = true;
+      cleanup();
+
+      void (async () => {
+        await terminate(false);
+        resolve({
+          success: false,
+          error,
+        });
+      })();
+    });
+
+    worker.on('exit', (code) => {
+      if (isResolved) return;
+
+      isResolved = true;
+      cleanup();
+
+      if (code !== 0) {
+        resolve({
+          success: false,
+          error: new Error(`Worker exited with code ${code}`),
+        });
+      }
+    });
+
+    const migrationContext: MigrationContext = {
+      cardRootPath: context.cardRootPath,
+      cardsConfigPath: context.cardsConfigPath,
+      fromVersion: context.fromVersion,
+      toVersion: context.toVersion,
+      backupDir: context.backupDir,
+    };
+
+    const executeMessage: WorkerMessage = {
+      type: 'execute',
+      migrationPath,
+      stepName,
+      context: migrationContext,
+    };
+
+    try {
+      worker.postMessage(executeMessage);
+    } catch (error) {
+      isResolved = true;
+      cleanup();
+      void terminate(false);
+      reject(error);
+    }
+  });
+}

package/src/project-settings.ts

@@ -35,6 +35,8 @@ export class ProjectConfiguration implements ProjectSettings {
   schemaVersion?: number;
   name: string;
   cardKeyPrefix: string;
+  category?: string;
+  description: string;
   modules: ModuleSetting[];
   hubs: HubSetting[];
   private logger = getChildLogger({ module: 'Project' });
@@ -45,6 +47,7 @@ export class ProjectConfiguration implements ProjectSettings {
     this.name = '';
     this.settingPath = path;
     this.cardKeyPrefix = '';
+    this.description = '';
     this.modules = [];
     this.hubs = [];
     this.autoSave = autoSave;
@@ -79,6 +82,8 @@ export class ProjectConfiguration implements ProjectSettings {
       this.schemaVersion = settings.schemaVersion;
       this.cardKeyPrefix = settings.cardKeyPrefix;
       this.name = settings.name;
+      this.category = settings.category;
+      this.description = settings.description || '';
       this.modules = settings.modules || [];
       this.hubs = settings.hubs || [];
     } else {
@@ -106,6 +111,8 @@ export class ProjectConfiguration implements ProjectSettings {
       schemaVersion: this.schemaVersion,
       cardKeyPrefix: this.cardKeyPrefix,
       name: this.name,
+      category: this.category,
+      description: this.description,
       modules: this.modules,
       hubs: this.hubs,
     };

package/src/resources/calculation-resource.ts

@@ -34,6 +34,11 @@ export class CalculationResource extends FolderResource<
   CalculationMetadata,
   CalculationContent
 > {
+  /**
+   * Creates instance of CalculationResource
+   * @param project Project to use
+   * @param name Resource name
+   */
   constructor(project: Project, name: ResourceName) {
     super(project, name, 'calculations');
 
@@ -41,9 +46,15 @@ export class CalculationResource extends FolderResource<
     this.contentSchema = super.contentSchemaContent(this.contentSchemaId);
   }
 
-  // When resource name changes
+  /**
+   * When resource name changes
+   * @param existingName Current resource name
+   */
   protected async onNameChange(existingName: string) {
-    await super.updateCalculations(existingName, this.content.name);
+    await Promise.all([
+      super.updateCalculations(existingName, this.content.name),
+      super.updateCardContentReferences(existingName, this.content.name),
+    ]);
     await this.write();
   }
 

package/src/resources/card-type-resource.ts

@@ -17,6 +17,7 @@ import { FileResource } from './file-resource.js';
 import { resourceName, resourceNameToString } from '../utils/resource-utils.js';
 import { ResourcesFrom } from '../containers/project.js';
 import { sortCards } from '../utils/card-utils.js';
+import { removeValue } from '../utils/common-utils.js';
 import { Validate } from '../commands/validate.js';
 
 import type {
@@ -39,6 +40,11 @@ import type { ResourceName } from '../utils/resource-utils.js';
  * Card type resource class.
  */
 export class CardTypeResource extends FileResource<CardType> {
+  /**
+   * Creates instance of CardTypeResource
+   * @param project Project to use
+   * @param name Resource name
+   */
   constructor(project: Project, name: ResourceName) {
     super(project, name, 'cardTypes');
 
@@ -69,7 +75,10 @@ export class CardTypeResource extends FileResource<CardType> {
     if (op && op.name === 'rank') return;
 
     // Collect both project cards and template cards.
-    const cards = await this.collectCards(this.content.name);
+    const cards = await this.collectCards(
+      this.content.name,
+      (card, cardTypeName) => card.metadata?.cardType === cardTypeName,
+    );
 
     if (op && op.name === 'change') {
       const from = (op as ChangeOperation<string>).target;
@@ -119,7 +128,10 @@ export class CardTypeResource extends FileResource<CardType> {
     op: ChangeOperation<Type>,
   ) {
     await this.verifyStateMapping(stateMapping, op);
-    const cards = await this.collectCards(this.content.name);
+    const cards = await this.collectCards(
+      this.content.name,
+      (card, cardTypeName) => card.metadata?.cardType === cardTypeName,
+    );
 
     const unmappedStates: string[] = [];
 
@@ -179,15 +191,6 @@ export class CardTypeResource extends FileResource<CardType> {
     return references;
   }
 
-  // Remove value from array.
-  // todo: make it as generic and move to utils
-  private removeValue(array: string[], value: string) {
-    const index = array.findIndex((element) => element === value);
-    if (index !== -1) {
-      array.splice(index, 1);
-    }
-  }
-
   // If value from 'customFields' is removed, remove it also from 'optionallyVisible' and 'alwaysVisible' arrays.
   private removeValueFromOtherArrays<Type>(
     op: Operation<Type>,
@@ -201,8 +204,8 @@ export class CardTypeResource extends FileResource<CardType> {
       field = { name: target['name' as keyof Type] };
     }
     const fieldName = (field ? field.name : target) as string;
-    this.removeValue(content.alwaysVisibleFields, fieldName);
-    this.removeValue(content.optionallyVisibleFields, fieldName);
+    removeValue(content.alwaysVisibleFields, fieldName);
+    removeValue(content.optionallyVisibleFields, fieldName);
   }
 
   // Sets content container values to be either '[]' or with proper values.
@@ -385,6 +388,7 @@ export class CardTypeResource extends FileResource<CardType> {
     await Promise.all([
       super.updateHandleBars(existingName, this.content.name),
       super.updateCalculations(existingName, this.content.name),
+      super.updateCardContentReferences(existingName, this.content.name),
       this.updateLinkTypes(existingName),
     ]);
 
@@ -394,7 +398,8 @@ export class CardTypeResource extends FileResource<CardType> {
   /**
    * Creates a new card type object. Base class writes the object to disk automatically.
    * @param workflowName Workflow name that this card type uses.
-   * @throws when workflow is empty, or does not exist in the project.
+   * @throws when workflow is empty, or
+   *         when workflow does not exist in the project.
    */
   public async createCardType(workflowName: string) {
     if (!workflowName) {

package/src/resources/field-type-resource.ts

@@ -50,6 +50,11 @@ export class FieldTypeResource extends FileResource<FieldType> {
   private fromType: DataType = 'integer';
   private toType: DataType = 'integer';
 
+  /**
+   * Creates an instance of FieldTypeResource
+   * @param project Project to use
+   * @param name Resource name
+   */
   constructor(project: Project, name: ResourceName) {
     super(project, name, 'fieldTypes');
 
@@ -206,7 +211,12 @@ export class FieldTypeResource extends FileResource<FieldType> {
     const removedValue = (op.target as EnumDefinition).enumValue;
     const cardTypes = this.relevantCardTypes();
     const allCards = await Promise.all(
-      cardTypes.map((cardType) => this.collectCards(cardType)),
+      cardTypes.map((cardType) =>
+        this.collectCards(
+          cardType,
+          (card, cardTypeName) => card.metadata?.cardType === cardTypeName,
+        ),
+      ),
     );
     const cardsToUpdate = allCards
       .flat()
@@ -283,6 +293,7 @@ export class FieldTypeResource extends FileResource<FieldType> {
     await Promise.all([
       super.updateHandleBars(existingName, this.content.name),
       super.updateCalculations(existingName, this.content.name),
+      super.updateCardContentReferences(existingName, this.content.name),
       this.updateCardTypes(existingName),
     ]);
     await this.write();
@@ -314,16 +325,16 @@ export class FieldTypeResource extends FileResource<FieldType> {
    */
   public static fieldDataTypes(): DataType[] {
     return [
-      'shortText',
-      'longText',
-      'number',
-      'integer',
       'boolean',
-      'enum',
-      'list',
       'date',
       'dateTime',
+      'enum',
+      'integer',
+      'list',
+      'longText',
+      'number',
       'person',
+      'shortText',
     ];
   }
 

package/src/resources/file-resource.ts

@@ -35,23 +35,35 @@ export abstract class FileResource<
     super(project, name, type);
     this.initialize();
   }
-  // Collects cards that are using the 'cardTypeName'.
-  protected async collectCards(cardTypeName: string) {
-    function filteredCards(cardSource: Card[], cardTypeName: string): Card[] {
-      const cards = cardSource;
-      return cards.filter((card) => card.metadata?.cardType === cardTypeName);
+  /**
+   * Collects cards that match the given filter function.
+   * @param resourceName The resource name to filter by
+   * @param filterFn Function that returns true for cards to include
+   * @returns Array of cards that match the filter
+   */
+  protected async collectCards(
+    resourceName: string,
+    filterFn: (card: Card, resourceName: string) => boolean,
+  ): Promise<Card[]> {
+    function filteredCards(
+      cardSource: Card[],
+      resourceName: string,
+      filterFn: (card: Card, resourceName: string) => boolean,
+    ): Card[] {
+      return cardSource.filter((card) => filterFn(card, resourceName));
     }
 
     // Collect both project cards ...
     const projectCards = filteredCards(
       this.project.cards(this.project.paths.cardRootFolder),
-      cardTypeName,
+      resourceName,
+      filterFn,
     );
     // ... and cards from each template that would be affected.
     const templates = this.project.resources.templates(ResourcesFrom.localOnly);
     const templateCards = templates.map((template) => {
       const templateObject = template.templateObject();
-      return filteredCards(templateObject.cards(), cardTypeName);
+      return filteredCards(templateObject.cards(), resourceName, filterFn);
     });
     // Return all affected cards
     const cards = [projectCards, ...templateCards].reduce(
@@ -67,7 +79,12 @@ export abstract class FileResource<
    */
   protected abstract onNameChange?(previousName: string): Promise<void>;
 
-  // Updates resource key to a new prefix
+  /**
+   * Updates resource key to a new prefix
+   * @param name Resource name
+   * @param prefixes list of prefixes in the project
+   * @returns updated resource name
+   */
   protected updatePrefixInResourceName(name: string, prefixes: string[]) {
     const { identifier, prefix, type } = resourceName(name);
     if (this.moduleResource) {

package/src/resources/graph-model-resource.ts

@@ -38,6 +38,11 @@ export class GraphModelResource extends FolderResource<
   GraphModelMetadata,
   GraphModelContent
 > {
+  /**
+   * Creates an instance of GraphModelResource
+   * @param project Project to use
+   * @param name Resource name
+   */
   constructor(project: Project, name: ResourceName) {
     super(project, name, 'graphModels');
 
@@ -55,6 +60,7 @@ export class GraphModelResource extends FolderResource<
         join(this.internalFolder, CONTENT_FILES.model),
       ]),
       super.updateCalculations(existingName, this.content.name),
+      super.updateCardContentReferences(existingName, this.content.name),
     ]);
     // Finally, write updated content.
     await this.write();

package/src/resources/graph-view-resource.ts

@@ -39,6 +39,11 @@ export class GraphViewResource extends FolderResource<
   GraphViewMetadata,
   GraphViewContent
 > {
+  /**
+   * Creates instance of GraphViewResource
+   * @param project Project to use
+   * @param name Resource name
+   */
   constructor(project: Project, name: ResourceName) {
     super(project, name, 'graphViews');
 
@@ -56,6 +61,7 @@ export class GraphViewResource extends FolderResource<
         await this.handleBarFile(),
       ]),
       super.updateCalculations(existingName, this.content.name),
+      super.updateCardContentReferences(existingName, this.content.name),
     ]);
     await this.write();
   }