@cyberismo/data-handler 0.0.18 → 0.0.19

package/src/containers/project/project-paths.ts

@@ -80,6 +80,10 @@ export class ProjectPaths {
     return join(this.resourcesFolder, 'graphViews');
   }
 
+  public get internalRootFolder(): string {
+    return join(this.path, '.cards');
+  }
+
   public get linkTypesFolder(): string {
     return join(this.resourcesFolder, 'linkTypes');
   }
@@ -93,7 +97,7 @@ export class ProjectPaths {
   }
 
   public get modulesFolder(): string {
-    return join(this.path, '.cards', 'modules');
+    return join(this.internalRootFolder, 'modules');
   }
 
   public moduleResourcePath(
@@ -105,7 +109,7 @@ export class ProjectPaths {
   }
 
   public get resourcesFolder(): string {
-    return join(this.path, '.cards', 'local');
+    return join(this.internalRootFolder, 'local');
   }
 
   public get reportsFolder(): string {

package/src/containers/project.ts

@@ -55,7 +55,9 @@ import { ResourceHandler } from './project/resource-handler.js';
 import { Validate } from '../commands/validate.js';
 import { ContentWatcher } from './project/project-content-watcher.js';
 import { getChildLogger } from '../utils/log-utils.js';
+import { MigrationExecutor } from '../migrations/migration-executor.js';
 
+import type { MigrationResult } from '@cyberismo/migrations';
 import type { Template } from './template.js';
 
 import { ROOT } from '../utils/constants.js';
@@ -572,12 +574,27 @@ export class Project extends CardContainer {
   public async handleCardDeleted(deletedCard: Card) {
     // Delete children from the cache first
     if (deletedCard.children && deletedCard.children.length > 0) {
+      const parentCachedCard = this.cardCache.getCard(deletedCard.key);
+      const parentLocation = parentCachedCard?.location || 'project';
+
       for (const child of deletedCard.children) {
         try {
           const childCard = this.findCard(child);
+          const childCachedCard = this.cardCache.getCard(child);
+
+          // Safety check: only delete children from the same location (project or template)
+          if (childCachedCard && childCachedCard.location !== parentLocation) {
+            const errorMessage =
+              `Cannot delete child card '${child}' from different location '${childCachedCard.location}' ` +
+              `than parent card '${deletedCard.key}' from '${parentLocation}'`;
+            this.logger.error(errorMessage);
+            throw new Error(errorMessage);
+          }
+
           await this.handleCardDeleted(childCard);
-        } catch {
+        } catch (error) {
           this.logger.warn(
+            { error },
             `Accessing child '${child}' of '${deletedCard.key}' when deleting cards caused an exception`,
           );
           continue;
@@ -753,6 +770,7 @@ export class Project extends CardContainer {
       );
       return {
         name: moduleConfig.name,
+        description: moduleConfig.description || '',
         modules: moduleConfig.modules,
         hubs: moduleConfig.hubs,
         path: modulePath,
@@ -970,6 +988,51 @@ export class Project extends CardContainer {
     return this.resourceHandler;
   }
 
+  /**
+   * Run migrations to bring project schema to target version.
+   * @param fromVersion Current schema version
+   * @param toVersion Target schema version
+   * @param backupDir Optional directory for backups. If undefined, no backup is created.
+   * @param timeoutMilliSeconds Optional timeout in milliseconds. If undefined, uses default (2 minutes).
+   * @returns Migration result
+   */
+  public async runMigrations(
+    fromVersion: number,
+    toVersion: number,
+    backupDir?: string,
+    timeoutMilliSeconds?: number,
+  ): Promise<MigrationResult> {
+    this.logger.info({ fromVersion, toVersion }, 'Starting schema migration');
+
+    const executor = new MigrationExecutor(
+      this,
+      backupDir,
+      timeoutMilliSeconds,
+    );
+    const result = await executor.migrate(
+      fromVersion,
+      toVersion,
+      async (version: number) => {
+        this.settings.schemaVersion = version;
+        await this.settings.save();
+      },
+    );
+
+    if (result.success) {
+      this.logger.info(
+        { fromVersion, toVersion },
+        'Migration completed successfully',
+      );
+    } else {
+      this.logger.error(
+        { error: result.error, message: result.message },
+        'Migration failed',
+      );
+    }
+
+    return result;
+  }
+
   /**
    * Shows details of a project.
    * @returns details of a project.
@@ -979,6 +1042,8 @@ export class Project extends CardContainer {
       name: this.settings.name,
       path: this.basePath,
       prefix: this.projectPrefix,
+      category: this.configuration.category,
+      description: this.configuration.description,
       hubs: this.configuration.hubs,
       modules: this.resources.moduleNames(),
       numberOfCards: (await this.listCards(CardLocation.projectOnly))[0].cards

package/src/containers/template.ts

@@ -152,6 +152,11 @@ export class Template extends CardContainer {
 
       card.key = templateIDMap.get(card.key) || card.key;
 
+      // Remap children keys from template keys to new project card keys
+      card.children = card.children.map(
+        (childKey) => templateIDMap.get(childKey) || childKey,
+      );
+
       // Set parent field based on template hierarchy and creation location
       // Store the original template parent before key remapping
       const originalParentKey = card.parent;

package/src/interfaces/command-options.ts

@@ -60,6 +60,12 @@ export interface ImportCommandOptions extends BaseCommandOptions {
   skipMigrationLog?: boolean;
 }
 
+// Options for 'migrate' command
+export interface MigrateCommandOptions extends BaseCommandOptions {
+  backup?: string;
+  timeout?: number;
+}
+
 // Options for 'move' command
 export type MoveCommandOptions = BaseCommandOptions;
 
@@ -111,6 +117,7 @@ export type AllCommandOptions =
   | ExportCommandOptions
   | FetchCommandOptions
   | ImportCommandOptions
+  | MigrateCommandOptions
   | MoveCommandOptions
   | RankCommandOptions
   | RemoveCommandOptions
@@ -132,6 +139,7 @@ export type CommandOptions<T extends CmdKey> = {
   export: ExportCommandOptions;
   fetch: FetchCommandOptions;
   import: ImportCommandOptions;
+  migrate: MigrateCommandOptions;
   move: MoveCommandOptions;
   rank: RankCommandOptions;
   remove: RemoveCommandOptions;

package/src/interfaces/project-interfaces.ts

@@ -171,6 +171,8 @@ export interface ProjectMetadata {
   name: string;
   path: string;
   prefix: string;
+  category?: string;
+  description?: string;
   modules: string[];
   hubs: HubSetting[];
   numberOfCards: number;
@@ -181,6 +183,8 @@ export interface ProjectSettings {
   schemaVersion?: number;
   cardKeyPrefix: string;
   name: string;
+  category?: string;
+  description: string;
   modules: ModuleSetting[];
   hubs: HubSetting[];
 }

package/src/migrations/index.ts

@@ -0,0 +1,20 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation.
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+  FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+  details. You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+export { MigrationExecutor } from './migration-executor.js';
+export type {
+  Migration,
+  MigrationContext,
+  MigrationResult,
+  MigrationStepResult,
+} from '@cyberismo/migrations';

package/src/migrations/migration-executor.ts

@@ -0,0 +1,478 @@
+/**
+  Cyberismo
+  Copyright © Cyberismo Ltd and contributors 2025
+  This program is free software: you can redistribute it and/or modify it under
+  the terms of the GNU Affero General Public License version 3 as published by
+  the Free Software Foundation.
+  This program is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+  FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+  details. You should have received a copy of the GNU Affero General Public
+  License along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { availableMigrations, migration } from '@cyberismo/migrations';
+import { availableSpace, folderSize } from '../utils/file-utils.js';
+import { executeStep } from './worker-executor.js';
+import { getChildLogger } from '../utils/log-utils.js';
+import { Validate } from '../commands/validate.js';
+
+import type {
+  Migration,
+  MigrationContext,
+  MigrationResult,
+  MigrationStepResult,
+} from '@cyberismo/migrations';
+import type { Project } from '../containers/project.js';
+
+const DEFAULT_MIGRATION_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes
+const MEGABYTES = 1024 * 1024; // 1 MB
+
+/**
+ * Messages sent from the main thread to worker threads.
+ */
+export interface WorkerMessage {
+  type: 'execute' | 'cancel';
+  migrationPath?: string;
+  stepName?: string;
+  context?: MigrationContext;
+}
+
+/**
+ * Response messages sent from worker threads back to the main thread.
+ */
+export interface WorkerResponse {
+  type: 'result' | 'error';
+  result?: MigrationStepResult;
+  error?: string;
+}
+
+/**
+ * Internal state for tracking migrations.
+ */
+interface ExecutionState {
+  context: MigrationContext;
+  stepsExecuted: string[];
+}
+
+/**
+ * Executes schema migrations for a project.
+ */
+export class MigrationExecutor {
+  private logger = getChildLogger({ module: 'MigrationExecutor' });
+  private timeoutMilliSeconds: number;
+
+  /**
+   * Constructs instance of MigrationExecutor
+   * @param project Project instance to use
+   * @param backupDir Backup directory, if any.
+   * @param timeoutMilliSeconds Timeout in milliseconds (defaults to 2 minutes)
+   */
+  constructor(
+    private project: Project,
+    private backupDir?: string,
+    timeoutMilliSeconds?: number,
+  ) {
+    this.timeoutMilliSeconds =
+      timeoutMilliSeconds ?? DEFAULT_MIGRATION_TIMEOUT_MS;
+  }
+
+  // Helper to create failure result from ExecutionState
+  private createFailureResult(
+    state: ExecutionState,
+    message: string,
+    error?: Error,
+  ): MigrationResult {
+    return {
+      success: false,
+      message,
+      error,
+      stepsExecuted: state.stepsExecuted,
+    };
+  }
+
+  // Execute a single migration step and handle failure
+  private async executeStep(
+    stepName: string,
+    migrationPath: string,
+    state: ExecutionState,
+  ): Promise<{ success: true } | MigrationResult> {
+    const result = await executeStep(
+      migrationPath,
+      stepName,
+      state.context,
+      this.timeoutMilliSeconds,
+    );
+    state.stepsExecuted.push(stepName);
+    if (!result.success) {
+      const messagePrefix =
+        {
+          before: 'Pre-migration check failed',
+          backup: 'Backup failed',
+          migrate: 'Migration failed',
+          after: 'Post-migration step failed',
+        }[stepName] || `Step '${stepName}' failed`;
+
+      return this.createFailureResult(
+        state,
+        `${messagePrefix}: ${result.message || 'Unknown error'}`,
+        result.error,
+      );
+    }
+    return { success: true };
+  }
+
+  // Execute a single migration.
+  private async executeMigration(
+    migrationPath: string,
+    fromVersion: number,
+    toVersion: number,
+    updateVersionCallback: (version: number) => Promise<void>,
+  ): Promise<MigrationResult> {
+    this.logger.info(
+      { fromVersion, toVersion },
+      `Executing migration from version ${fromVersion} to ${toVersion}`,
+    );
+
+    const state: ExecutionState = {
+      context: {
+        cardRootPath: this.project.paths.cardRootFolder,
+        cardsConfigPath: this.project.paths.internalRootFolder,
+        fromVersion,
+        toVersion,
+      },
+      stepsExecuted: [],
+    };
+
+    try {
+      // Load migration to check which steps exist
+      const migration = this.loadMigration(toVersion);
+      if (!migration) {
+        return this.createFailureResult(
+          state,
+          `Failed to load migration for version ${toVersion}`,
+        );
+      }
+
+      if (migration.before) {
+        const result = await this.executeStep('before', migrationPath, state);
+        if (!result.success) return result;
+      }
+
+      if (migration.backup && this.backupDir !== undefined) {
+        state.context.backupDir = this.backupDir;
+        const result = await this.executeStep('backup', migrationPath, state);
+        if (!result.success) return result;
+      }
+
+      const migrateResult = await this.executeStep(
+        'migrate',
+        migrationPath,
+        state,
+      );
+      if (!migrateResult.success) return migrateResult;
+
+      // Update schema version in project after successful migration
+      await updateVersionCallback(toVersion);
+      state.stepsExecuted.push('update-version');
+
+      if (migration.after) {
+        const result = await this.executeStep('after', migrationPath, state);
+        if (!result.success) return result;
+      }
+
+      // Run validation after migration
+      state.stepsExecuted.push('validate');
+      const validationErrors = await this.validate();
+      if (validationErrors) {
+        return this.createFailureResult(
+          state,
+          `Post-migration validation failed: ${validationErrors}`,
+        );
+      }
+      this.logger.info('Post-migration validation passed');
+
+      return {
+        success: true,
+        message: `Successfully migrated from version ${fromVersion} to ${toVersion}`,
+        stepsExecuted: state.stepsExecuted,
+      };
+    } catch (error) {
+      return this.createFailureResult(
+        state,
+        `Migration threw an exception: ${error}`,
+        error instanceof Error ? error : new Error(String(error)),
+      );
+    }
+  }
+
+  // Checks if pre-migration validation succeeds.
+  private async preMigrateValidation(
+    fromVersion: number,
+    toVersion: number,
+    stepsExecuted: string[],
+  ): Promise<MigrationResult> {
+    const validationErrors = await this.validate();
+    if (validationErrors) {
+      this.logger.error(
+        { errors: validationErrors },
+        'Pre-migration validation failed',
+      );
+      return {
+        success: false,
+        message: `Pre-migration validation failed. Please fix the following errors before migrating:\n${validationErrors}`,
+        stepsExecuted,
+      };
+    }
+    this.logger.info('Pre-migration validation passed');
+    return {
+      success: true,
+      stepsExecuted,
+    };
+  }
+
+  // Validate the project.
+  private async validate() {
+    const validator = Validate.getInstance();
+    return await validator.validate(this.project.basePath, () => this.project);
+  }
+
+  // Validates that migration versions are valid.
+  private validateMigrationVersions(
+    fromVersion: number,
+    toVersion: number,
+    stepsExecuted: string[],
+  ): MigrationResult {
+    const valid = fromVersion < toVersion;
+    return {
+      success: valid,
+      message: valid
+        ? undefined
+        : `Current version (${fromVersion}) is not lower than target version (${toVersion})`,
+      stepsExecuted,
+    };
+  }
+
+  // Checks if there is enough disk space.
+  protected async checkDiskSpace(
+    fromVersion: number,
+    toVersion: number,
+    stepsExecuted: string[],
+  ): Promise<MigrationResult> {
+    try {
+      const internalSize = await folderSize(
+        this.project.paths.internalRootFolder,
+      );
+      const cardRootSize = await folderSize(this.project.paths.cardRootFolder);
+      const projectSize = internalSize + cardRootSize;
+      const requiredSpace = projectSize * 2;
+      const spaceAvailable = await availableSpace(this.project.basePath);
+      const projectSizeMB = (projectSize / MEGABYTES).toFixed(2);
+      const requiredSpaceMB = (requiredSpace / MEGABYTES).toFixed(2);
+      const availableSpaceMB = (spaceAvailable / MEGABYTES).toFixed(2);
+
+      if (spaceAvailable < requiredSpace) {
+        return {
+          success: false,
+          message: `Insufficient disk space. Required: ${requiredSpaceMB} MB, Available: ${availableSpaceMB} MB. Migration needs at least 2x the project size (${projectSizeMB} MB).`,
+          stepsExecuted,
+        };
+      }
+
+      this.logger.info('Disk space check passed');
+      return {
+        success: true,
+        stepsExecuted,
+      };
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to check disk space');
+      return {
+        success: false,
+        message: `Failed to check disk space: ${error instanceof Error ? error.message : String(error)}`,
+        error: error instanceof Error ? error : new Error(String(error)),
+        stepsExecuted,
+      };
+    }
+  }
+
+  /**
+   * Discover available migrations between two versions.
+   * @param fromVersion Starting version (exclusive)
+   * @param toVersion Target version (inclusive)
+   * @returns Sorted list of migration version numbers
+   */
+  protected migrationsAvailable(
+    fromVersion: number,
+    toVersion: number,
+  ): number[] {
+    const allVersions = availableMigrations();
+    return allVersions.filter(
+      (version) => version > fromVersion && version <= toVersion,
+    );
+  }
+
+  // Validates and discovers available migrations
+  protected availableMigrations(
+    fromVersion: number,
+    toVersion: number,
+    stepsExecuted: string[],
+  ): MigrationResult & { migrationVersions: number[] } {
+    const migrationVersions = this.migrationsAvailable(fromVersion, toVersion);
+    const found = migrationVersions.length > 0;
+    if (found) {
+      this.logger.info(
+        { versions: migrationVersions },
+        `Found ${migrationVersions.length} migration(s)`,
+      );
+    }
+    return {
+      success: found,
+      message: found
+        ? undefined
+        : `No migrations found between version ${fromVersion} and ${toVersion}`,
+      stepsExecuted,
+      migrationVersions,
+    };
+  }
+
+  // Load a migration module for a specific version.
+  protected loadMigration(version: number): Migration | undefined {
+    this.logger.debug({ version }, 'Loading migration');
+
+    try {
+      const migrationObject = migration(version);
+
+      if (!migrationObject) {
+        this.logger.error({ version }, `Migration not found`);
+        return undefined;
+      }
+
+      if (typeof migrationObject.migrate !== 'function') {
+        throw new Error(
+          `Migration ${version} does not implement migrate() function`,
+        );
+      }
+
+      return migrationObject;
+    } catch (error) {
+      this.logger.error({ error, version }, `Failed to load migration`);
+      return undefined;
+    }
+  }
+
+  /**
+   * Get the file path for a migration worker.
+   * @param version Migration version number
+   * @returns Path to migration file
+   */
+  protected migrationWorkerPath(version: number): string {
+    const migrationsPackageUrl = import.meta.resolve('@cyberismo/migrations');
+    const migrationsIndexPath = fileURLToPath(migrationsPackageUrl);
+    const migrationsDistDir = dirname(migrationsIndexPath);
+    return join(migrationsDistDir, version.toString(), 'index.js');
+  }
+
+  /**
+   * Execute all necessary migrations to bring project to target version.
+   * @param fromVersion Current project version
+   * @param toVersion Target version
+   * @param updateVersionCallback Callback to update project schema version after each migration
+   * @returns Overall migration result
+   */
+  public async migrate(
+    fromVersion: number,
+    toVersion: number,
+    updateVersionCallback: (version: number) => Promise<void>,
+  ): Promise<MigrationResult> {
+    const stepsDone: string[] = [];
+
+    // Step: Validate migration versions
+    const versionResult = this.validateMigrationVersions(
+      fromVersion,
+      toVersion,
+      stepsDone,
+    );
+    if (!versionResult.success) return versionResult;
+    stepsDone.push('version-validation');
+
+    // Step: Pre-migration validation
+    const validationResult = await this.preMigrateValidation(
+      fromVersion,
+      toVersion,
+      stepsDone,
+    );
+    if (!validationResult.success) return validationResult;
+    stepsDone.push('pre-validation');
+
+    // Step: Check disk space
+    const diskSpaceResult = await this.checkDiskSpace(
+      fromVersion,
+      toVersion,
+      stepsDone,
+    );
+    if (!diskSpaceResult.success) return diskSpaceResult;
+    stepsDone.push('disk-space-check');
+
+    // Step: Discover available migrations
+    const discoveryResult = this.availableMigrations(
+      fromVersion,
+      toVersion,
+      stepsDone,
+    );
+    if (!discoveryResult.success) return discoveryResult;
+    stepsDone.push('migration-versions');
+
+    const migrationVersions = discoveryResult.migrationVersions;
+
+    // Step(s): Execute migrations in sequence
+    let currentVersion = fromVersion;
+    try {
+      for (const targetVersion of migrationVersions) {
+        const migrationPath = this.migrationWorkerPath(targetVersion);
+
+        const result = await this.executeMigration(
+          migrationPath,
+          currentVersion,
+          targetVersion,
+          updateVersionCallback,
+        );
+
+        if (!result.success) {
+          return {
+            success: false,
+            message: result.message || 'Migration failed',
+            error: result.error,
+            stepsExecuted: stepsDone,
+          };
+        }
+        stepsDone.push(
+          ...result.stepsExecuted.map((step) => `v${targetVersion}:${step}`),
+        );
+
+        currentVersion = targetVersion;
+        this.logger.info(
+          { targetVersion },
+          `Migration to version ${targetVersion} completed successfully`,
+        );
+      }
+    } catch (error) {
+      return {
+        success: false,
+        message:
+          error instanceof Error
+            ? error.message
+            : `Migration timeout or error: ${error}`,
+        error: error instanceof Error ? error : new Error(String(error)),
+        stepsExecuted: stepsDone,
+      };
+    }
+    return {
+      success: true,
+      message: `Successfully migrated from version ${fromVersion} to ${currentVersion}`,
+      stepsExecuted: stepsDone,
+    };
+  }
+}