@solidxai/core 0.1.13-beta.0 → 0.1.13-beta.2

package/src/jobs/redis/workflow-execution-queue-options-redis.ts

@@ -0,0 +1,9 @@
+import { BrokerType } from '../../interfaces';
+
+const QUEUE_NAME = 'solid_workflow_execution_queue_redis';
+
+export default {
+  name: QUEUE_NAME,
+  type: BrokerType.Redis,
+  queueName: QUEUE_NAME,
+};

package/src/jobs/redis/workflow-execution-subscriber-redis.service.ts

@@ -0,0 +1,42 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { QueueMessage } from '../../interfaces/mq';
+import { QueuesModuleOptions } from '../../interfaces';
+import { WorkflowExecutionQueuePayload } from '../../types/workflow-execution-queue.types';
+import { MqMessageQueueService } from '../../services/mq-message-queue.service';
+import { MqMessageService } from '../../services/mq-message.service';
+import { RedisSubscriber } from '../../services/queues/redis-subscriber.service';
+import { WorkflowRuntimeService } from '../../services/workflow/workflow-runtime.service';
+import workflowExecutionQueueOptions from './workflow-execution-queue-options-redis';
+
+@Injectable()
+export class WorkflowExecutionSubscriberRedis extends RedisSubscriber<WorkflowExecutionQueuePayload> {
+  private readonly workflowExecutionLogger = new Logger(WorkflowExecutionSubscriberRedis.name);
+
+  constructor(
+    readonly mqMessageService: MqMessageService,
+    readonly mqMessageQueueService: MqMessageQueueService,
+    private readonly workflowRuntimeService: WorkflowRuntimeService,
+  ) {
+    super(mqMessageService, mqMessageQueueService);
+  }
+
+  options(): QueuesModuleOptions {
+    return {
+      ...workflowExecutionQueueOptions,
+    };
+  }
+
+  async subscribe(message: QueueMessage<WorkflowExecutionQueuePayload>) {
+    const payload = message.payload;
+    this.workflowExecutionLogger.log(
+      `Processing queued workflow execution message ${message.messageId ?? '<unknown>'}.`,
+    );
+
+    if (payload.executionId !== undefined && payload.executionId !== null) {
+      return this.workflowRuntimeService.executeEnqueuedExecution(payload.executionId);
+    }
+
+    throw new Error('Workflow execution queue payload must include executionId.');
+  }
+}

package/src/repository/workflow-definition.repository.ts

@@ -0,0 +1,19 @@
+import { Injectable } from '@nestjs/common';
+import { SecurityRuleRepository } from './security-rule.repository';
+import { SolidBaseRepository } from './solid-base.repository';
+import { RequestContextService } from 'src/services/request-context.service';
+import { DataSource } from 'typeorm';
+import { InjectDataSource } from '@nestjs/typeorm';
+import { WorkflowDefinition } from '../entities/workflow-definition.entity';
+
+@Injectable()
+export class WorkflowDefinitionRepository extends SolidBaseRepository<WorkflowDefinition> {
+    constructor(
+        @InjectDataSource("default")
+        readonly dataSource: DataSource,
+        readonly requestContextService: RequestContextService,
+        readonly securityRuleRepository: SecurityRuleRepository,
+    ) {
+        super(WorkflowDefinition, dataSource, requestContextService, securityRuleRepository);
+    }
+}

package/src/repository/workflow-execution-artifact.repository.ts

@@ -0,0 +1,19 @@
+import { Injectable } from '@nestjs/common';
+import { SecurityRuleRepository } from './security-rule.repository';
+import { SolidBaseRepository } from './solid-base.repository';
+import { RequestContextService } from 'src/services/request-context.service';
+import { DataSource } from 'typeorm';
+import { InjectDataSource } from '@nestjs/typeorm';
+import { WorkflowExecutionArtifact } from '../entities/workflow-execution-artifact.entity';
+
+@Injectable()
+export class WorkflowExecutionArtifactRepository extends SolidBaseRepository<WorkflowExecutionArtifact> {
+    constructor(
+        @InjectDataSource("default")
+        readonly dataSource: DataSource,
+        readonly requestContextService: RequestContextService,
+        readonly securityRuleRepository: SecurityRuleRepository,
+    ) {
+        super(WorkflowExecutionArtifact, dataSource, requestContextService, securityRuleRepository);
+    }
+}

package/src/repository/workflow-execution-log.repository.ts

@@ -0,0 +1,19 @@
+import { Injectable } from '@nestjs/common';
+import { SecurityRuleRepository } from './security-rule.repository';
+import { SolidBaseRepository } from './solid-base.repository';
+import { RequestContextService } from 'src/services/request-context.service';
+import { DataSource } from 'typeorm';
+import { InjectDataSource } from '@nestjs/typeorm';
+import { WorkflowExecutionLog } from '../entities/workflow-execution-log.entity';
+
+@Injectable()
+export class WorkflowExecutionLogRepository extends SolidBaseRepository<WorkflowExecutionLog> {
+    constructor(
+        @InjectDataSource("default")
+        readonly dataSource: DataSource,
+        readonly requestContextService: RequestContextService,
+        readonly securityRuleRepository: SecurityRuleRepository,
+    ) {
+        super(WorkflowExecutionLog, dataSource, requestContextService, securityRuleRepository);
+    }
+}

package/src/repository/workflow-execution.repository.ts

@@ -0,0 +1,19 @@
+import { Injectable } from '@nestjs/common';
+import { SecurityRuleRepository } from './security-rule.repository';
+import { SolidBaseRepository } from './solid-base.repository';
+import { RequestContextService } from 'src/services/request-context.service';
+import { DataSource } from 'typeorm';
+import { InjectDataSource } from '@nestjs/typeorm';
+import { WorkflowExecution } from '../entities/workflow-execution.entity';
+
+@Injectable()
+export class WorkflowExecutionRepository extends SolidBaseRepository<WorkflowExecution> {
+    constructor(
+        @InjectDataSource("default")
+        readonly dataSource: DataSource,
+        readonly requestContextService: RequestContextService,
+        readonly securityRuleRepository: SecurityRuleRepository,
+    ) {
+        super(WorkflowExecution, dataSource, requestContextService, securityRuleRepository);
+    }
+}

package/src/repository/workflow-secret.repository.ts

@@ -0,0 +1,19 @@
+import { Injectable } from '@nestjs/common';
+import { SecurityRuleRepository } from './security-rule.repository';
+import { SolidBaseRepository } from './solid-base.repository';
+import { RequestContextService } from 'src/services/request-context.service';
+import { DataSource } from 'typeorm';
+import { InjectDataSource } from '@nestjs/typeorm';
+import { WorkflowSecret } from '../entities/workflow-secret.entity';
+
+@Injectable()
+export class WorkflowSecretRepository extends SolidBaseRepository<WorkflowSecret> {
+    constructor(
+        @InjectDataSource("default")
+        readonly dataSource: DataSource,
+        readonly requestContextService: RequestContextService,
+        readonly securityRuleRepository: SecurityRuleRepository,
+    ) {
+        super(WorkflowSecret, dataSource, requestContextService, securityRuleRepository);
+    }
+}

package/src/repository/workflow-step-execution.repository.ts

@@ -0,0 +1,19 @@
+import { Injectable } from '@nestjs/common';
+import { SecurityRuleRepository } from './security-rule.repository';
+import { SolidBaseRepository } from './solid-base.repository';
+import { RequestContextService } from 'src/services/request-context.service';
+import { DataSource } from 'typeorm';
+import { InjectDataSource } from '@nestjs/typeorm';
+import { WorkflowStepExecution } from '../entities/workflow-step-execution.entity';
+
+@Injectable()
+export class WorkflowStepExecutionRepository extends SolidBaseRepository<WorkflowStepExecution> {
+    constructor(
+        @InjectDataSource("default")
+        readonly dataSource: DataSource,
+        readonly requestContextService: RequestContextService,
+        readonly securityRuleRepository: SecurityRuleRepository,
+    ) {
+        super(WorkflowStepExecution, dataSource, requestContextService, securityRuleRepository);
+    }
+}

package/src/repository/workflow-trigger-execution.repository.ts

@@ -0,0 +1,19 @@
+import { Injectable } from '@nestjs/common';
+import { SecurityRuleRepository } from './security-rule.repository';
+import { SolidBaseRepository } from './solid-base.repository';
+import { RequestContextService } from 'src/services/request-context.service';
+import { DataSource } from 'typeorm';
+import { InjectDataSource } from '@nestjs/typeorm';
+import { WorkflowTriggerExecution } from '../entities/workflow-trigger-execution.entity';
+
+@Injectable()
+export class WorkflowTriggerExecutionRepository extends SolidBaseRepository<WorkflowTriggerExecution> {
+    constructor(
+        @InjectDataSource("default")
+        readonly dataSource: DataSource,
+        readonly requestContextService: RequestContextService,
+        readonly securityRuleRepository: SecurityRuleRepository,
+    ) {
+        super(WorkflowTriggerExecution, dataSource, requestContextService, securityRuleRepository);
+    }
+}

package/src/seeders/module-metadata-seeder.service.ts

@@ -3,6 +3,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
+import { createHash } from 'crypto';
 import { v4 as uuidv4 } from 'uuid';
 
 import { CreateEmailTemplateDto } from 'src/dtos/create-email-template.dto';
@@ -66,6 +67,7 @@ import { FieldMetadata } from 'src/entities/field-metadata.entity';
 import { ModelMetadata } from 'src/entities/model-metadata.entity';
 import { PermissionMetadata } from 'src/entities/permission-metadata.entity';
 import { ViewMetadata } from 'src/entities/view-metadata.entity';
+import { WorkflowDefinition } from 'src/entities/workflow-definition.entity';
 import { EventDetails, EventType, ModuleMetadataSeederEventPayload } from 'src/interfaces';
 
 /**
@@ -236,10 +238,15 @@ export class ModuleMetadataSeederService {
                         const moduleModelFieldCounts = await this.timeOperation('seed-module-model-fields', () => this.seedModuleModelFields(moduleMetadata), { moduleName: moduleMetadata.name, component: 'module-model-fields' });
                         console.log(`${this.formatSeedResult(moduleMetadata.name, 'Module/Model/Fields', moduleModelFieldCounts)}`);
 
-                currentStep = 'seedLocales';
-                this.logger.log(`Seeding Locales`);
-                const localeCounts = await this.seedLocales(overallMetadata);
-                console.log(`${this.formatSeedResult(moduleMetadata.name, 'Locales', localeCounts)}`);
+                        currentStep = 'seedWorkflowDefinitions';
+                        this.logger.log(`Seeding Workflow Definitions`);
+                        const workflowDefinitionCounts = await this.timeOperation('seed-workflow-definitions', () => this.seedWorkflowDefinitions(moduleMetadata, overallMetadata), { moduleName: moduleMetadata.name, component: 'workflow-definitions' });
+                        console.log(`${this.formatSeedResult(moduleMetadata.name, 'Workflow Definitions', workflowDefinitionCounts)}`);
+
+                        currentStep = 'seedLocales';
+                        this.logger.log(`Seeding Locales`);
+                        const localeCounts = await this.seedLocales(overallMetadata);
+                        console.log(`${this.formatSeedResult(moduleMetadata.name, 'Locales', localeCounts)}`);
 
                         currentStep = 'seedPermissions';
                         this.logger.log(`Seeding Permissions`);
@@ -550,6 +557,28 @@ export class ModuleMetadataSeederService {
         return { pruned, upserted: savedFilters.length };
     }
 
+    private async seedWorkflowDefinitions(moduleMetadata: CreateModuleMetadataDto, overallMetadata: any): Promise<{ pruned: number; upserted: number }> {
+        const workflowDefinitions = this.getSeedArray<any>(overallMetadata?.workflowDefinitions)
+            .map((definition) => ({
+                ...definition,
+                moduleUserKey: definition.moduleUserKey ?? moduleMetadata.name,
+                definitionYaml: this.resolveSeedFileBackedWorkflowDefinitionYaml(definition.definitionYaml, overallMetadata?.__seedFilePath),
+            }));
+        const pruned = this.enablePruning ? await this.timeOperation('prune-workflow-definitions', () => this.pruneWorkflowDefinitions(workflowDefinitions, moduleMetadata.name), {
+            moduleName: moduleMetadata.name,
+            component: 'workflow-definitions',
+            serviceCall: 'pruneWorkflowDefinitions',
+        }) : 0;
+        if (workflowDefinitions.length > 0) {
+            await this.timeOperation('handle-workflow-definitions', () => this.handleSeedWorkflowDefinitions(workflowDefinitions), {
+                moduleName: moduleMetadata.name,
+                component: 'workflow-definitions',
+                serviceCall: 'handleSeedWorkflowDefinitions',
+            });
+        }
+        return { pruned, upserted: workflowDefinitions.length };
+    }
+
     private async seedListOfValues(moduleMetadata: CreateModuleMetadataDto, overallMetadata: any): Promise<{ pruned: number; upserted: number }> {
         const listOfValues = this.getSeedArray<CreateListOfValuesDto>(overallMetadata?.listOfValues);
         const pruned = this.enablePruning ? await this.timeOperation('prune-list-of-values', () => this.pruneListOfValues(listOfValues, moduleMetadata.name), {
@@ -785,6 +814,7 @@ export class ModuleMetadataSeederService {
 
             if (fs.existsSync(fullPath)) {
                 const overallMetadata = JSON.parse(fs.readFileSync(fullPath, 'utf-8'));
+                overallMetadata.__seedFilePath = fullPath;
                 seedDataFiles.push(overallMetadata);
             }
         }
@@ -792,6 +822,28 @@ export class ModuleMetadataSeederService {
         return seedDataFiles;
     }
 
+    private resolveSeedFileBackedWorkflowDefinitionYaml(definitionYaml: string, seedFilePath?: string): string {
+        if (typeof definitionYaml !== 'string' || !definitionYaml.startsWith('file:')) {
+            return definitionYaml;
+        }
+
+        if (!seedFilePath) {
+            throw new Error(`Cannot resolve workflow definition YAML file reference "${definitionYaml}" because the seed file path is unavailable.`);
+        }
+
+        const relativePath = definitionYaml.slice('file:'.length).trim();
+        if (!relativePath) {
+            throw new Error(`Cannot resolve workflow definition YAML file reference "${definitionYaml}" because no relative path was provided.`);
+        }
+
+        const resolvedPath = path.resolve(path.dirname(seedFilePath), relativePath);
+        if (!fs.existsSync(resolvedPath)) {
+            throw new Error(`Workflow definition YAML file "${resolvedPath}" referenced from "${seedFilePath}" does not exist.`);
+        }
+
+        return fs.readFileSync(resolvedPath, 'utf-8');
+    }
+
     // OK
     private async seedGlobalMetadata() {
         this.logger.log(`Seeding Permissions`);
@@ -2147,6 +2199,52 @@ export class ModuleMetadataSeederService {
         }
     }
 
+    private async handleSeedWorkflowDefinitions(workflowDefinitions: any[]) {
+        if (!workflowDefinitions || workflowDefinitions.length === 0) {
+            this.logger.debug(`No workflow definitions found to seed`);
+            return;
+        }
+
+        const repo = this.dataSource.getRepository(WorkflowDefinition);
+        for (const definition of workflowDefinitions) {
+            const moduleMetadata = await this.getModuleByUserKeyCached(definition.moduleUserKey, {
+                moduleName: definition.moduleUserKey,
+                component: 'workflow-definitions',
+                details: `module=${definition.moduleUserKey}`,
+            });
+
+            if (!moduleMetadata) {
+                throw new Error(`Cannot seed workflow definition "${definition.key}": module metadata "${definition.moduleUserKey}" was not found.`);
+            }
+
+            const existing = await repo.findOne({
+                where: {
+                    key: definition.key,
+                } as any,
+            });
+            const payload = repo.create({
+                ...existing,
+                key: definition.key,
+                moduleMetadata,
+                displayName: definition.displayName,
+                namespace: definition.namespace,
+                description: definition.description,
+                status: definition.status ?? 'active',
+                definitionVersion: definition.definitionVersion,
+                definitionChecksum: this.workflowDefinitionChecksum(definition.definitionYaml),
+                definitionYaml: definition.definitionYaml,
+                tags: definition.tags,
+            });
+
+            await this.timeOperation('workflow-definition-save', () => repo.save(payload), {
+                moduleName: definition.moduleUserKey,
+                component: 'workflow-definitions',
+                serviceCall: 'workflowDefinitionRepository.save',
+                details: `workflow=${definition.key}`,
+            });
+        }
+    }
+
     private async handleSeedLocales(localesDto: CreateLocaleDto[]) {
         if (!localesDto || localesDto.length === 0) {
             this.logger.debug(`No locales found to seed`);
@@ -2396,6 +2494,49 @@ export class ModuleMetadataSeederService {
         return 0;
     }
 
+    private async pruneWorkflowDefinitions(workflowDefinitionsDto: any[] | undefined, moduleName?: string): Promise<number> {
+        if (!moduleName) {
+            this.logger.warn(`Skipping workflow definition prune: missing module name in metadata.`);
+            return 0;
+        }
+        const workflowDefinitions = workflowDefinitionsDto ?? [];
+
+        const module = await this.getModuleByUserKeyCached(moduleName, {
+            moduleName,
+            component: 'workflow-definitions',
+            details: `module=${moduleName}`,
+        });
+        if (!module) {
+            this.logger.warn(`Skipping workflow definition prune: module not found for ${moduleName}.`);
+            return 0;
+        }
+
+        const workflowKeys = [...new Set(workflowDefinitions.map(dto => dto.key).filter(Boolean))];
+        const repo = this.dataSource.getRepository(WorkflowDefinition);
+        const idsToDeleteQuery = repo
+            .createQueryBuilder('workflowDefinition')
+            .select('workflowDefinition.id', 'id')
+            .innerJoin('workflowDefinition.moduleMetadata', 'moduleMetadata')
+            .where('moduleMetadata.id = :moduleId', { moduleId: module.id });
+
+        if (workflowKeys.length > 0) {
+            idsToDeleteQuery.andWhere('workflowDefinition.key NOT IN (:...workflowKeys)', { workflowKeys });
+        }
+
+        const rows = await idsToDeleteQuery.getRawMany();
+        const ids = rows.map((row) => row.id);
+        if (ids.length > 0) {
+            const result = await repo
+                .createQueryBuilder()
+                .delete()
+                .from(WorkflowDefinition)
+                .whereInIds(ids)
+                .execute();
+            return result.affected ?? 0;
+        }
+        return 0;
+    }
+
     private async pruneListOfValues(listOfValuesDto: CreateListOfValuesDto[] | undefined, moduleName?: string): Promise<number> {
         if (!moduleName) {
             this.logger.warn(`Skipping list of values prune: missing module name in metadata.`);
@@ -2824,4 +2965,25 @@ export class ModuleMetadataSeederService {
         return `✔ [${moduleName}] ${label} seeded (upserted ${counts.upserted})`;
     }
 
+    private workflowDefinitionChecksum(value: any): string {
+        return createHash('sha256')
+            .update(typeof value === 'string' ? value : this.stableStringify(value))
+            .digest('hex');
+    }
+
+    private stableStringify(value: any): string {
+        if (Array.isArray(value)) {
+            return `[${value.map((entry) => this.stableStringify(entry)).join(',')}]`;
+        }
+
+        if (value && typeof value === 'object') {
+            return `{${Object.keys(value)
+                .sort()
+                .map((key) => `${JSON.stringify(key)}:${this.stableStringify(value[key])}`)
+                .join(',')}}`;
+        }
+
+        return JSON.stringify(value);
+    }
+
 }