@wildo-ai/saas-backend-lib 1.0.1

package/src/queue/job-executor.backend.service.ts

@@ -0,0 +1,315 @@
+/**
+ * Job Executor Backend Service
+ *
+ * Executes queued jobs on worker pods.
+ * Handles resource operations, custom batches, and scheduled crons.
+ */
+
+import { injectable, inject } from 'inversify';
+import * as os from 'os';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';
+import { ServicesRegistryHandlerBackendService } from '~wildo-backend-lib/base-classes/services/services/services-registry-handler.backend.service';
+import { ExecutionContextCreatorBackendService } from '~wildo-backend-lib/runtime/services/execution-context-creator.backend.service';
+import { BatchesCronjobsExecutor_BackendService } from '~wildo-backend-lib/base-classes/services/batches/batches-cronjobs-executor.backend.service';
+import { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service';
+import { JobPayloadStorageBackendService } from './job-payload-storage.backend.service';
+import {
+  JobPayload,
+  JobResult,
+  JobStatus,
+  JobType,
+  ErrorType,
+  ErrorCustomMessageReference,
+  ResourceSerializationUtils,
+  ResourcesConfig_OperationPathUtils,
+  WildoError_Backend,
+} from '@wildo-ai/saas-models';
+
+/**
+ * Type guard to check if an error is a WildoError_Backend.
+ * Uses duck typing since WildoError_Backend is a type, not a class.
+ */
+function isWildoError(error: unknown): error is WildoError_Backend {
+  return (
+    error !== null &&
+    typeof error === 'object' &&
+    'type' in error &&
+    'statusCode' in error &&
+    'backendActions' in error
+  );
+}
+import type { ConsumeMessage, Channel } from 'amqplib';
+
+@injectable()
+export class JobExecutorBackendService {
+  protected logDebug: (message: string, context?: Record<string, unknown>) => void;
+  private readonly workerId: string;
+
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.ServicesRegistryHandlerService)
+    private readonly servicesRegistry: ServicesRegistryHandlerBackendService,
+    @inject(SAAS_SERVICE_TYPES.ExecutionContextCreatorBackendService)
+    private readonly executionContextCreator: ExecutionContextCreatorBackendService,
+    @inject(SAAS_SERVICE_TYPES.BatchesCronjobsExecutorService)
+    private readonly batchExecutor: BatchesCronjobsExecutor_BackendService,
+    @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)
+    private readonly adapter: RabbitMQAdapterBackendService,
+    @inject(SAAS_SERVICE_TYPES.JobPayloadStorageService)
+    private readonly payloadStorage: JobPayloadStorageBackendService,
+    @inject(SAAS_SERVICE_TYPES.Logger)
+    private readonly logger: LoggerService,
+    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)
+    private readonly errorBuilder: ErrorBuilderBackendService,
+  ) {
+    this.logDebug = this.logger.debugByType('JobExecutorBackendService');
+    this.logger.setDebugTypeLogging('JobExecutorBackendService', false);
+    this.workerId = process.env.K8S_POD_NAME || os.hostname();
+  }
+
+  /**
+   * Process a message from the queue.
+   * Main entry point called by consumer.
+   */
+  public async processMessage(msg: ConsumeMessage, channel: Channel): Promise<void> {
+    let job: JobPayload;
+
+    try {
+      job = JSON.parse(msg.content.toString());
+    } catch (error) {
+      this.logger.error('Failed to parse job message', {
+        error: error instanceof Error ? error.message : 'Unknown error',
+      });
+      channel.nack(msg, false, false); // Don't requeue malformed messages
+      return;
+    }
+
+    const startedAt = new Date();
+    this.logger.info('Executing job', {
+      jobId: job.jobId,
+      jobType: job.jobType,
+      operationPathKey: job.operationPathKey,
+      workerId: this.workerId,
+    });
+
+    try {
+      // Execute the job
+      const result = await this.execute(job);
+
+      // Send reply if RPC pattern
+      if (msg.properties.replyTo && msg.properties.correlationId) {
+        await this.adapter.publishResult(
+          msg.properties.replyTo,
+          msg.properties.correlationId,
+          result
+        );
+      }
+
+      // Acknowledge message
+      channel.ack(msg);
+
+      this.logger.info('Job completed', {
+        jobId: job.jobId,
+        status: result.status,
+        durationMs: result.durationMs,
+      });
+
+    } catch (error) {
+      // Build error result
+      const wildoError = isWildoError(error)
+        ? error
+        : this.errorBuilder.buildFromStandardError(error as Error);
+
+      const result: JobResult = {
+        jobId: job.jobId,
+        status: JobStatus.FAILED,
+        error: wildoError,
+        startedAt,
+        completedAt: new Date(),
+        durationMs: Date.now() - startedAt.getTime(),
+      };
+
+      // Send error reply if RPC pattern
+      if (msg.properties.replyTo && msg.properties.correlationId) {
+        await this.adapter.publishResult(
+          msg.properties.replyTo,
+          msg.properties.correlationId,
+          result
+        );
+      }
+
+      // Handle retry logic
+      if (job.retryCount < job.maxRetries) {
+        // Requeue for retry with delay (handled by DLQ with TTL or delayed message plugin)
+        this.logger.warn('Job failed, will retry', {
+          jobId: job.jobId,
+          retryCount: job.retryCount,
+          maxRetries: job.maxRetries,
+        });
+        channel.nack(msg, false, true); // Requeue
+      } else {
+        // Max retries exceeded, move to dead letter
+        this.logger.error('Job failed, max retries exceeded', {
+          jobId: job.jobId,
+          error: wildoError.customMessageReference,
+        });
+        channel.nack(msg, false, false); // Don't requeue (goes to DLQ)
+      }
+    }
+  }
+
+  /**
+   * Execute a job and return result.
+   */
+  public async execute(job: JobPayload): Promise<JobResult> {
+    const startedAt = new Date();
+
+    try {
+      // Resolve input data - check if stored externally first
+      let inputData: unknown;
+      if (job.inputPayloadStorageId) {
+        // Retrieve from external storage
+        inputData = await this.payloadStorage.retrieve(job.inputPayloadStorageId);
+        this.logDebug('Retrieved external payload', {
+          jobId: job.jobId,
+          storageId: job.inputPayloadStorageId,
+        });
+      } else if (job.inputDataJson) {
+        inputData = JSON.parse(job.inputDataJson);
+      }
+
+      // Execute based on job type
+      let result: any;
+
+      switch (job.jobType) {
+        case JobType.RESOURCE_OPERATION:
+          result = await this.executeResourceOperation(job, inputData);
+          break;
+        case JobType.CUSTOM_BATCH:
+          result = await this.executeCustomBatch(job, inputData);
+          break;
+        case JobType.SCHEDULED_CRON:
+          result = await this.executeScheduledCron(job, inputData);
+          break;
+        default:
+          throw this.errorBuilder.buildError(
+            ErrorType.INTERNAL,
+            undefined,
+            {
+              customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_FAILED,
+              context: { jobType: job.jobType, message: 'Unknown job type' },
+            }
+          );
+      }
+
+      // Build success result
+      const jobResult: JobResult = {
+        jobId: job.jobId,
+        status: JobStatus.COMPLETED,
+        resultJson: result !== undefined ? JSON.stringify(result) : undefined,
+        startedAt,
+        completedAt: new Date(),
+        durationMs: Date.now() - startedAt.getTime(),
+      };
+
+      return jobResult;
+
+    } catch (error) {
+      // Convert to WildoError
+      const wildoError = isWildoError(error)
+        ? error
+        : this.errorBuilder.buildFromStandardError(error as Error);
+
+      const jobResult: JobResult = {
+        jobId: job.jobId,
+        status: JobStatus.FAILED,
+        error: wildoError,
+        startedAt,
+        completedAt: new Date(),
+        durationMs: Date.now() - startedAt.getTime(),
+      };
+
+      return jobResult;
+    }
+  }
+
+  /**
+   * Execute resource operation with reconstructed execution context.
+   */
+  private async executeResourceOperation(job: JobPayload, inputData: unknown): Promise<any> {
+    if (!job.executionContextJson || !job.operationPathJson) {
+      throw this.errorBuilder.buildError(
+        ErrorType.CONFIGURATION,
+        undefined,
+        {
+          customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_CONTEXT_INVALID,
+          context: { message: 'Resource operation requires executionContextJson and operationPathJson' },
+        }
+      );
+    }
+
+    // Reconstruct execution context from serialized data
+    const executionContext = await this.executionContextCreator.reconstructFromSerializedData(
+      job.executionContextJson,
+      job.operationPathJson
+    );
+
+    // Execute via services registry with isWorkerExecution flag
+    // This prevents infinite loop by skipping serviceRuntimeMode check
+    return this.servicesRegistry.doByExecutionContext(executionContext, inputData, {
+      isWorkerExecution: true,
+    });
+  }
+
+  /**
+   * Execute custom batch job.
+   */
+  private async executeCustomBatch(job: JobPayload, inputData: unknown): Promise<any> {
+    if (!job.batchRef) {
+      throw this.errorBuilder.buildError(
+        ErrorType.CONFIGURATION,
+        undefined,
+        {
+          customMessageReference: ErrorCustomMessageReference.JOB_BATCH_REF_NOT_FOUND,
+          context: { message: 'Custom batch job requires batchRef' },
+        }
+      );
+    }
+
+    // Custom batch executor creates its own internal context
+    return this.batchExecutor.executeCustomBatch(job.batchRef, inputData, new Map());
+  }
+
+  /**
+   * Execute scheduled cron job (after JWT verification in consumer).
+   */
+  private async executeScheduledCron(job: JobPayload, inputData: unknown): Promise<any> {
+    // Scheduled jobs can be either resource-oriented or custom batches
+    if (job.operationPathKey) {
+      // Resource-oriented scheduled job
+      // operationPathKey is in format: "resource | operation | variantType | variant | isBulk:false"
+      const operationPath = ResourcesConfig_OperationPathUtils.parseOperationPathKey(job.operationPathKey);
+      return this.batchExecutor.executeBatchFromResource(operationPath, inputData);
+    } else if (job.batchRef) {
+      // Custom batch
+      return this.batchExecutor.executeCustomBatch(job.batchRef, inputData, new Map());
+    }
+
+    throw this.errorBuilder.buildError(
+      ErrorType.CONFIGURATION,
+      undefined,
+      {
+        customMessageReference: ErrorCustomMessageReference.JOB_EXECUTION_FAILED,
+        context: { message: 'Scheduled job has neither operationPathKey nor batchRef' },
+      }
+    );
+  }
+
+  /**
+   * Get worker ID for this executor.
+   */
+  public getWorkerId(): string {
+    return this.workerId;
+  }
+}

package/src/queue/job-payload-storage.backend.service.ts

@@ -0,0 +1,297 @@
+/**
+ * Job Payload Storage Backend Service
+ *
+ * Stores large job payloads (>100KB) in MongoDB to avoid RabbitMQ performance issues.
+ * Referenced by storageId in JobPayload.inputPayloadStorageId.
+ *
+ * Uses the same pattern as FilesBackendService:
+ * - Get repository via RepositoriesRegistryHandlerBackendService
+ * - Create internal execution context via getOperationPathDefault
+ * - Use direct repository methods (create, read, delete)
+ *
+ * Features:
+ * - Automatic compression for payloads > 10KB
+ * - TTL-based automatic cleanup (24 hours default)
+ * - Storage size tracking
+ */
+
+import { injectable, inject } from 'inversify';
+import { v4 as uuidv4 } from 'uuid';
+import * as zlib from 'zlib';
+import { promisify } from 'util';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';
+import type { BackendResourceRepository } from '~wildo-backend-lib/base-classes/services/repositories/repositories-registry-handler.backend.service';
+import { RepositoriesRegistryHandlerBackendService } from '~wildo-backend-lib/base-classes/services/repositories/repositories-registry-handler.backend.service';
+import { ExecutionContextCreatorBackendService } from '~wildo-backend-lib/runtime/services/execution-context-creator.backend.service';
+import { ResourcesRegistryBackendService } from '~wildo-backend-lib/resources/services/resources-registry.backend.service';
+import { RepositoryOptions } from '~wildo-backend-lib/base-classes/models/repositories.backend.schemas';
+import {
+  JobPayloadStorage,
+  PayloadCompressionType,
+  PAYLOAD_SIZE_THRESHOLD_BYTES,
+  ErrorType,
+  ErrorCustomMessageReference,
+  CoreResourceType,
+  CoreResourceOperation,
+  JOB_PAYLOAD_STORAGE_TTL_SECONDS,
+} from '@wildo-ai/saas-models';
+import { ExecutionContext } from '~wildo-backend-lib/runtime/models/execution-context.backend.schemas';
+import { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';
+
+const gzip = promisify(zlib.gzip);
+const gunzip = promisify(zlib.gunzip);
+
+/**
+ * Default expiry for stored payloads in milliseconds.
+ */
+const DEFAULT_EXPIRY_MS = JOB_PAYLOAD_STORAGE_TTL_SECONDS * 1000;
+
+/**
+ * Compression threshold: compress payloads > 10KB for storage efficiency.
+ */
+const COMPRESSION_THRESHOLD_BYTES = 10 * 1024;
+
+@injectable()
+export class JobPayloadStorageBackendService {
+  protected logDebug: (message: string, context?: Record<string, unknown>) => void;
+  private _applicationId: string | null = null;
+
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.AppConfigService)
+    private readonly appConfigService: AppConfigurationService,
+    @inject(SAAS_SERVICE_TYPES.Logger)
+    private readonly logger: LoggerService,
+    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)
+    private readonly errorBuilder: ErrorBuilderBackendService,
+    @inject(SAAS_SERVICE_TYPES.RepositoriesRegistryHandlerService)
+    private readonly repositoriesRegistry: RepositoriesRegistryHandlerBackendService,
+    @inject(SAAS_SERVICE_TYPES.ExecutionContextCreatorBackendService)
+    private readonly executionContextCreator: ExecutionContextCreatorBackendService,
+    @inject(SAAS_SERVICE_TYPES.ResourceRegistryService)
+    private readonly resourcesRegistry: ResourcesRegistryBackendService,
+  ) {
+    this.logDebug = this.logger.debugByType('JobPayloadStorageBackendService');
+    this.logger.setDebugTypeLogging('JobPayloadStorageBackendService', false);
+  }
+
+  /** Lazy getter for applicationId */
+  private get applicationId(): string {
+    if (!this._applicationId) {
+      this._applicationId = this.appConfigService.applicationId;
+    }
+    return this._applicationId;
+  }
+
+  /**
+   * Get the resource type for this service.
+   */
+  static get resourceType(): CoreResourceType {
+    return CoreResourceType.JOB_PAYLOAD_STORAGE;
+  }
+
+  /**
+   * Get repository for job payload storage.
+   * Lazy getter to ensure repository is initialized.
+   */
+  private get payloadRepository(): BackendResourceRepository<any, any, any> {
+    return this.repositoriesRegistry.getRepositoryOrThrow(
+      CoreResourceType.JOB_PAYLOAD_STORAGE,
+      {} as ExecutionContext<any>
+    );
+  }
+
+  /**
+   * Create an internal execution context for payload storage operations.
+   * Uses default operation path (no variant) following FilesBackendService pattern.
+   */
+  private async createInternalContext(
+    operation: CoreResourceOperation
+  ): Promise<ExecutionContext<any>> {
+    const operationPath = this.resourcesRegistry.getOperationPathDefault(
+      CoreResourceType.JOB_PAYLOAD_STORAGE,
+      operation
+    );
+
+    return this.executionContextCreator.createForInternalOperation({
+      operation: operationPath,
+      contextResourcesFieldIdentifier: new Map(),
+      initiatorIds: null,
+      initiatorRoles: [],
+    });
+  }
+
+  /**
+   * Check if payload should be stored externally.
+   *
+   * @param payload - Payload to check (will be stringified if not already)
+   * @returns True if payload exceeds threshold
+   */
+  shouldStore(payload: unknown): boolean {
+    const json = typeof payload === 'string' ? payload : JSON.stringify(payload);
+    return Buffer.byteLength(json, 'utf8') >= PAYLOAD_SIZE_THRESHOLD_BYTES;
+  }
+
+  /**
+   * Store a payload and return the storage ID.
+   * Compresses if payload is large enough to benefit from it.
+   *
+   * @param payload - Payload to store (will be stringified)
+   * @param expiryMs - Time until expiry in milliseconds (default: 24 hours)
+   * @param options - Optional repository options
+   * @returns Storage ID that can be used to retrieve the payload
+   */
+  async store(
+    payload: unknown,
+    expiryMs: number = DEFAULT_EXPIRY_MS,
+    options?: RepositoryOptions
+  ): Promise<string> {
+    const payloadJson = typeof payload === 'string' ? payload : JSON.stringify(payload);
+    const payloadSizeBytes = Buffer.byteLength(payloadJson, 'utf8');
+
+    // Determine if compression is beneficial
+    let storedPayload = payloadJson;
+    let isCompressed = false;
+    let compressionType = PayloadCompressionType.NONE;
+
+    if (payloadSizeBytes > COMPRESSION_THRESHOLD_BYTES) {
+      try {
+        const compressed = await gzip(Buffer.from(payloadJson, 'utf8'));
+        // Only use compression if it actually reduces size
+        if (compressed.length < payloadSizeBytes * 0.8) {
+          storedPayload = compressed.toString('base64');
+          isCompressed = true;
+          compressionType = PayloadCompressionType.GZIP;
+
+          this.logDebug('Payload compressed', {
+            originalSize: payloadSizeBytes,
+            compressedSize: compressed.length,
+            ratio: (compressed.length / payloadSizeBytes).toFixed(2),
+          });
+        }
+      } catch (error) {
+        this.logger.warn('Payload compression failed, storing uncompressed', {
+          error: error instanceof Error ? error.message : 'Unknown error',
+        });
+      }
+    }
+
+    const storageId = uuidv4();
+    const now = new Date();
+
+    // Build storage record
+    const createData: Partial<JobPayloadStorage> = {
+      storageId,
+      applicationId: this.applicationId,
+      payloadJson: storedPayload,
+      payloadSizeBytes,
+      isCompressed,
+      compressionType,
+      createdAt: now,
+      expiresAt: new Date(now.getTime() + expiryMs),
+    };
+
+    // Create via repository
+    const context = await this.createInternalContext(CoreResourceOperation.CREATE);
+    await this.payloadRepository.create<JobPayloadStorage>(
+      context,
+      createData as JobPayloadStorage,
+      options
+    );
+
+    this.logDebug('Payload stored', {
+      storageId,
+      sizeBytes: payloadSizeBytes,
+      isCompressed,
+    });
+
+    return storageId;
+  }
+
+  /**
+   * Retrieve a stored payload by storage ID.
+   * Automatically decompresses if needed.
+   *
+   * @param storageId - Storage ID from store()
+   * @param options - Optional repository options
+   * @returns Parsed payload or null if not found
+   */
+  async retrieve<T = unknown>(
+    storageId: string,
+    options?: RepositoryOptions
+  ): Promise<T | null> {
+    const context = await this.createInternalContext(CoreResourceOperation.READ);
+
+    const storage = await this.payloadRepository.read<JobPayloadStorage>(
+      context,
+      { storageId },
+      options
+    );
+
+    if (!storage) {
+      this.logDebug('Payload storage not found', { storageId });
+      return null;
+    }
+
+    let payloadJson = storage.payloadJson;
+
+    // Decompress if needed
+    if (storage.isCompressed && storage.compressionType === PayloadCompressionType.GZIP) {
+      try {
+        const compressed = Buffer.from(payloadJson, 'base64');
+        const decompressed = await gunzip(compressed);
+        payloadJson = decompressed.toString('utf8');
+      } catch (error) {
+        this.logger.error('Payload decompression failed', {
+          storageId,
+          error: error instanceof Error ? error.message : 'Unknown error',
+        });
+        throw this.errorBuilder.buildError(
+          ErrorType.INTERNAL,
+          undefined,
+          {
+            customMessageReference: ErrorCustomMessageReference.JOB_PAYLOAD_RETRIEVAL_FAILED,
+            context: { storageId, reason: 'Decompression failed' },
+          }
+        );
+      }
+    }
+
+    try {
+      return JSON.parse(payloadJson) as T;
+    } catch (error) {
+      this.logger.error('Payload JSON parse failed', {
+        storageId,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      });
+      throw this.errorBuilder.buildError(
+        ErrorType.INTERNAL,
+        undefined,
+        {
+          customMessageReference: ErrorCustomMessageReference.JOB_PAYLOAD_RETRIEVAL_FAILED,
+          context: { storageId, reason: 'JSON parse failed' },
+        }
+      );
+    }
+  }
+
+  /**
+   * Delete a stored payload by storage ID.
+   * Usually not needed as TTL handles cleanup, but available for immediate cleanup.
+   *
+   * @param storageId - Storage ID to delete
+   * @param options - Optional repository options
+   */
+  async delete(storageId: string, options?: RepositoryOptions): Promise<void> {
+    const context = await this.createInternalContext(CoreResourceOperation.DELETE);
+
+    await this.payloadRepository.delete(
+      context,
+      { storageId },
+      options
+    );
+
+    this.logDebug('Payload storage deleted', { storageId });
+  }
+}