@wildo-ai/saas-backend-lib 1.0.1

package/src/queue/scheduled-message-verifier.backend.service.ts

@@ -0,0 +1,228 @@
+/**
+ * Scheduled Message Verifier Backend Service
+ *
+ * Verifies JWT signatures on scheduled job messages from crontabs-batches-manager.
+ * Only messages signed by the platform's crontabs-batches-manager are accepted.
+ *
+ * This prevents unauthorized job execution in horizontally scaled K8s environments.
+ */
+
+import { injectable, inject } from 'inversify';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import { JwtBackendService } from '~wildo-backend-lib/security/jwt/services/jwt.backend.service';
+import { JwtPublicKeys } from '~wildo-backend-lib/security/authentications/models/jwt.backend.schemas';
+import { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';
+import {
+  JobPayload,
+  JobType,
+  JobInitiator,
+  ErrorType,
+  ErrorCustomMessageReference,
+} from '@wildo-ai/saas-models';
+import { z } from 'zod';
+
+/**
+ * Schema for the signed scheduled job token payload.
+ *
+ * This matches what ScheduledJobTokenSignerService in crontabs-batches-manager sends.
+ */
+const ScheduledJobTokenSchema = z.object({
+  /** Job ID (unique per execution) */
+  jobId: z.string().uuid(),
+
+  /** Application ID the job belongs to */
+  applicationId: z.string(),
+
+  /** Job type - always SCHEDULED_CRON */
+  jobType: z.literal(JobType.SCHEDULED_CRON),
+
+  /** Operation path key for resource operations (format: "resource | operation | variantType | variant | isBulk:false") */
+  operationPathKey: z.string().optional(),
+
+  /** Custom batch reference */
+  batchRef: z.string().optional(),
+
+  /** Input data for the job (JSON stringified) */
+  inputDataJson: z.string().optional(),
+
+  /** Token issued at (standard JWT claim) */
+  iat: z.number(),
+
+  /** Token expiry (standard JWT claim) */
+  exp: z.number(),
+
+  /** Issuer (should be PLATFORM_CRONTABS_BATCHES_MANAGER) */
+  iss: z.string(),
+
+  /** Audience (should be SCHEDULED_JOB) */
+  aud: z.string(),
+});
+
+export type ScheduledJobToken = z.infer<typeof ScheduledJobTokenSchema>;
+
+/**
+ * Result of scheduled message verification.
+ */
+export interface ScheduledMessageVerificationResult {
+  /** Whether the message is valid */
+  isValid: boolean;
+
+  /** The verified token payload (if valid) */
+  token?: ScheduledJobToken;
+
+  /** Error message (if invalid) */
+  error?: string;
+}
+
+@injectable()
+export class ScheduledMessageVerifierBackendService {
+  protected logDebug: (message: string, context?: Record<string, unknown>) => void;
+
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.JwtBackendService)
+    private readonly jwtService: JwtBackendService,
+    @inject(SAAS_SERVICE_TYPES.Logger)
+    private readonly logger: LoggerService,
+    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)
+    private readonly errorBuilder: ErrorBuilderBackendService,
+  ) {
+    this.logDebug = this.logger.debugByType('ScheduledMessageVerifierBackendService');
+    this.logger.setDebugTypeLogging('ScheduledMessageVerifierBackendService', false);
+  }
+
+  /**
+   * Verify a scheduled job message token.
+   *
+   * @param token - The JWT token from the message
+   * @returns Verification result with token payload if valid
+   */
+  async verifyScheduledMessage(token: string): Promise<ScheduledMessageVerificationResult> {
+    try {
+      // Verify the JWT using the crontabs-batches-manager public key
+      const decoded = await this.jwtService.verify(
+        token,
+        JwtPublicKeys.PLATFORM_CRONTABS_BATCHES_MANAGER
+      );
+
+      // Parse and validate the token payload
+      const parseResult = ScheduledJobTokenSchema.safeParse(decoded);
+
+      if (!parseResult.success) {
+        this.logger.warn('Scheduled message token has invalid structure', {
+          issues: parseResult.error.issues,
+        });
+        return {
+          isValid: false,
+          error: 'Invalid token structure',
+        };
+      }
+
+      const tokenPayload = parseResult.data;
+
+      // Validate issuer
+      if (tokenPayload.iss !== 'PLATFORM_CRONTABS_BATCHES_MANAGER') {
+        this.logger.warn('Scheduled message has invalid issuer', {
+          issuer: tokenPayload.iss,
+        });
+        return {
+          isValid: false,
+          error: 'Invalid token issuer',
+        };
+      }
+
+      // Validate audience
+      if (tokenPayload.aud !== 'SCHEDULED_JOB') {
+        this.logger.warn('Scheduled message has invalid audience', {
+          audience: tokenPayload.aud,
+        });
+        return {
+          isValid: false,
+          error: 'Invalid token audience',
+        };
+      }
+
+      this.logDebug('Scheduled message verified successfully', {
+        jobId: tokenPayload.jobId,
+        hasBatchRef: !!tokenPayload.batchRef,
+        hasOperationPathKey: !!tokenPayload.operationPathKey,
+        applicationId: tokenPayload.applicationId,
+      });
+
+      return {
+        isValid: true,
+        token: tokenPayload,
+      };
+
+    } catch (error) {
+      this.logger.warn('Scheduled message verification failed', {
+        error: error instanceof Error ? error.message : 'Unknown error',
+      });
+
+      return {
+        isValid: false,
+        error: error instanceof Error ? error.message : 'Token verification failed',
+      };
+    }
+  }
+
+  /**
+   * Convert a verified token to a JobPayload for execution.
+   *
+   * @param token - The verified token payload
+   * @returns JobPayload ready for execution
+   */
+  convertToJobPayload(token: ScheduledJobToken): JobPayload {
+    const now = new Date();
+
+    return {
+      jobId: token.jobId,
+      jobType: JobType.SCHEDULED_CRON,
+      applicationId: token.applicationId,
+
+      // operationPathKey is the human-readable key format, NOT JSON
+      // The executor will parse it using ResourcesConfig_OperationPathUtils.parseOperationPathKey()
+      operationPathKey: token.operationPathKey,
+      batchRef: token.batchRef,
+
+      inputDataJson: token.inputDataJson,
+
+      initiatedBy: JobInitiator.CRONTAB,
+      priority: 'normal',
+      retryCount: 0,
+      maxRetries: 3,
+      retryDelayMs: 1000,
+      timeoutMs: 300000,
+
+      createdAt: now,
+    };
+  }
+
+  /**
+   * Verify and convert a scheduled message in one step.
+   * Throws if verification fails.
+   *
+   * @param token - The JWT token from the message
+   * @returns JobPayload ready for execution
+   * @throws WildoError if verification fails
+   */
+  async verifyAndConvert(token: string): Promise<JobPayload> {
+    const result = await this.verifyScheduledMessage(token);
+
+    if (!result.isValid || !result.token) {
+      throw this.errorBuilder.buildError(
+        ErrorType.AUTHENTICATION,
+        undefined,
+        {
+          customMessageReference: ErrorCustomMessageReference.AUTHENTICATION_INVALID_TOKEN,
+          context: {
+            reason: result.error || 'Scheduled message verification failed',
+            source: 'crontabs-batches-manager',
+          },
+        }
+      );
+    }
+
+    return this.convertToJobPayload(result.token);
+  }
+}

package/src/queue/worker-shutdown-manager.backend.service.ts

@@ -0,0 +1,298 @@
+/**
+ * Worker Shutdown Manager Backend Service
+ *
+ * Handles graceful shutdown for queue workers in K8s environment.
+ *
+ * Kubernetes termination flow:
+ * 1. Pod receives SIGTERM
+ * 2. PreStop hook (if configured) gives extra time
+ * 3. terminationGracePeriodSeconds countdown begins
+ * 4. If not terminated, SIGKILL is sent
+ *
+ * Our handling:
+ * 1. Receive SIGTERM
+ * 2. Stop accepting new jobs (via registered stop handlers)
+ * 3. Wait for in-flight jobs to complete (with timeout)
+ * 4. Close connections
+ * 5. Exit cleanly
+ *
+ * Architecture: Uses Registry pattern to avoid circular dependencies.
+ * Components register themselves as "stoppable" via registerStopHandler().
+ */
+
+import { injectable, inject } from 'inversify';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import { RabbitMQAdapterBackendService } from './adapters/rabbitmq-adapter.backend.service';
+
+/**
+ * Shutdown configuration.
+ */
+export interface ShutdownConfig {
+  /**
+   * Maximum time to wait for in-flight jobs (default: 25s)
+   * Should be less than K8s terminationGracePeriodSeconds
+   */
+  gracePeriodMs?: number;
+
+  /**
+   * Interval to check for job completion (default: 500ms)
+   */
+  checkIntervalMs?: number;
+
+  /**
+   * Exit code on successful shutdown (default: 0)
+   */
+  exitCode?: number;
+}
+
+const DEFAULT_SHUTDOWN_CONFIG: Required<ShutdownConfig> = {
+  gracePeriodMs: 25000, // 25 seconds (K8s default is 30s)
+  checkIntervalMs: 500,
+  exitCode: 0,
+};
+
+/**
+ * Handler called during shutdown (e.g., to stop consuming).
+ * Called BEFORE waiting for in-flight jobs.
+ */
+type StopHandler = () => Promise<void>;
+
+/**
+ * Handler called during final cleanup (e.g., close connections).
+ * Called AFTER waiting for in-flight jobs.
+ */
+type ShutdownHandler = () => Promise<void>;
+
+@injectable()
+export class WorkerShutdownManagerBackendService {
+  protected logDebug: (message: string, context?: Record<string, unknown>) => void;
+  private isShuttingDown = false;
+  private inFlightJobCount = 0;
+
+  /**
+   * Handlers to stop accepting new work (called first).
+   * E.g., JobConsumer.stopConsuming(), DomainQueueConsumer.stopAll()
+   */
+  private stopHandlers: StopHandler[] = [];
+
+  /**
+   * Handlers for final cleanup (called last, in LIFO order).
+   * E.g., close database connections, flush logs
+   */
+  private shutdownHandlers: ShutdownHandler[] = [];
+
+  private signalsRegistered = false;
+
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.RabbitMQAdapterService)
+    private readonly adapter: RabbitMQAdapterBackendService,
+    @inject(SAAS_SERVICE_TYPES.Logger)
+    private readonly logger: LoggerService,
+  ) {
+    this.logDebug = this.logger.debugByType('WorkerShutdownManagerBackendService');
+    this.logger.setDebugTypeLogging('WorkerShutdownManagerBackendService', false);
+  }
+
+  /**
+   * Register shutdown signal handlers.
+   * Should be called once during worker startup.
+   */
+  registerSignalHandlers(config: ShutdownConfig = {}): void {
+    if (this.signalsRegistered) {
+      this.logger.warn('Shutdown handlers already registered');
+      return;
+    }
+
+    const mergedConfig = { ...DEFAULT_SHUTDOWN_CONFIG, ...config };
+
+    // Handle SIGTERM (K8s termination signal)
+    process.on('SIGTERM', async () => {
+      this.logDebug('Received SIGTERM, initiating graceful shutdown');
+      await this.initiateShutdown(mergedConfig);
+    });
+
+    // Handle SIGINT (Ctrl+C in development)
+    process.on('SIGINT', async () => {
+      this.logDebug('Received SIGINT, initiating graceful shutdown');
+      await this.initiateShutdown(mergedConfig);
+    });
+
+    // Handle uncaught exceptions
+    process.on('uncaughtException', async (error) => {
+      this.logger.error('Uncaught exception, forcing shutdown', {
+        error: error.message,
+        stack: error.stack,
+      });
+      await this.forceShutdown(1);
+    });
+
+    // Handle unhandled promise rejections
+    process.on('unhandledRejection', async (reason) => {
+      this.logger.error('Unhandled rejection, forcing shutdown', {
+        reason: reason instanceof Error ? reason.message : String(reason),
+      });
+      await this.forceShutdown(1);
+    });
+
+    this.signalsRegistered = true;
+    this.logDebug('Shutdown signal handlers registered');
+  }
+
+  /**
+   * Register a stop handler (called first during shutdown).
+   * Use for stopping consumers, accepting new requests, etc.
+   * Handlers are called in order of registration.
+   *
+   * @example
+   * // JobConsumer registers itself
+   * shutdownManager.registerStopHandler(() => this.stopConsuming());
+   */
+  registerStopHandler(handler: StopHandler): void {
+    this.stopHandlers.push(handler);
+    this.logDebug('Stop handler registered', { totalStopHandlers: this.stopHandlers.length });
+  }
+
+  /**
+   * Register a custom shutdown handler (called last during shutdown).
+   * Handlers are called in LIFO order.
+   * Use for final cleanup: close connections, flush logs, etc.
+   */
+  onShutdown(handler: ShutdownHandler): void {
+    this.shutdownHandlers.push(handler);
+  }
+
+  /**
+   * Increment in-flight job counter.
+   * Call when a job starts processing.
+   */
+  jobStarted(): void {
+    this.inFlightJobCount++;
+  }
+
+  /**
+   * Decrement in-flight job counter.
+   * Call when a job completes (success or failure).
+   */
+  jobCompleted(): void {
+    this.inFlightJobCount = Math.max(0, this.inFlightJobCount - 1);
+  }
+
+  /**
+   * Get current in-flight job count.
+   */
+  getInFlightCount(): number {
+    return this.inFlightJobCount;
+  }
+
+  /**
+   * Check if shutdown is in progress.
+   */
+  isShutdownInProgress(): boolean {
+    return this.isShuttingDown;
+  }
+
+  /**
+   * Initiate graceful shutdown.
+   */
+  async initiateShutdown(config: Required<ShutdownConfig>): Promise<void> {
+    if (this.isShuttingDown) {
+      this.logger.warn('Shutdown already in progress');
+      return;
+    }
+
+    this.isShuttingDown = true;
+    const startTime = Date.now();
+
+    try {
+      // Step 1: Stop accepting new work (run all stop handlers)
+      this.logDebug('Running stop handlers...', { count: this.stopHandlers.length });
+      for (const handler of this.stopHandlers) {
+        try {
+          await handler();
+        } catch (error) {
+          this.logger.error('Stop handler failed', {
+            error: error instanceof Error ? error.message : 'Unknown error',
+          });
+        }
+      }
+      this.logDebug('Stop handlers completed');
+
+      // Step 2: Wait for in-flight jobs to complete
+      this.logDebug('Waiting for in-flight jobs...', { count: this.inFlightJobCount });
+
+      const waitResult = await this.waitForInFlightJobs(config);
+
+      if (!waitResult) {
+        this.logger.warn('Grace period expired, some jobs may be lost', {
+          remainingJobs: this.inFlightJobCount,
+        });
+      }
+
+      // Step 3: Run custom shutdown handlers (LIFO)
+      this.logDebug('Running shutdown handlers...');
+      for (const handler of [...this.shutdownHandlers].reverse()) {
+        try {
+          await handler();
+        } catch (error) {
+          this.logger.error('Shutdown handler failed', {
+            error: error instanceof Error ? error.message : 'Unknown error',
+          });
+        }
+      }
+
+      // Step 4: Close RabbitMQ connection
+      this.logDebug('Closing RabbitMQ connection...');
+      await this.adapter.close();
+
+      // Step 5: Exit
+      const duration = Date.now() - startTime;
+      this.logDebug('Graceful shutdown complete', {
+        durationMs: duration,
+        remainingJobs: this.inFlightJobCount,
+      });
+
+      process.exit(config.exitCode);
+
+    } catch (error) {
+      this.logger.error('Shutdown error, forcing exit', {
+        error: error instanceof Error ? error.message : 'Unknown error',
+      });
+      await this.forceShutdown(1);
+    }
+  }
+
+  /**
+   * Wait for in-flight jobs to complete.
+   * Returns true if all jobs completed, false if timeout.
+   */
+  private async waitForInFlightJobs(config: Required<ShutdownConfig>): Promise<boolean> {
+    const deadline = Date.now() + config.gracePeriodMs;
+
+    while (this.inFlightJobCount > 0 && Date.now() < deadline) {
+      await new Promise(resolve => setTimeout(resolve, config.checkIntervalMs));
+
+      this.logDebug('Waiting for jobs', {
+        inFlight: this.inFlightJobCount,
+        remainingMs: deadline - Date.now(),
+      });
+    }
+
+    return this.inFlightJobCount === 0;
+  }
+
+  /**
+   * Force immediate shutdown.
+   */
+  private async forceShutdown(exitCode: number): Promise<void> {
+    this.logger.error('Forcing shutdown', { exitCode });
+
+    try {
+      await this.adapter.close();
+    } catch {
+      // Ignore errors during force shutdown
+    }
+
+    process.exit(exitCode);
+  }
+}

package/src/requests/models/api-routes.backend.defintions.ts

@@ -0,0 +1,45 @@
+export const API_ROUTES_BACKEND_DEFINITIONS = {
+    APPS_MANAGER: {
+        // Wildo Applications (called by wildo apps)
+        GET_APPLICATION_CONFIG: ()  => `/apps-to-manager/application-config`,
+        GET_APPLICATION_SECRETS: () => `/apps-to-manager/application-secrets`,
+        UPDATE_APPLICATION_CONFIG: () => `/apps-to-manager/application-config`,
+        UPDATE_APPLICATION_SECRETS: () => `/apps-to-manager/application-secrets`,
+        
+        // Platform Services (called by platform microservices)
+        GET_PLATFORM_SERVICE_CONFIG_SECRETS: () => `/platform-services-to-manager/config-secrets`,
+        
+        // Admin endpoints (secret-gated, workflow-oriented)
+        ADMIN_PLATFORM_INITIALIZE: () => `/admin/platform/initialize`,
+        ADMIN_PLATFORM_PROVISION_SERVICE: () => `/admin/platform/provision-service`,
+        ADMIN_PLATFORM_REFRESH: () => `/admin/platform/refresh`,
+        ADMIN_APPS_INITIALIZE: () => `/admin/apps/initialize`,
+        ADMIN_APP_INITIALIZE: (appSlug: string) => `/admin/apps/${appSlug}/initialize`,
+        ADMIN_APP_REFRESH: (appSlug: string) => `/admin/apps/${appSlug}/refresh`,
+        ADMIN_APP_DEPLOY: (appSlug: string) => `/admin/apps/${appSlug}/deploy`,
+        ADMIN_APP_UNDEPLOY: (appSlug: string) => `/admin/apps/${appSlug}/undeploy`,
+        ADMIN_APP_STATUS: (appSlug: string) => `/admin/apps/${appSlug}/status`,
+        ADMIN_APP_CREDENTIALS: (appSlug: string, env: string) => `/admin/apps/${appSlug}/credentials/${env}`,
+        ADMIN_STATUS: () => `/admin/status`,
+    },
+
+    LIFECYCLE_API: {
+        BASE_PATH: `/api/lifecycle`,
+        // Health
+        HEALTH: () => `/`,
+        // Meta endpoints
+        RESOURCES_REGISTRY: () => `/_meta/resources-registry`,
+        // Structure endpoints
+        STRUCTURE: () => `/structure`,
+        // File operations
+        FILES_UPDATE: () => `/files/update`,
+        FILE_GET: () => `/file`,
+        FILES_LIST: () => `/files/list`,
+        // Labelling operations
+        LABELLING_GENERATE_APP_SPECIFIC: () => `/labelling/generate-app-specific`,
+    },
+    FILE_STORAGE: {
+        STORAGE_KEY_APPLICATION_FILE: (appId: string, fileId: string) =>
+            `applications/${appId}/files/${fileId}`,
+    },
+}

package/src/requests/models/requests.backend.schemas.ts

@@ -0,0 +1,73 @@
+import { Request, Response, NextFunction } from 'express';
+import { Roles } from '@wildo-ai/saas-models';
+import type { AllApplicationTypes } from '~wildo-backend-lib/applications/models/applications.backend.schemas';
+
+
+export interface RequestWithLogging extends Request {
+  requestUUID: string;
+  startTime: number;
+  organizationId?: string;
+  userId?: string;
+}
+
+
+export interface BaseRequestContext {
+  userid?: string;
+  organizationId?: string;
+
+  ipAddress?: string;
+  userAgent?: string;
+
+  // Audit trail
+  requestUUID: string;
+  timestamp: Date;
+  forceTrace?: boolean;
+}
+
+
+
+export interface RequestContext extends BaseRequestContext {
+  // to be refined, deeply
+  contextParameters: Record<string, string>;
+  cacheObject: Record<string, any>;
+  sessionObject: Record<string, any>;
+  rolesAllowed: Roles[];
+
+
+}
+
+export interface PlatformRequestContext {
+  // High-level origin typing to distinguish who is calling the platform
+  origin: PlatformRequestOrigin;
+
+  // Identifiers from headers
+  serviceId?: AllApplicationTypes; // X-Service-Id
+  applicationId?: string; // X-Application-Id
+
+  // Auth headers
+  authorizationToken?: string; // Authorization: Bearer <token>
+  platformSecret?: string; // X-Platform-Secret (app-to-platform bootstrap)
+
+  // Observability / request metadata
+  requestUUID?: string; // X-Request-UUID
+  ipAddress?: string;
+  userAgent?: string;
+
+  // Raw headers (lower-cased keys as provided by Express)
+  headers: Record<string, string | string[] | undefined>;
+}
+
+export enum PlatformRequestOrigin {
+  PLATFORM_TO_PLATFORM = 'PLATFORM_TO_PLATFORM',
+  APP_TO_PLATFORM = 'APP_TO_PLATFORM',
+  UNKNOWN = 'UNKNOWN'
+}
+
+
+export interface ApiError {
+  message: string;
+  code: string;
+  statusCode: number;
+  details?: any;
+  stacktrace?: string;
+}