@wildo-ai/saas-backend-lib 1.0.1

package/src/files/services/file-rag-indexing.backend.service.ts

@@ -0,0 +1,229 @@
+/**
+ * File RAG Indexing Backend Service
+ *
+ * Orchestrates the RAG pipeline for file content:
+ *
+ *   extract text → chunk → embed → store vectors
+ *
+ * The service transitions files through EXTRACTING → INDEXED (or FAILED)
+ * and persists the `ragDocumentId` on the File record.
+ *
+ * Vector storage and embedding are **adapter-based**: the first slice provides
+ * the pipeline skeleton with pluggable adapters. Until a vector store adapter
+ * is bound, the service extracts and chunks but skips embedding/storage,
+ * logging a warning.
+ *
+ * Chunking strategies come from the `z_file({ rag })` metadata.
+ */
+
+import { injectable, inject } from 'inversify';
+import {
+  ErrorType,
+  ErrorCustomMessageReference,
+  FileStatus,
+  type File,
+} from '@wildo-ai/saas-models';
+import type { FileRAGConfiguration, FileSchemaMetadata } from '@wildo-ai/zod-decorators';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import type { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';
+import { ExecutionContext } from '~wildo-backend-lib/runtime/models/execution-context.backend.schemas';
+import { FileTextExtractionBackendService } from './file-text-extraction.backend.service';
+import { FilesBackendService } from './files.backend.service';
+import { WeaviateVectorStoreBackendService } from '~wildo-backend-lib/files/rag/weaviate-vector-store.backend.service';
+import { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Chunking
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface TextChunk {
+  index: number;
+  text: string;
+  startOffset: number;
+  endOffset: number;
+}
+
+const DEFAULT_CHUNK_SIZE = 512;
+const DEFAULT_CHUNK_OVERLAP = 50;
+
+function chunkText(
+  text: string,
+  chunkSize: number = DEFAULT_CHUNK_SIZE,
+  chunkOverlap: number = DEFAULT_CHUNK_OVERLAP,
+): TextChunk[] {
+  if (!text || text.length === 0) return [];
+
+  const chunks: TextChunk[] = [];
+  let offset = 0;
+  let index = 0;
+
+  while (offset < text.length) {
+    const end = Math.min(offset + chunkSize, text.length);
+    chunks.push({
+      index,
+      text: text.slice(offset, end),
+      startOffset: offset,
+      endOffset: end,
+    });
+    index++;
+    offset += chunkSize - chunkOverlap;
+    if (offset >= text.length) break;
+  }
+
+  return chunks;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Result
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface FileRAGIndexingResult {
+  ragDocumentId: string;
+  chunkCount: number;
+  tokenCount?: number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Service
+// ─────────────────────────────────────────────────────────────────────────────
+
+@injectable()
+export class FileRAGIndexingBackendService {
+  protected logDebug: (message: string, context?: Record<string, unknown>) => void;
+
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.Logger)
+    private readonly logger: LoggerService,
+    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)
+    private readonly errorBuilder: ErrorBuilderBackendService,
+    @inject(SAAS_SERVICE_TYPES.FileTextExtractionService)
+    private readonly textExtractionService: FileTextExtractionBackendService,
+    @inject(SAAS_SERVICE_TYPES.FilesBackendService)
+    private readonly filesService: FilesBackendService,
+    @inject(SAAS_SERVICE_TYPES.WeaviateVectorStoreService)
+    private readonly vectorStore: WeaviateVectorStoreBackendService,
+    @inject(SAAS_SERVICE_TYPES.AppConfigService)
+    private readonly appConfigService: AppConfigurationService,
+  ) {
+    this.logDebug = this.logger.debugByType('FileRAGIndexingBackendService');
+    this.logger.setDebugTypeLogging('FileRAGIndexingBackendService', false);
+  }
+
+  /**
+   * Determine if a file should enter the RAG pipeline based on its field metadata.
+   */
+  shouldIndex(fileMetadata?: FileSchemaMetadata): boolean {
+    return fileMetadata?.rag?.enabled === true;
+  }
+
+  /**
+   * Full RAG pipeline for a single file:
+   *
+   * 1. Transition to EXTRACTING
+   * 2. Extract text via storage provider
+   * 3. Chunk text per z_file({ rag }) config
+   * 4. (future) Embed chunks + store in vector DB
+   * 5. Transition to INDEXED
+   *
+   * Failure at any step transitions to FAILED.
+   */
+  async indexFile(
+    file: File,
+    ragConfig: FileRAGConfiguration,
+    executionContext: ExecutionContext<any>,
+  ): Promise<FileRAGIndexingResult> {
+    const { fileId } = file;
+
+    await this.filesService.updateFileStatus(fileId, FileStatus.EXTRACTING, executionContext);
+
+    try {
+      // Step 1: Extract text
+      if (!this.textExtractionService.isExtractionSupported(file.mimeType ?? '')) {
+        this.logDebug('Skipping RAG — MIME type not extractable', { fileId, mimeType: file.mimeType });
+        await this.filesService.updateFileStatus(fileId, FileStatus.INDEXED, executionContext);
+        return { ragDocumentId: fileId, chunkCount: 0 };
+      }
+
+      const extraction = await this.textExtractionService.extractText(file, executionContext);
+
+      // Step 2: Chunk
+      const chunkSize = ragConfig.chunkSize ?? DEFAULT_CHUNK_SIZE;
+      const chunkOverlap = ragConfig.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP;
+      const chunks = chunkText(extraction.text, chunkSize, chunkOverlap);
+
+      this.logDebug('Text extracted and chunked', {
+        fileId,
+        textLength: extraction.text.length,
+        chunkCount: chunks.length,
+        chunkSize,
+        chunkOverlap,
+        strategy: ragConfig.chunkingStrategy ?? 'fixed_size',
+      });
+
+      // Step 3: Store chunks in vector database
+      if (this.vectorStore.isConfigured()) {
+        const applicationId = executionContext.applicationId
+          || this.appConfigService.applicationId
+          || 'default';
+
+        await this.vectorStore.upsertChunks(
+          applicationId,
+          fileId,
+          chunks,
+          {
+            applicationId,
+            organizationId: file.organizationId,
+            userId: file.userId,
+            resourceType: file.linkedResourceType,
+            fieldName: file.linkedFieldName,
+            mimeType: file.mimeType,
+          },
+        );
+
+        this.logDebug('Chunks stored in Weaviate', { fileId, chunkCount: chunks.length });
+      } else {
+        this.logger.warn(
+          'Weaviate is not configured — text extracted and chunked but not stored in vector DB',
+          { fileId, chunkCount: chunks.length },
+        );
+      }
+
+      // Step 4: Transition to INDEXED
+      await this.filesService.updateFileStatus(fileId, FileStatus.INDEXED, executionContext);
+
+      return {
+        ragDocumentId: fileId,
+        chunkCount: chunks.length,
+        tokenCount: extraction.text.length,
+      };
+    } catch (error) {
+      this.logger.error('RAG indexing failed', {
+        fileId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+
+      try {
+        await this.filesService.updateFileStatus(fileId, FileStatus.FAILED, executionContext);
+      } catch (statusError) {
+        this.logger.error('Failed to transition file to FAILED after RAG error', {
+          fileId,
+          error: statusError instanceof Error ? statusError.message : String(statusError),
+        });
+      }
+
+      throw this.errorBuilder.buildError(
+        ErrorType.INTERNAL,
+        executionContext,
+        {
+          customMessageReference: ErrorCustomMessageReference.INTERNAL,
+          context: {
+            message: 'RAG indexing failed',
+            fileId,
+            error: error instanceof Error ? error.message : String(error),
+          },
+        },
+      );
+    }
+  }
+}

package/src/files/services/file-scanning.backend.service.ts

@@ -0,0 +1,373 @@
+/**
+ * File Scanning Backend Service
+ *
+ * Orchestrates antivirus scanning for uploaded files. After a file
+ * reaches UPLOADED status, this service:
+ *
+ * 1. Transitions the file to SCANNING
+ * 2. Downloads the file from storage
+ * 3. Submits to ClamAV via TCP (signature-based)
+ * 4. If ClamAV passes, submits to YARA via REST (pattern-based, optional)
+ * 5. Processes combined results (CLEAN / INFECTED)
+ * 6. Quarantines infected files
+ *
+ * Scanning is **optimistic**: the upload response returns immediately
+ * with status UPLOADED. The file becomes downloadable only after
+ * transitioning to CLEAN (when scanning is enabled). If scanning is
+ * disabled, the file stays at UPLOADED and remains downloadable.
+ *
+ * Configuration:
+ * - Global: `appConfig.files.scanning.enabled` (default: false)
+ * - Per-field: `z_file({ scanPolicy: 'required' | 'skip' })` overrides global
+ */
+
+import { injectable, inject } from 'inversify';
+import {
+  ErrorType,
+  ErrorCustomMessageReference,
+  FileStatus,
+  FILE_STATUS_TRANSITIONS,
+} from '@wildo-ai/saas-models';
+import type { FileSchemaMetadata } from '@wildo-ai/zod-decorators';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import type { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';
+import { ExecutionContext } from '~wildo-backend-lib/runtime/models/execution-context.backend.schemas';
+import { FilesBackendService } from '~wildo-backend-lib/files/services/files.backend.service';
+import { AppConfigurationService } from '~wildo-backend-lib/config/services/app-configuration.backend.service';
+import { ClamAVScannerBackendService } from '~wildo-backend-lib/files/scanning/clamav-scanner.backend.service';
+import {
+  FileScanVerdict,
+  type FileScanResult,
+} from '~wildo-backend-lib/files/scanning/file-scan.types';
+import { YaraScannerBackendService } from '~wildo-backend-lib/files/scanning/yara-scanner.backend.service';
+import { FileStorageRouterBackendService } from '~wildo-backend-lib/files/storage/file-storage-router.backend.service';
+export {
+  FileScanVerdict,
+  type FileScanResult,
+} from '~wildo-backend-lib/files/scanning/file-scan.types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Service
+// ─────────────────────────────────────────────────────────────────────────────
+
+@injectable()
+export class FileScanningBackendService {
+  protected logDebug: (message: string, context?: Record<string, unknown>) => void;
+
+  private readonly onCleanCallbacks = new Map<string, () => Promise<void>>();
+
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.Logger)
+    private readonly logger: LoggerService,
+    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)
+    private readonly errorBuilder: ErrorBuilderBackendService,
+    @inject(SAAS_SERVICE_TYPES.FilesBackendService)
+    private readonly filesService: FilesBackendService,
+    @inject(SAAS_SERVICE_TYPES.AppConfigService)
+    private readonly appConfigService: AppConfigurationService,
+    @inject(SAAS_SERVICE_TYPES.ClamAVScannerService)
+    private readonly clamavScanner: ClamAVScannerBackendService,
+    @inject(SAAS_SERVICE_TYPES.YaraScannerService)
+    private readonly yaraScanner: YaraScannerBackendService,
+    @inject(SAAS_SERVICE_TYPES.FileStorageRouter)
+    private readonly fileStorageRouter: FileStorageRouterBackendService,
+  ) {
+    this.logDebug = this.logger.debugByType('FileScanningBackendService');
+  }
+
+  /**
+   * Determine whether a file should be scanned after upload.
+   *
+   * Resolution order:
+   * 1. Per-field `z_file({ scanPolicy })` wins if set
+   * 2. Falls back to global `appConfig.files.scanning.enabled`
+   */
+  shouldScan(fileMetadata?: FileSchemaMetadata): boolean {
+    const scanPolicy = fileMetadata?.scanPolicy;
+
+    if (scanPolicy === 'required') return true;
+    if (scanPolicy === 'skip') return false;
+
+    return this.isGlobalScanningEnabled();
+  }
+
+  /**
+   * Check if global scanning is enabled in the app configuration.
+   */
+  isGlobalScanningEnabled(): boolean {
+    if (!this.appConfigService.isInitialized()) return false;
+    const filesConfig = (this.appConfigService.config as Record<string, any>)?.files;
+    return filesConfig?.scanning?.enabled === true;
+  }
+
+  /**
+   * Initiate scanning for a file that just reached UPLOADED status.
+   *
+   * Transitions to SCANNING, then dispatches the scan job.
+   * The response is fire-and-forget: the caller does not wait for scan
+   * completion — scan results are processed asynchronously via
+   * `handleScanResult()`.
+   *
+   * @param onClean - Optional callback invoked when scan verdict is CLEAN.
+   *                  Used to chain post-scan pipelines (e.g., RAG indexing).
+   */
+  async initiateScan(
+    fileId: string,
+    executionContext: ExecutionContext<any>,
+    onClean?: () => Promise<void>,
+  ): Promise<void> {
+    const allowedTransitions = FILE_STATUS_TRANSITIONS[FileStatus.UPLOADED];
+    if (!allowedTransitions.includes(FileStatus.SCANNING)) {
+      throw this.errorBuilder.buildError(
+        ErrorType.VALIDATION,
+        executionContext,
+        {
+          customMessageReference: ErrorCustomMessageReference.VALIDATION,
+          context: {
+            message: `Cannot transition file to SCANNING from current status`,
+            fileId,
+          },
+        },
+      );
+    }
+
+    await this.filesService.updateFileStatus(
+      fileId,
+      FileStatus.SCANNING,
+      executionContext,
+    );
+
+    if (onClean) {
+      this.onCleanCallbacks.set(fileId, onClean);
+    }
+
+    this.logDebug('File transitioned to SCANNING, dispatching scan', { fileId });
+
+    // Run the full scanning pipeline asynchronously. The file is already in
+    // SCANNING status so downloads are blocked until the result handler
+    // transitions it to CLEAN / INFECTED / FAILED.
+    this.executeScan(fileId, executionContext).catch((err) => {
+      this.logger.error('Scan execution failed unexpectedly', {
+        fileId,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    });
+  }
+
+  /**
+   * Download the file from storage and run the scanning pipeline.
+   *
+   * Pipeline order:
+   * 1. ClamAV (signature-based) — if configured
+   * 2. YARA (pattern-based) — if enabled and ClamAV passed
+   *
+   * Short-circuits on first INFECTED verdict. YARA only runs if the
+   * file survived ClamAV — no point in pattern-matching a known virus.
+   */
+  private async executeScan(
+    fileId: string,
+    executionContext: ExecutionContext<any>,
+  ): Promise<void> {
+    const file = await this.filesService.getFile(fileId, executionContext);
+    if (!file) {
+      this.logger.error('File not found for scanning', { fileId });
+      return;
+    }
+
+    const provider = this.fileStorageRouter.getProviderById(file.storageProvider);
+    if (!provider) {
+      await this.handleScanResult(fileId, {
+        verdict: FileScanVerdict.ERROR,
+        errorMessage: `Storage provider '${file.storageProvider}' not available for scan download`,
+        scannerRef: 'scan-pipeline',
+      }, executionContext);
+      return;
+    }
+
+    const downloadParams = {
+      fileId: file.fileId,
+      storageProvider: file.storageProvider,
+      storageRef: file.storageRef,
+    };
+    const executedScannerLayers: string[] = [];
+
+    // ── Layer 1: ClamAV ────────────────────────────────────────────────
+    if (this.clamavScanner.isConfigured()) {
+      executedScannerLayers.push('clamav');
+      let downloadResult;
+      try {
+        downloadResult = await provider.download(downloadParams, executionContext);
+      } catch (err) {
+        await this.handleScanResult(fileId, {
+          verdict: FileScanVerdict.ERROR,
+          errorMessage: `Failed to download file for ClamAV: ${err instanceof Error ? err.message : String(err)}`,
+          scannerRef: 'clamav',
+        }, executionContext);
+        return;
+      }
+
+      try {
+        const clamResult = await this.clamavScanner.scanStream(downloadResult.stream, fileId);
+
+        if (clamResult.verdict !== FileScanVerdict.CLEAN) {
+          await this.handleScanResult(fileId, clamResult, executionContext);
+          return;
+        }
+
+        this.logDebug('ClamAV layer passed', { fileId, durationMs: clamResult.durationMs });
+      } finally {
+        if (!downloadResult.stream.destroyed) {
+          downloadResult.stream.destroy();
+        }
+      }
+    } else {
+      this.logDebug('ClamAV not configured — skipping signature scan', { fileId });
+    }
+
+    // ── Layer 2: YARA (optional, platform-level) ───────────────────────
+    if (this.yaraScanner.isEnabled()) {
+      executedScannerLayers.push('yara');
+      let downloadResult;
+      try {
+        downloadResult = await provider.download(downloadParams, executionContext);
+      } catch (err) {
+        await this.handleScanResult(fileId, {
+          verdict: FileScanVerdict.ERROR,
+          errorMessage: `Failed to download file for YARA: ${err instanceof Error ? err.message : String(err)}`,
+          scannerRef: 'yara',
+        }, executionContext);
+        return;
+      }
+
+      try {
+        const yaraResult = await this.yaraScanner.scanStream(downloadResult.stream, fileId);
+
+        if (yaraResult.verdict !== FileScanVerdict.CLEAN) {
+          await this.handleScanResult(fileId, yaraResult, executionContext);
+          return;
+        }
+
+        this.logDebug('YARA layer passed', { fileId, durationMs: yaraResult.durationMs });
+      } finally {
+        if (!downloadResult.stream.destroyed) {
+          downloadResult.stream.destroy();
+        }
+      }
+    }
+
+    // ── All layers passed ──────────────────────────────────────────────
+    if (!this.clamavScanner.isConfigured() && !this.yaraScanner.isEnabled()) {
+      this.logger.warn('No scanners configured — marking as CLEAN', { fileId });
+    }
+
+    await this.handleScanResult(fileId, {
+      verdict: FileScanVerdict.CLEAN,
+      scannerRef: executedScannerLayers.join('+') || 'none',
+    }, executionContext);
+  }
+
+  /**
+   * Process the result of a scan job.
+   *
+   * - CLEAN: transition to CLEAN (file becomes downloadable)
+   * - INFECTED: transition to INFECTED, then quarantine
+   * - ERROR: transition to FAILED (retryable via re-scan)
+   */
+  async handleScanResult(
+    fileId: string,
+    result: FileScanResult,
+    executionContext: ExecutionContext<any>,
+  ): Promise<void> {
+    this.logDebug('Processing scan result', {
+      fileId,
+      verdict: result.verdict,
+      threatName: result.threatName,
+      scannerRef: result.scannerRef,
+      durationMs: result.durationMs,
+    });
+
+    switch (result.verdict) {
+      case FileScanVerdict.CLEAN:
+        await this.filesService.updateFileStatus(
+          fileId,
+          FileStatus.CLEAN,
+          executionContext,
+        );
+        this.logDebug('File marked as CLEAN after scan', { fileId });
+
+        // Invoke post-clean callback (e.g., RAG indexing pipeline)
+        const onClean = this.onCleanCallbacks.get(fileId);
+        if (onClean) {
+          this.onCleanCallbacks.delete(fileId);
+          onClean().catch((err) => {
+            this.logger.error('Post-clean callback failed', {
+              fileId,
+              error: err instanceof Error ? err.message : String(err),
+            });
+          });
+        }
+        break;
+
+      case FileScanVerdict.INFECTED:
+        this.onCleanCallbacks.delete(fileId);
+        await this.filesService.updateFileStatus(
+          fileId,
+          FileStatus.INFECTED,
+          executionContext,
+        );
+        this.logger.warn('File flagged as INFECTED', {
+          fileId,
+          threatName: result.threatName,
+          scannerRef: result.scannerRef,
+        });
+
+        await this.quarantineInfectedFile(fileId, result, executionContext);
+        break;
+
+      case FileScanVerdict.ERROR:
+        this.onCleanCallbacks.delete(fileId);
+        this.logger.error('Scan failed with error', {
+          fileId,
+          errorMessage: result.errorMessage,
+          scannerRef: result.scannerRef,
+        });
+        await this.filesService.updateFileStatus(
+          fileId,
+          FileStatus.FAILED,
+          executionContext,
+        );
+        break;
+    }
+  }
+
+  /**
+   * Quarantine an infected file.
+   *
+   * Transitions INFECTED → QUARANTINED. The blob remains in storage
+   * but the file is permanently blocked from download. Operators can
+   * delete quarantined files via admin tooling.
+   */
+  private async quarantineInfectedFile(
+    fileId: string,
+    result: FileScanResult,
+    executionContext: ExecutionContext<any>,
+  ): Promise<void> {
+    try {
+      await this.filesService.updateFileStatus(
+        fileId,
+        FileStatus.QUARANTINED,
+        executionContext,
+      );
+      this.logDebug('Infected file quarantined', {
+        fileId,
+        threatName: result.threatName,
+      });
+    } catch (error) {
+      this.logger.error('Failed to quarantine infected file', {
+        fileId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+}