@wildo-ai/saas-backend-lib 1.0.1

package/src/files/rag/weaviate-vector-store.backend.service.ts

@@ -0,0 +1,271 @@
+/**
+ * Weaviate Vector Store Backend Service
+ *
+ * Adapter for the Weaviate vector database. Provides:
+ * - Collection management (per-application class)
+ * - Document upsert (chunked text with metadata)
+ * - Similarity search
+ * - Document deletion
+ *
+ * Configured via env vars:
+ * - WEAVIATE_HOST (default: localhost)
+ * - WEAVIATE_PORT (default: 8080)
+ * - WEAVIATE_GRPC_PORT (default: 50051)
+ * - WEAVIATE_SCHEME (default: http)
+ * - WEAVIATE_API_KEY (optional, for authenticated clusters)
+ *
+ * The service is lazy-initialized: the Weaviate client is created on first
+ * use, not at DI construction time. This allows the backend to start even
+ * when Weaviate is not configured (no z_file({ rag }) declared).
+ */
+
+import { injectable, inject } from 'inversify';
+import { createHash } from 'crypto';
+import {
+  ErrorType,
+  ErrorCustomMessageReference,
+} from '@wildo-ai/saas-models';
+import { ApplicationBackendDataEnvKeys } from '@wildo-ai/platform-core-shared';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import type { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import { ErrorBuilderBackendService } from '~wildo-backend-lib/errors/services/error-builder.backend.service';
+import type { TextChunk } from '~wildo-backend-lib/files/services/file-rag-indexing.backend.service';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Vector document contract
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface VectorDocumentMetadata {
+  fileId: string;
+  applicationId?: string;
+  organizationId?: string;
+  userId?: string;
+  resourceType?: string;
+  fieldName?: string;
+  mimeType?: string;
+  chunkIndex: number;
+  totalChunks: number;
+}
+
+export interface VectorSearchResult {
+  fileId: string;
+  chunkIndex: number;
+  text: string;
+  score: number;
+  metadata: VectorDocumentMetadata;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Service
+// ─────────────────────────────────────────────────────────────────────────────
+
+@injectable()
+export class WeaviateVectorStoreBackendService {
+  protected logDebug: (message: string, context?: Record<string, unknown>) => void;
+
+  private client: any | null = null;
+  private initialized = false;
+
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.Logger)
+    private readonly logger: LoggerService,
+    @inject(SAAS_SERVICE_TYPES.ErrorBuilderService)
+    private readonly errorBuilder: ErrorBuilderBackendService,
+  ) {
+    this.logDebug = this.logger.debugByType('WeaviateVectorStoreBackendService');
+  }
+
+  /**
+   * Whether Weaviate connection env vars are present.
+   */
+  isConfigured(): boolean {
+    return !!process.env[ApplicationBackendDataEnvKeys.WEAVIATE_HOST];
+  }
+
+  /**
+   * Lazily initialize the Weaviate client.
+   */
+  private async ensureClient(): Promise<any> {
+    if (this.client) return this.client;
+
+    const host = process.env[ApplicationBackendDataEnvKeys.WEAVIATE_HOST];
+    if (!host) {
+      throw this.errorBuilder.buildError(
+        ErrorType.CONFIGURATION,
+        undefined,
+        {
+          customMessageReference: ErrorCustomMessageReference.CONFIGURATION,
+          context: { message: 'WEAVIATE_HOST is not configured' },
+        },
+      );
+    }
+
+    const port = Number(process.env[ApplicationBackendDataEnvKeys.WEAVIATE_PORT] ?? '8080');
+    const grpcPort = Number(process.env[ApplicationBackendDataEnvKeys.WEAVIATE_GRPC_PORT] ?? '50051');
+    const scheme = process.env[ApplicationBackendDataEnvKeys.WEAVIATE_SCHEME] ?? 'http';
+    const apiKey = process.env[ApplicationBackendDataEnvKeys.WEAVIATE_API_KEY];
+
+    let weaviate: any;
+    try {
+      weaviate = await import('weaviate-client' as string);
+    } catch {
+      throw new Error(
+        'weaviate-client is required for RAG vector storage. Install it with: pnpm add weaviate-client',
+      );
+    }
+
+    const connectionOptions: Record<string, unknown> = {
+      httpHost: host,
+      httpPort: port,
+      httpSecure: scheme === 'https',
+      grpcHost: host,
+      grpcPort,
+      grpcSecure: scheme === 'https',
+    };
+
+    if (apiKey) {
+      connectionOptions.authCredentials = new weaviate.ApiKey(apiKey);
+    }
+
+    this.client = await weaviate.default.connectToCustom(connectionOptions);
+
+    this.initialized = true;
+    this.logDebug('Weaviate client initialized', { host, port, grpcPort, scheme });
+    return this.client;
+  }
+
+  /**
+   * Build the Weaviate class name for an application.
+   * Each application gets its own class to isolate data.
+   */
+  private buildClassName(applicationId: string): string {
+    const sanitized = applicationId.replace(/[^a-zA-Z0-9]/g, '_');
+    const normalized = sanitized.replace(/_+/g, '_').replace(/^_+|_+$/g, '') || 'app';
+    const hashedSuffix = createHash('sha1')
+      .update(applicationId)
+      .digest('hex')
+      .slice(0, 8);
+    return `WildoFile_${normalized}_${hashedSuffix}`;
+  }
+
+  /**
+   * Ensure the Weaviate class exists for the application.
+   */
+  async ensureCollection(applicationId: string): Promise<void> {
+    const client = await this.ensureClient();
+    const className = this.buildClassName(applicationId);
+
+    try {
+      const exists = await client.collections.exists(className);
+      if (exists) return;
+
+      await client.collections.create({
+        name: className,
+        properties: [
+          { name: 'text', dataType: 'text' },
+          { name: 'fileId', dataType: 'text' },
+          { name: 'chunkIndex', dataType: 'int' },
+          { name: 'totalChunks', dataType: 'int' },
+          { name: 'organizationId', dataType: 'text' },
+          { name: 'userId', dataType: 'text' },
+          { name: 'resourceType', dataType: 'text' },
+          { name: 'fieldName', dataType: 'text' },
+          { name: 'mimeType', dataType: 'text' },
+        ],
+      });
+
+      this.logDebug('Weaviate collection created', { className, applicationId });
+    } catch (error) {
+      this.logger.error('Failed to ensure Weaviate collection', {
+        className,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  /**
+   * Store chunked text with metadata in Weaviate.
+   *
+   * Each chunk becomes a separate Weaviate object in the application's class.
+   * The `fileId` + `chunkIndex` pair is the logical primary key.
+   */
+  async upsertChunks(
+    applicationId: string,
+    fileId: string,
+    chunks: TextChunk[],
+    metadata: Omit<VectorDocumentMetadata, 'fileId' | 'chunkIndex' | 'totalChunks'>,
+  ): Promise<{ objectCount: number }> {
+    const client = await this.ensureClient();
+    const className = this.buildClassName(applicationId);
+
+    await this.ensureCollection(applicationId);
+
+    // Delete existing chunks for this fileId (idempotent re-index)
+    await this.deleteByFileId(applicationId, fileId);
+
+    const collection = client.collections.get(className);
+    const objects = chunks.map((chunk) => ({
+      properties: {
+        text: chunk.text,
+        fileId,
+        chunkIndex: chunk.index,
+        totalChunks: chunks.length,
+        organizationId: metadata.organizationId ?? '',
+        userId: metadata.userId ?? '',
+        resourceType: metadata.resourceType ?? '',
+        fieldName: metadata.fieldName ?? '',
+        mimeType: metadata.mimeType ?? '',
+      },
+    }));
+
+    if (objects.length > 0) {
+      await collection.data.insertMany(objects);
+    }
+
+    this.logDebug('Chunks stored in Weaviate', {
+      applicationId,
+      fileId,
+      chunkCount: objects.length,
+    });
+
+    return { objectCount: objects.length };
+  }
+
+  /**
+   * Delete all chunks for a file.
+   */
+  async deleteByFileId(applicationId: string, fileId: string): Promise<void> {
+    const client = await this.ensureClient();
+    const className = this.buildClassName(applicationId);
+
+    try {
+      const collection = client.collections.get(className);
+      await collection.data.deleteMany(
+        collection.filter.byProperty('fileId').equal(fileId),
+      );
+      this.logDebug('Deleted Weaviate chunks for file', { applicationId, fileId });
+    } catch (error) {
+      this.logDebug('No existing chunks to delete (or collection missing)', {
+        applicationId,
+        fileId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
+  /**
+   * Close the Weaviate client connection.
+   */
+  async close(): Promise<void> {
+    if (this.client) {
+      try {
+        await this.client.close();
+      } catch {
+        // Ignore close errors
+      }
+      this.client = null;
+      this.initialized = false;
+    }
+  }
+}

package/src/files/scanning/clamav-scanner.backend.service.ts

@@ -0,0 +1,214 @@
+/**
+ * ClamAV Scanner Backend Service
+ *
+ * Adapter for ClamAV antivirus daemon (clamd). Connects via TCP to the
+ * `clamd` socket and submits files for scanning using the INSTREAM protocol.
+ *
+ * Configuration:
+ * - CLAMAV_HOST (default: localhost)
+ * - CLAMAV_PORT (default: 3310)
+ *
+ * The service is lazy-initialized: the connection is established on first
+ * scan, not at DI construction time.
+ *
+ * Protocol:
+ * - TCP connection to clamd on CLAMAV_HOST:CLAMAV_PORT
+ * - Send "zINSTREAM\0" command
+ * - Stream file data in length-prefixed chunks (4-byte big-endian length + data)
+ * - Send zero-length chunk to signal end of stream
+ * - Read response: "stream: OK\0" or "stream: <threat> FOUND\0"
+ */
+
+import { injectable, inject } from 'inversify';
+import { Readable } from 'stream';
+import * as net from 'net';
+import { ApplicationBackendDataEnvKeys } from '@wildo-ai/platform-core-shared';
+import { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';
+import type { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';
+import {
+  FileScanVerdict,
+  type FileScanResult,
+} from '~wildo-backend-lib/files/scanning/file-scan.types';
+
+const CLAMAV_DEFAULT_PORT = 3310;
+const CLAMAV_TIMEOUT_MS = 60_000;
+const CHUNK_SIZE = 8192;
+
+@injectable()
+export class ClamAVScannerBackendService {
+  protected logDebug: (message: string, context?: Record<string, unknown>) => void;
+
+  constructor(
+    @inject(SAAS_SERVICE_TYPES.Logger)
+    private readonly logger: LoggerService,
+  ) {
+    this.logDebug = this.logger.debugByType('ClamAVScannerBackendService');
+  }
+
+  /**
+   * Whether ClamAV connection env vars are present.
+   */
+  isConfigured(): boolean {
+    return !!process.env[ApplicationBackendDataEnvKeys.CLAMAV_HOST];
+  }
+
+  private getConnectionParams(): { host: string; port: number } {
+    const host = process.env[ApplicationBackendDataEnvKeys.CLAMAV_HOST] ?? 'localhost';
+    const port = Number(process.env[ApplicationBackendDataEnvKeys.CLAMAV_PORT] ?? CLAMAV_DEFAULT_PORT);
+    return { host, port };
+  }
+
+  /**
+   * Ping the ClamAV daemon to check connectivity.
+   */
+  async ping(): Promise<boolean> {
+    const { host, port } = this.getConnectionParams();
+
+    return new Promise<boolean>((resolve) => {
+      const socket = net.createConnection({ host, port, timeout: 5000 }, () => {
+        socket.write('zPING\0');
+      });
+
+      let data = '';
+      socket.on('data', (chunk) => { data += chunk.toString(); });
+      socket.on('end', () => {
+        socket.destroy();
+        resolve(data.trim() === 'PONG');
+      });
+      socket.on('error', () => {
+        socket.destroy();
+        resolve(false);
+      });
+      socket.on('timeout', () => {
+        socket.destroy();
+        resolve(false);
+      });
+    });
+  }
+
+  /**
+   * Scan a file stream via the ClamAV INSTREAM protocol.
+   *
+   * @param fileStream - Readable stream of file bytes
+   * @param fileId - File identifier for logging
+   * @returns Scan verdict (CLEAN, INFECTED, or ERROR)
+   */
+  async scanStream(fileStream: Readable, fileId: string): Promise<FileScanResult> {
+    const { host, port } = this.getConnectionParams();
+    const startTime = Date.now();
+
+    this.logDebug('Starting ClamAV scan', { fileId, host, port });
+
+    return new Promise<FileScanResult>((resolve) => {
+      const socket = net.createConnection({ host, port, timeout: CLAMAV_TIMEOUT_MS }, () => {
+        // Send INSTREAM command
+        socket.write('zINSTREAM\0');
+
+        // Stream file data in length-prefixed chunks
+        fileStream.on('data', (chunk: Buffer) => {
+          const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+          let offset = 0;
+
+          while (offset < buf.length) {
+            const end = Math.min(offset + CHUNK_SIZE, buf.length);
+            const slice = buf.subarray(offset, end);
+            const lengthBuf = Buffer.alloc(4);
+            lengthBuf.writeUInt32BE(slice.length, 0);
+            socket.write(lengthBuf);
+            socket.write(slice);
+            offset = end;
+          }
+        });
+
+        fileStream.on('end', () => {
+          // Send zero-length chunk to signal end of stream
+          const endBuf = Buffer.alloc(4);
+          endBuf.writeUInt32BE(0, 0);
+          socket.write(endBuf);
+        });
+
+        fileStream.on('error', (err) => {
+          socket.destroy();
+          resolve({
+            verdict: FileScanVerdict.ERROR,
+            errorMessage: `File stream error: ${err.message}`,
+            scannerRef: 'clamav',
+            durationMs: Date.now() - startTime,
+          });
+        });
+      });
+
+      let responseData = '';
+      socket.on('data', (chunk) => { responseData += chunk.toString(); });
+
+      socket.on('end', () => {
+        socket.destroy();
+        const durationMs = Date.now() - startTime;
+        const result = this.parseResponse(responseData, fileId, durationMs);
+        this.logDebug('ClamAV scan completed', { fileId, verdict: result.verdict, durationMs });
+        resolve(result);
+      });
+
+      socket.on('error', (err) => {
+        socket.destroy();
+        this.logger.error('ClamAV connection error', { fileId, error: err.message });
+        resolve({
+          verdict: FileScanVerdict.ERROR,
+          errorMessage: `ClamAV connection error: ${err.message}`,
+          scannerRef: 'clamav',
+          durationMs: Date.now() - startTime,
+        });
+      });
+
+      socket.on('timeout', () => {
+        socket.destroy();
+        resolve({
+          verdict: FileScanVerdict.ERROR,
+          errorMessage: 'ClamAV scan timed out',
+          scannerRef: 'clamav',
+          durationMs: Date.now() - startTime,
+        });
+      });
+    });
+  }
+
+  /**
+   * Parse the ClamAV INSTREAM response.
+   *
+   * Expected formats:
+   * - "stream: OK\0" → CLEAN
+   * - "stream: <threatName> FOUND\0" → INFECTED
+   * - "stream: <error> ERROR\0" → ERROR
+   */
+  private parseResponse(raw: string, fileId: string, durationMs: number): FileScanResult {
+    const cleaned = raw.replace(/\0/g, '').trim();
+
+    if (cleaned.endsWith('OK')) {
+      return {
+        verdict: FileScanVerdict.CLEAN,
+        scannerRef: 'clamav',
+        durationMs,
+      };
+    }
+
+    if (cleaned.endsWith('FOUND')) {
+      // Format: "stream: Win.Trojan.Agent-123 FOUND"
+      const match = cleaned.match(/^stream:\s*(.+?)\s+FOUND$/);
+      const threatName = match?.[1] ?? 'Unknown threat';
+      return {
+        verdict: FileScanVerdict.INFECTED,
+        threatName,
+        scannerRef: 'clamav',
+        durationMs,
+      };
+    }
+
+    // Unexpected response — treat as error
+    return {
+      verdict: FileScanVerdict.ERROR,
+      errorMessage: `Unexpected ClamAV response: ${cleaned}`,
+      scannerRef: 'clamav',
+      durationMs,
+    };
+  }
+}

package/src/files/scanning/file-scan.types.ts

@@ -0,0 +1,17 @@
+export enum FileScanVerdict {
+  CLEAN = 'CLEAN',
+  INFECTED = 'INFECTED',
+  ERROR = 'ERROR',
+}
+
+export interface FileScanResult {
+  verdict: FileScanVerdict;
+  /** Engine/signature that matched (e.g., "Win.Trojan.Agent-123") */
+  threatName?: string;
+  /** Scanner identifier (e.g., "clamav", "virustotal") */
+  scannerRef?: string;
+  /** Scan duration in milliseconds */
+  durationMs?: number;
+  /** Error message when verdict is ERROR */
+  errorMessage?: string;
+}