http-request-manager 18.15.32 → 18.15.34

package/src/lib/services/request-manager-services/rxjs-operators/request-streaming.ts

@@ -0,0 +1,468 @@
+import { OperatorFunction } from 'rxjs';
+import { filter, map } from 'rxjs/operators';
+import { HttpEventType } from '@angular/common/http';
+import { StreamType } from '../../request-manager-state-service/models/stream-type.enum';
+import { StreamConfigModel } from '../../request-manager-state-service/models/stream-config.model';
+import { StreamProgressModel } from '../../request-manager-state-service/models/stream-progress.model';
+import { StreamOutputModel } from '../../request-manager-state-service/models/stream-output.model';
+
+/**
+ * Stateful processor for managing streaming data and events
+ */
+class StreamingProcessor<T = any> {
+  private buffer = '';
+  private contentType = '';
+  private readonly maxBufferSize = 10 * 1024 * 1024; // 10MB limit
+  private streamConfig: StreamConfigModel;
+  private lastParsedLength = 0; // Track parsed position to avoid duplicates
+  private totalFromHeader: number | undefined;
+
+  constructor(config?: StreamConfigModel) {
+    this.streamConfig = config || new StreamConfigModel(StreamType.AI_STREAMING);
+  }
+
+  /**
+   * Process HTTP events and extract streaming data with progress
+   */
+  process(event: any): StreamOutputModel<T> | null {
+    switch (event.type) {
+      case HttpEventType.ResponseHeader:
+        this.contentType = event.headers?.get('content-type') || '';
+        // Read total from response header if configured
+        if (this.streamConfig.totalHeader && event.headers) {
+          const totalVal = event.headers.get(this.streamConfig.totalHeader);
+          if (totalVal !== undefined && totalVal !== null) {
+            const parsed = parseInt(totalVal, 10);
+            if (!isNaN(parsed)) {
+              this.totalFromHeader = parsed;
+            }
+          }
+        }
+        return null;
+
+      case HttpEventType.DownloadProgress:
+        if (event.partialText) {
+          this.buffer = event.partialText;
+          const parsedData = this.parseBuffer();
+
+          // Only return NEW items since last parse (progressive updates)
+          const newItems = parsedData.slice(this.lastParsedLength);
+          this.lastParsedLength = parsedData.length;
+
+          // Calculate progress — prefer record count from header, fall back to byte-level
+          const total = this.totalFromHeader ?? event.total;
+          const received = this.totalFromHeader ? this.lastParsedLength : event.loaded;
+          const percent = total
+            ? Math.round((received / total) * 100)
+            : 0;
+
+          const progress = StreamProgressModel.adapt({
+            received,
+            total,
+            percent,
+            stage: 'streaming'
+          });
+
+          return StreamOutputModel.adapt<T>({
+            data: newItems.length > 0 ? newItems : [],
+            progress
+          });
+        }
+        return null;
+
+      case HttpEventType.Response:
+        if (event.body) {
+          this.buffer = typeof event.body === 'string' ? event.body : this.buffer;
+          const parsedData = this.parseBuffer();
+
+          // Calculate final progress
+          const received = this.totalFromHeader ? parsedData.length : event.loaded;
+
+          const progress = StreamProgressModel.adapt({
+            received,
+            percent: 100,
+            stage: 'complete'
+          });
+
+          return StreamOutputModel.adapt<T>({
+            data: parsedData.length > 0 ? parsedData : [],
+            progress
+          });
+        }
+        return null;
+
+      default:
+        return null;
+    }
+  }
+
+  /**
+   * Append data to buffer with size management
+   */
+  private appendToBuffer(data: string): void {
+    this.buffer += data;
+
+    // Implement sliding window if buffer gets too large
+    if (this.buffer.length > this.maxBufferSize) {
+      this.buffer = this.buffer.slice(-this.maxBufferSize / 2);
+    }
+  }
+
+  /**
+   * Parse current buffer content
+   */
+  private parseBuffer(): T[] {
+    if (!this.buffer.trim()) {
+      return [];
+    }
+
+    try {
+      const result = parseStreamData(this.buffer, this.contentType, this.streamConfig);
+      return result as T[];
+    } catch (error) {
+      console.warn('Failed to parse streaming data:', error);
+      return [];
+    }
+  }
+
+  /**
+   * Reset processor state
+   */
+  reset(): void {
+    this.buffer = '';
+    this.contentType = '';
+    this.lastParsedLength = 0;
+    this.totalFromHeader = undefined;
+  }
+
+  /**
+   * Update configuration
+   */
+  updateConfig(config: StreamConfigModel): void {
+    this.streamConfig = config;
+  }
+}
+
+/**
+ * Safe JSON parsing with error handling
+ */
+function safeJsonParse<T>(jsonStr: string, fallback?: T): T | null {
+  try {
+    return JSON.parse(jsonStr);
+  } catch {
+    return fallback || null;
+  }
+}
+
+/**
+ * Safe JSON array parsing
+ */
+function safeJsonArrayParse<T>(jsonStr: string, fallback: T[] = []): T[] {
+  try {
+    const parsed = JSON.parse(jsonStr);
+    return Array.isArray(parsed) ? parsed : fallback;
+  } catch {
+    return fallback;
+  }
+}
+
+/**
+ * Enhanced JSON object extraction with better error handling
+ */
+function extractJsonObjects(text: string): string[] {
+  const objects: string[] = [];
+  let braceCount = 0;
+  let currentObject = '';
+  let inString = false;
+  let escaped = false;
+
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+
+    if (escaped) {
+      currentObject += char;
+      escaped = false;
+      continue;
+    }
+
+    if (char === '\\') {
+      escaped = true;
+      currentObject += char;
+      continue;
+    }
+
+    if (char === '"' && !escaped) {
+      inString = !inString;
+    }
+
+    if (!inString) {
+      if (char === '{') braceCount++;
+      else if (char === '}') braceCount--;
+    }
+
+    currentObject += char;
+
+    if (braceCount === 0 && currentObject.trim()) {
+      objects.push(currentObject.trim());
+      currentObject = '';
+    }
+  }
+
+  return objects;
+}
+
+/**
+ * COMPREHENSIVE REQUEST STREAMING OPERATOR - FULLY ABSTRACTED
+ * Refactored for better type safety and memory management
+ *
+ * Single function that handles ALL streaming formats without hardcoded assumptions:
+ * - JSON format (Individual JSON objects)
+ * - NDJSON format (Newline delimited JSON)
+ * - AI streaming format (Real-time AI responses)
+ * - Server-Sent Events format
+ * - Auto-detection mode
+ *
+ * Usage:
+ *   this.http.get(url, options).pipe(requestStreaming()).subscribe(data => {...})
+ *   this.http.get(url, options).pipe(requestStreaming({ streamType: StreamType.NDJSON })).subscribe(data => {...})
+ */
+export function requestStreaming<T = any>(config?: StreamConfigModel): OperatorFunction<any, StreamOutputModel<T>> {
+  const processor = new StreamingProcessor<T>(config);
+
+  return input$ => input$.pipe(
+    // Filter only streaming-related events
+    filter((event: any) =>
+      event.type === HttpEventType.DownloadProgress ||
+      event.type === HttpEventType.Response ||
+      event.type === HttpEventType.ResponseHeader
+    ),
+
+    // Process events using the streaming processor
+    map((event: any) => {
+      const result = processor.process(event);
+      return result;
+    }),
+
+    // Filter out null results
+    filter((result): result is StreamOutputModel<T> => result !== null)
+  );
+}
+
+/**
+ * Main streaming data parser - handles ALL formats based on streamType
+ */
+function parseStreamData(buffer: string, contentType: string, config: StreamConfigModel): any[] {
+  const results: any[] = [];
+
+  if (!buffer.trim()) {
+    return results;
+  }
+
+  // Determine parsing format based on streamType and content type
+  const format = config.streamType === StreamType.AI_STREAMING ? detectFormat(buffer, contentType) : config.streamType;
+
+  switch (format) {
+    case StreamType.NDJSON:
+      return parseNdjson(buffer, config);
+
+    case StreamType.JSON:
+      return parseJsonObjects(buffer, config);
+
+    case StreamType.AI_STREAMING:
+      return parseAiStreaming(buffer, config);
+
+    case StreamType.EVENT_STREAM:
+      return parseEventStream(buffer, config);
+
+    default:
+      return autoParse(buffer, contentType, config);
+  }
+}
+
+/**
+ * Parse NDJSON format (JSON objects per line)
+ */
+function parseNdjson(buffer: string, config: StreamConfigModel): any[] {
+  const results: any[] = [];
+  const lines = buffer.split('\n');
+
+  // If the buffer doesn't end with \n, the last segment is an incomplete partial line — skip it
+  const completeLines = buffer.endsWith('\n') ? lines : lines.slice(0, -1);
+
+  completeLines.filter(line => line.trim()).forEach(line => {
+    const trimmedLine = line.trim();
+    if (trimmedLine && trimmedLine.startsWith('{')) {
+      const parsed = safeJsonParse(trimmedLine);
+      if (parsed) results.push(parsed);
+    }
+  });
+
+  return results;
+}
+
+/**
+ * Parse individual JSON objects
+ */
+function parseJsonObjects(buffer: string, config: StreamConfigModel): any[] {
+  const results: any[] = [];
+  const jsonObjects = extractJsonObjects(buffer);
+
+  jsonObjects.forEach(jsonStr => {
+    const parsed = safeJsonParse(jsonStr);
+    if (parsed) results.push(parsed);
+  });
+
+  return results;
+}
+
+/**
+ * Parse AI streaming format
+ */
+function parseAiStreaming(buffer: string, config: StreamConfigModel): any[] {
+  const results: any[] = [];
+  const jsonObjects = extractJsonObjects(buffer);
+
+  jsonObjects.forEach(jsonStr => {
+    const parsed = safeJsonParse(jsonStr);
+    if (parsed && typeof parsed === 'object' && parsed !== null && 'type' in parsed) {
+      const typedParsed = parsed as any;
+      if (['ai_thinking', 'ai_response', 'ai_suggestion', 'ai_status'].includes(typedParsed.type)) {
+        results.push(typedParsed);
+      }
+    }
+  });
+
+  return results;
+}
+
+/**
+ * Parse Server-Sent Events format
+ */
+function parseEventStream(buffer: string, config: StreamConfigModel): any[] {
+  const results: any[] = [];
+  const events = buffer.split('\n\n').filter(event => event.trim());
+
+  events.forEach(event => {
+    const lines = event.split('\n');
+    let eventData = '';
+
+    lines.forEach(line => {
+      if (line.startsWith('data: ')) {
+        eventData += line.substring(6);
+      }
+    });
+
+    if (eventData) {
+      try {
+        const parsed = JSON.parse(eventData);
+        if(parsed) results.push(parsed);
+      } catch (e) {
+        results.push({
+          content: eventData,
+          _streamType: 'event_stream',
+          _format: StreamType.EVENT_STREAM,
+          _timestamp: new Date().toISOString()
+        });
+      }
+    }
+  });
+
+  return results;
+}
+
+/**
+ * Auto-detect and parse unknown formats
+ */
+function autoParse(buffer: string, contentType: string, config: StreamConfigModel): any[] {
+  // Check content type hints
+  if (contentType.includes('application/x-ndjson') || isNdjsonFormat(buffer)) {
+    return parseNdjson(buffer, config);
+  }
+
+  if (contentType.includes('application/json') || buffer.includes('{')) {
+    // Check if it looks like AI streaming
+    if (buffer.includes('"type":') && buffer.includes('"ai_')) {
+      return parseAiStreaming(buffer, config);
+    }
+    return parseJsonObjects(buffer, config);
+  }
+
+  if (contentType.includes('text/event-stream')) {
+    return parseEventStream(buffer, config);
+  }
+
+  // Fallback: try to parse as JSON, otherwise treat as text
+  const jsonObjects = extractJsonObjects(buffer);
+  if (jsonObjects.length > 0) {
+    return parseJsonObjects(buffer, config);
+  } else {
+    // Return as text chunks
+    return buffer.split('\n').filter(line => line.trim()).map(line => ({
+      content: line.trim(),
+      _streamType: 'text',
+      _format: 'plain',
+      _timestamp: new Date().toISOString()
+    }));
+  }
+}
+
+/**
+ * Detect streaming format from content
+ */
+function detectFormat(buffer: string, contentType: string): StreamType {
+  if (contentType.includes('application/x-ndjson') || isNdjsonFormat(buffer)) {
+    return StreamType.NDJSON;
+  }
+
+  if (contentType.includes('text/event-stream')) {
+    return StreamType.EVENT_STREAM;
+  }
+
+  if (contentType.includes('application/json') || buffer.includes('{')) {
+    // Check if it looks like AI streaming
+    if (buffer.includes('"type":') && buffer.includes('"ai_')) {
+      return StreamType.AI_STREAMING;
+    }
+    return StreamType.JSON;
+  }
+
+  return StreamType.JSON; // Default
+}
+
+/**
+ * Check if buffer is NDJSON format
+ */
+function isNdjsonFormat(buffer: string): boolean {
+  const lines = buffer.split('\n').filter(line => line.trim());
+  const jsonLines = lines.filter(line => line.trim().startsWith('{') && line.trim().endsWith('}'));
+  return jsonLines.length > 1 || (lines.length > 1 && jsonLines.length > 0);
+}
+
+/**
+ * Convenience functions for common use cases
+ */
+
+// Stream with JSON format
+export function streamJSON<T = any>(): OperatorFunction<any, StreamOutputModel<T>> {
+  return requestStreaming<T>(StreamConfigModel.adapt({ streamType: StreamType.JSON }));
+}
+
+// Stream with NDJSON format
+export function streamNDJSON<T = any>(): OperatorFunction<any, StreamOutputModel<T>> {
+  return requestStreaming<T>(StreamConfigModel.adapt({ streamType: StreamType.NDJSON }));
+}
+
+// Stream AI responses
+export function streamAI<T = any>(): OperatorFunction<any, StreamOutputModel<T>> {
+  return requestStreaming<T>(StreamConfigModel.adapt({ streamType: StreamType.AI_STREAMING }));
+}
+
+// Stream Server-Sent Events
+export function streamEvents<T = any>(): OperatorFunction<any, StreamOutputModel<T>> {
+  return requestStreaming<T>(StreamConfigModel.adapt({ streamType: StreamType.EVENT_STREAM }));
+}
+
+// Auto-detect streaming format
+export function streamAuto<T = any>(): OperatorFunction<any, StreamOutputModel<T>> {
+  return requestStreaming<T>(StreamConfigModel.adapt({ streamType: StreamType.AUTO }));
+}