http-request-manager 18.7.19 → 18.7.21

package/src/lib/services/request-manager-services/rxjs-operators/request-streaming.ts

@@ -0,0 +1,436 @@
+import { OperatorFunction } from 'rxjs';
+import { filter, map } from 'rxjs/operators';
+import { HttpEventType } from '@angular/common/http';
+import { StreamType } from '../../request-manager-state-service/models/stream-type.enum';
+
+export interface StreamConfig {
+  streamType: StreamType;
+}
+
+// TypeScript interfaces for proper type safety
+export interface StreamEvent<T = any> {
+  type: 'progress' | 'complete' | 'data';
+  data: T;
+  metadata?: {
+    timestamp: Date;
+    streamType: StreamType;
+    contentLength?: number;
+  };
+}
+
+export interface ParsingResult<T = any> {
+  success: boolean;
+  data?: T[];
+  error?: string;
+}
+
+/**
+ * Stateful processor for managing streaming data and events
+ */
+class StreamingProcessor<T = any> {
+  private buffer = '';
+  private contentType = '';
+  private readonly maxBufferSize = 10 * 1024 * 1024; // 10MB limit
+  private streamConfig: StreamConfig;
+
+  constructor(config?: StreamConfig) {
+    this.streamConfig = config || { streamType: StreamType.AI_STREAMING };
+  }
+
+  /**
+   * Process HTTP events and extract streaming data
+   */
+  process(event: any): T[] | null {
+    switch (event.type) {
+      case HttpEventType.ResponseHeader:
+        this.contentType = event.headers?.get('content-type') || '';
+        return null;
+
+      case HttpEventType.DownloadProgress:
+        if (event.partialText) {
+          this.appendToBuffer(event.partialText);
+          const parsedData = this.parseBuffer();
+          return parsedData.length > 0 ? [parsedData[0]] : null;
+        }
+        return null;
+
+      case HttpEventType.Response:
+        if (event.body) {
+          this.appendToBuffer(event.body);
+          const parsedData = this.parseBuffer();
+          return parsedData.length > 0 ? parsedData : null;
+        }
+        return null;
+
+      default:
+        return null;
+    }
+  }
+
+  /**
+   * Append data to buffer with size management
+   */
+  private appendToBuffer(data: string): void {
+    this.buffer += data;
+
+    // Implement sliding window if buffer gets too large
+    if (this.buffer.length > this.maxBufferSize) {
+      this.buffer = this.buffer.slice(-this.maxBufferSize / 2);
+    }
+  }
+
+  /**
+   * Parse current buffer content
+   */
+  private parseBuffer(): T[] {
+    if (!this.buffer.trim()) {
+      return [];
+    }
+
+    try {
+      const result = parseStreamData(this.buffer, this.contentType, this.streamConfig);
+      return result as T[];
+    } catch (error) {
+      console.warn('Failed to parse streaming data:', error);
+      return [];
+    }
+  }
+
+  /**
+   * Reset processor state
+   */
+  reset(): void {
+    this.buffer = '';
+    this.contentType = '';
+  }
+
+  /**
+   * Update configuration
+   */
+  updateConfig(config: StreamConfig): void {
+    this.streamConfig = config;
+  }
+}
+
+/**
+ * Safe JSON parsing with error handling
+ */
+function safeJsonParse<T>(jsonStr: string, fallback?: T): T | null {
+  try {
+    return JSON.parse(jsonStr);
+  } catch (error) {
+    console.warn('Failed to parse JSON:', jsonStr.substring(0, 100));
+    return fallback || null;
+  }
+}
+
+/**
+ * Safe JSON array parsing
+ */
+function safeJsonArrayParse<T>(jsonStr: string, fallback: T[] = []): T[] {
+  try {
+    const parsed = JSON.parse(jsonStr);
+    return Array.isArray(parsed) ? parsed : fallback;
+  } catch (error) {
+    console.warn('Failed to parse JSON array:', jsonStr.substring(0, 100));
+    return fallback;
+  }
+}
+
+/**
+ * Enhanced JSON object extraction with better error handling
+ */
+function extractJsonObjects(text: string): string[] {
+  const objects: string[] = [];
+  let braceCount = 0;
+  let currentObject = '';
+  let inString = false;
+  let escaped = false;
+
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+
+    if (escaped) {
+      currentObject += char;
+      escaped = false;
+      continue;
+    }
+
+    if (char === '\\') {
+      escaped = true;
+      currentObject += char;
+      continue;
+    }
+
+    if (char === '"' && !escaped) {
+      inString = !inString;
+    }
+
+    if (!inString) {
+      if (char === '{') braceCount++;
+      else if (char === '}') braceCount--;
+    }
+
+    currentObject += char;
+
+    if (braceCount === 0 && currentObject.trim()) {
+      objects.push(currentObject.trim());
+      currentObject = '';
+    }
+  }
+
+  return objects;
+}
+
+/**
+ * COMPREHENSIVE REQUEST STREAMING OPERATOR - FULLY ABSTRACTED
+ * Refactored for better type safety and memory management
+ *
+ * Single function that handles ALL streaming formats without hardcoded assumptions:
+ * - JSON format (Individual JSON objects)
+ * - NDJSON format (Newline delimited JSON)
+ * - AI streaming format (Real-time AI responses)
+ * - Server-Sent Events format
+ * - Auto-detection mode
+ *
+ * Usage:
+ *   this.http.get(url, options).pipe(requestStreaming()).subscribe(data => {...})
+ *   this.http.get(url, options).pipe(requestStreaming({ streamType: StreamType.NDJSON })).subscribe(data => {...})
+ */
+export function requestStreaming<T = any>(config?: StreamConfig): OperatorFunction<any, T[]> {
+  const processor = new StreamingProcessor<T>(config);
+
+  return input$ => input$.pipe(
+    // Filter only streaming-related events
+    filter((event: any) =>
+      event.type === HttpEventType.DownloadProgress ||
+      event.type === HttpEventType.Response ||
+      event.type === HttpEventType.ResponseHeader
+    ),
+
+    // Process events using the streaming processor
+    map((event: any) => {
+      const result = processor.process(event);
+      return result;
+    }),
+
+    // Filter out null results
+    filter((result): result is T[] => result !== null)
+  );
+}
+
+/**
+ * Main streaming data parser - handles ALL formats based on streamType
+ */
+function parseStreamData(buffer: string, contentType: string, config: StreamConfig): any[] {
+  const results: any[] = [];
+
+  if (!buffer.trim()) {
+    return results;
+  }
+
+  // Determine parsing format based on streamType and content type
+  const format = config.streamType === StreamType.AI_STREAMING ? detectFormat(buffer, contentType) : config.streamType;
+
+  switch (format) {
+    case StreamType.NDJSON:
+      return parseNdjson(buffer, config);
+
+    case StreamType.JSON:
+      return parseJsonObjects(buffer, config);
+
+    case StreamType.AI_STREAMING:
+      return parseAiStreaming(buffer, config);
+
+    case StreamType.EVENT_STREAM:
+      return parseEventStream(buffer, config);
+
+    default:
+      return autoParse(buffer, contentType, config);
+  }
+}
+
+/**
+ * Parse NDJSON format (JSON objects per line)
+ */
+function parseNdjson(buffer: string, config: StreamConfig): any[] {
+  const results: any[] = [];
+  const lines = buffer.split('\n').filter(line => line.trim());
+
+  lines.forEach(line => {
+    const trimmedLine = line.trim();
+    if (trimmedLine && trimmedLine.startsWith('{') && trimmedLine.endsWith('}')) {
+      const parsed = safeJsonParse(trimmedLine);
+      if (parsed) results.push(parsed);
+    }
+  });
+
+  return results;
+}
+
+/**
+ * Parse individual JSON objects
+ */
+function parseJsonObjects(buffer: string, config: StreamConfig): any[] {
+  const results: any[] = [];
+  const jsonObjects = extractJsonObjects(buffer);
+
+  jsonObjects.forEach(jsonStr => {
+    const parsed = safeJsonParse(jsonStr);
+    if (parsed) results.push(parsed);
+  });
+
+  return results;
+}
+
+/**
+ * Parse AI streaming format
+ */
+function parseAiStreaming(buffer: string, config: StreamConfig): any[] {
+  const results: any[] = [];
+  const jsonObjects = extractJsonObjects(buffer);
+
+  jsonObjects.forEach(jsonStr => {
+    const parsed = safeJsonParse(jsonStr);
+    if (parsed && typeof parsed === 'object' && parsed !== null && 'type' in parsed) {
+      const typedParsed = parsed as any;
+      if (['ai_thinking', 'ai_response', 'ai_suggestion', 'ai_status'].includes(typedParsed.type)) {
+        results.push(typedParsed);
+      }
+    }
+  });
+
+  return results;
+}
+
+/**
+ * Parse Server-Sent Events format
+ */
+function parseEventStream(buffer: string, config: StreamConfig): any[] {
+  const results: any[] = [];
+  const events = buffer.split('\n\n').filter(event => event.trim());
+
+  events.forEach(event => {
+    const lines = event.split('\n');
+    let eventData = '';
+
+    lines.forEach(line => {
+      if (line.startsWith('data: ')) {
+        eventData += line.substring(6);
+      }
+    });
+
+    if (eventData) {
+      try {
+        const parsed = JSON.parse(eventData);
+        if(parsed) results.push(parsed);
+      } catch (e) {
+        results.push({
+          content: eventData,
+          _streamType: 'event_stream',
+          _format: StreamType.EVENT_STREAM,
+          _timestamp: new Date().toISOString()
+        });
+      }
+    }
+  });
+
+  return results;
+}
+
+/**
+ * Auto-detect and parse unknown formats
+ */
+function autoParse(buffer: string, contentType: string, config: StreamConfig): any[] {
+  // Check content type hints
+  if (contentType.includes('application/x-ndjson') || isNdjsonFormat(buffer)) {
+    return parseNdjson(buffer, config);
+  }
+
+  if (contentType.includes('application/json') || buffer.includes('{')) {
+    // Check if it looks like AI streaming
+    if (buffer.includes('"type":') && buffer.includes('"ai_')) {
+      return parseAiStreaming(buffer, config);
+    }
+    return parseJsonObjects(buffer, config);
+  }
+
+  if (contentType.includes('text/event-stream')) {
+    return parseEventStream(buffer, config);
+  }
+
+  // Fallback: try to parse as JSON, otherwise treat as text
+  const jsonObjects = extractJsonObjects(buffer);
+  if (jsonObjects.length > 0) {
+    return parseJsonObjects(buffer, config);
+  } else {
+    // Return as text chunks
+    return buffer.split('\n').filter(line => line.trim()).map(line => ({
+      content: line.trim(),
+      _streamType: 'text',
+      _format: 'plain',
+      _timestamp: new Date().toISOString()
+    }));
+  }
+}
+
+/**
+ * Detect streaming format from content
+ */
+function detectFormat(buffer: string, contentType: string): StreamType {
+  if (contentType.includes('application/x-ndjson') || isNdjsonFormat(buffer)) {
+    return StreamType.NDJSON;
+  }
+
+  if (contentType.includes('text/event-stream')) {
+    return StreamType.EVENT_STREAM;
+  }
+
+  if (contentType.includes('application/json') || buffer.includes('{')) {
+    // Check if it looks like AI streaming
+    if (buffer.includes('"type":') && buffer.includes('"ai_')) {
+      return StreamType.AI_STREAMING;
+    }
+    return StreamType.JSON;
+  }
+
+  return StreamType.JSON; // Default
+}
+
+/**
+ * Check if buffer is NDJSON format
+ */
+function isNdjsonFormat(buffer: string): boolean {
+  const lines = buffer.split('\n').filter(line => line.trim());
+  const jsonLines = lines.filter(line => line.trim().startsWith('{') && line.trim().endsWith('}'));
+  return jsonLines.length > 1 || (lines.length > 1 && jsonLines.length > 0);
+}
+
+/**
+ * Convenience functions for common use cases
+ */
+
+// Stream with JSON format
+export function streamJSON<T = any>(): OperatorFunction<any, T[]> {
+  return requestStreaming<T>({ streamType: StreamType.JSON });
+}
+
+// Stream with NDJSON format
+export function streamNDJSON<T = any>(): OperatorFunction<any, T[]> {
+  return requestStreaming<T>({ streamType: StreamType.NDJSON });
+}
+
+// Stream AI responses
+export function streamAI<T = any>(): OperatorFunction<any, T[]> {
+  return requestStreaming<T>({ streamType: StreamType.AI_STREAMING });
+}
+
+// Stream Server-Sent Events
+export function streamEvents<T = any>(): OperatorFunction<any, T[]> {
+  return requestStreaming<T>({ streamType: StreamType.EVENT_STREAM });
+}
+
+// Auto-detect streaming format
+export function streamAuto<T = any>(): OperatorFunction<any, T[]> {
+  return requestStreaming<T>({ streamType: StreamType.AUTO });
+}