autotel-subscribers 4.0.0

package/examples/subscriber-databricks.ts

@@ -0,0 +1,280 @@
+/**
+ * Databricks Subscriber Example
+ *
+ * Sends events events to Databricks Delta Lake via REST API.
+ * This is a complete, production-ready implementation.
+ *
+ * Installation:
+ * ```bash
+ * # No additional dependencies required (uses fetch)
+ * ```
+ *
+ * Setup Databricks table:
+ * ```sql
+ * CREATE TABLE events.events (
+ *   event_id STRING NOT NULL,
+ *   event_type STRING NOT NULL,
+ *   event_name STRING NOT NULL,
+ *   attributes MAP<STRING, STRING>,
+ *   funnel STRING,
+ *   step STRING,
+ *   operation STRING,
+ *   outcome STRING,
+ *   value DECIMAL(18,2),
+ *   timestamp TIMESTAMP NOT NULL,
+ *   created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP()
+ * )
+ * USING DELTA
+ * PARTITIONED BY (DATE(timestamp))
+ * TBLPROPERTIES (
+ *   'delta.autoOptimize.optimizeWrite' = 'true',
+ *   'delta.autoOptimize.autoCompact' = 'true'
+ * );
+ * ```
+ *
+ * Setup Authentication:
+ * 1. Generate Personal Access Token in Databricks
+ * 2. Get your workspace URL (e.g., 'https://dbc-1234567-890.cloud.databricks.com')
+ * 3. Get your SQL warehouse ID
+ *
+ * Usage:
+ * ```typescript
+ * import { Events } from 'autotel/events';
+ * import { DatabricksSubscriber } from './adapter-databricks';
+ *
+ * const events = new Events('app', {
+ *   subscribers: [
+ *     new DatabricksSubscriber({
+ *       host: 'https://dbc-1234567-890.cloud.databricks.com',
+ *       token: process.env.DATABRICKS_TOKEN!,
+ *       catalog: 'main',
+ *       schema: 'events',
+ *       table: 'events',
+ *       warehouseId: 'abc123def456' // SQL warehouse ID
+ *     })
+ *   ]
+ * });
+ *
+ * events.trackEvent('order.completed', { orderId: 'ord_123', amount: 99.99 });
+ * ```
+ */
+
+import {
+  EventSubscriber,
+  type EventPayload,
+} from '../src/event-subscriber-base';
+
+export interface DatabricksSubscriberConfig {
+  /** Databricks workspace URL (e.g., 'https://dbc-1234567-890.cloud.databricks.com') */
+  host: string;
+  /** Personal Access Token */
+  token: string;
+  /** Unity Catalog name (default: 'main') */
+  catalog?: string;
+  /** Schema/database name */
+  schema: string;
+  /** Table name */
+  table: string;
+  /** SQL Warehouse ID (for SQL execution) */
+  warehouseId: string;
+  /** Enable/disable subscriber */
+  enabled?: boolean;
+  /** Batch size (default: 200) */
+  batchSize?: number;
+  /** Flush interval in ms (default: 10000) */
+  flushInterval?: number;
+  /** Request timeout in ms (default: 30000) */
+  timeout?: number;
+}
+
+interface SQLExecutionResponse {
+  statement_id: string;
+  status: {
+    state: 'PENDING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'CANCELED';
+  };
+}
+
+export class DatabricksSubscriber extends EventSubscriber {
+  readonly name = 'DatabricksSubscriber';
+  readonly version = '1.0.0';
+
+  private config: Required<DatabricksSubscriberConfig>;
+  private buffer: EventPayload[] = [];
+  private flushIntervalHandle: NodeJS.Timeout | null = null;
+
+  constructor(config: DatabricksSubscriberConfig) {
+    super();
+
+    // Set defaults
+    this.config = {
+      catalog: 'main',
+      enabled: true,
+      batchSize: 200,
+      flushInterval: 10_000,
+      timeout: 30_000,
+      ...config,
+    };
+
+    this.enabled = this.config.enabled;
+
+    if (this.enabled) {
+      this.startFlushInterval();
+    }
+  }
+
+  private startFlushInterval(): void {
+    this.flushIntervalHandle = setInterval(() => {
+      void this.flushBuffer();
+    }, this.config.flushInterval);
+  }
+
+  protected async sendToDestination(payload: EventPayload): Promise<void> {
+    this.buffer.push(payload);
+
+    // Auto-flush at batch size
+    if (this.buffer.length >= this.config.batchSize) {
+      await this.flushBuffer();
+    }
+  }
+
+  private async flushBuffer(): Promise<void> {
+    if (this.buffer.length === 0) return;
+
+    const batch = [...this.buffer];
+    this.buffer = [];
+
+    try {
+      await this.insertBatch(batch);
+    } catch (error) {
+      console.error('[DatabricksSubscriber] Failed to flush batch:', error);
+      // Re-add to buffer for retry
+      this.buffer.unshift(...batch);
+    }
+  }
+
+  private async insertBatch(events: EventPayload[]): Promise<void> {
+    // Build VALUES clause
+    const values = events
+      .map((event) => {
+        const eventId = crypto.randomUUID();
+        const attributes = event.attributes
+          ? Object.entries(event.attributes)
+              .map(
+                ([key, value]) =>
+                  `'${this.escapeSql(key)}', '${this.escapeSql(String(value))}'`
+              )
+              .join(', ')
+          : '';
+
+        return `(
+          '${eventId}',
+          '${this.escapeSql(event.type)}',
+          '${this.escapeSql(event.name)}',
+          ${attributes ? `map(${attributes})` : 'map()'},
+          ${event.funnel ? `'${this.escapeSql(event.funnel)}'` : 'NULL'},
+          ${event.step ? `'${this.escapeSql(event.step)}'` : 'NULL'},
+          ${event.operation ? `'${this.escapeSql(event.operation)}'` : 'NULL'},
+          ${event.outcome ? `'${this.escapeSql(event.outcome)}'` : 'NULL'},
+          ${event.value === undefined ? 'NULL' : event.value},
+          CAST('${event.timestamp}' AS TIMESTAMP)
+        )`;
+      })
+      .join(',\n');
+
+    const sql = `
+      INSERT INTO ${this.config.catalog}.${this.config.schema}.${this.config.table}
+      (event_id, event_type, event_name, attributes, funnel, step, operation, outcome, value, timestamp)
+      VALUES ${values}
+    `;
+
+    await this.executeSql(sql);
+  }
+
+  private async executeSql(sql: string): Promise<void> {
+    // Execute SQL via Databricks SQL API
+    const response = await fetch(
+      `${this.config.host}/api/2.0/sql/statements`,
+      {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${this.config.token}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          statement: sql,
+          warehouse_id: this.config.warehouseId,
+          wait_timeout: `${this.config.timeout / 1000}s`,
+        }),
+        signal: AbortSignal.timeout(this.config.timeout),
+      }
+    );
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(
+        `Databricks API returned ${response.status}: ${error}`
+      );
+    }
+
+    const result: SQLExecutionResponse = await response.json();
+
+    // Check execution status
+    if (result.status.state === 'FAILED') {
+      throw new Error('SQL execution failed');
+    }
+
+    // For long-running queries, you might want to poll for completion
+    // This example assumes synchronous execution (wait_timeout)
+  }
+
+  private escapeSql(value: string): string {
+    // Escape single quotes for SQL
+    return value.replaceAll('\'', "''");
+  }
+
+  protected handleError(error: Error, payload: EventPayload): void {
+    console.error(
+      `[DatabricksSubscriber] Failed to send ${payload.type}:`,
+      error,
+      {
+        eventName: payload.name,
+        attributes: payload.attributes,
+      }
+    );
+
+    // Databricks-specific error handling
+    if (error.message.includes('401')) {
+      console.error(
+        '[DatabricksSubscriber] Authentication failed - check your token'
+      );
+    }
+
+    if (error.message.includes('warehouse')) {
+      console.error(
+        '[DatabricksSubscriber] SQL warehouse error - check warehouse ID and status'
+      );
+    }
+
+    if (error.message.includes('timeout')) {
+      console.error(
+        '[DatabricksSubscriber] Timeout - consider increasing timeout or reducing batch size'
+      );
+    }
+  }
+
+  async shutdown(): Promise<void> {
+    // Clear flush interval
+    if (this.flushIntervalHandle) {
+      clearInterval(this.flushIntervalHandle);
+      this.flushIntervalHandle = null;
+    }
+
+    // Flush remaining events
+    await this.flushBuffer();
+
+    // Wait for pending requests
+    await super.shutdown();
+
+    console.log('[DatabricksSubscriber] Shutdown complete');
+  }
+}

package/examples/subscriber-kafka.ts

@@ -0,0 +1,326 @@
+/**
+ * Kafka Streaming Subscriber Example
+ *
+ * Production-ready Kafka subscriber for high-throughput, ordered event streaming.
+ *
+ * Installation:
+ * ```bash
+ * pnpm add kafkajs
+ * ```
+ *
+ * Features:
+ * - Partitioning by userId for ordered events per user
+ * - High-throughput batching (configurable up to 10,000+ events/batch)
+ * - Backpressure handling
+ * - Automatic retries with exponential backoff
+ * - Compression support (gzip, snappy, lz4, zstd)
+ * - Graceful shutdown with buffer draining
+ *
+ * Usage:
+ * ```typescript
+ * import { Events } from 'autotel/events';
+ * import { KafkaSubscriber } from './adapter-kafka';
+ *
+ * const events = new Events('app', {
+ *   subscribers: [
+ *     new KafkaSubscriber({
+ *       clientId: 'events-producer',
+ *       brokers: ['kafka1:9092', 'kafka2:9092', 'kafka3:9092'],
+ *       topic: 'events.events',
+ *       partitionStrategy: 'userId', // or 'tenantId', 'eventType', 'round-robin'
+ *       compression: 'gzip',
+ *       maxBufferSize: 10000,
+ *       maxBatchSize: 1000,
+ *       bufferOverflowStrategy: 'block'
+ *     })
+ *   ]
+ * });
+ *
+ * // High-throughput: 10k+ events/sec
+ * for (let i = 0; i < 10000; i++) {
+ *   await events.trackEvent('page.viewed', { userId: `user_${i % 100}` });
+ * }
+ *
+ * // Graceful shutdown
+ * await events.flush();
+ * ```
+ */
+
+import {
+  StreamingEventSubscriber,
+  type BufferOverflowStrategy,
+} from '../src/streaming-event-subscriber';
+import type { EventPayload } from '../src/event-subscriber-base';
+import { Kafka, Producer, CompressionTypes, type ProducerRecord } from 'kafkajs';
+
+type CompressionType = 'gzip' | 'snappy' | 'lz4' | 'zstd' | 'none';
+type PartitionStrategy = 'userId' | 'tenantId' | 'eventType' | 'round-robin';
+
+export interface KafkaSubscriberConfig {
+  /** Kafka client ID */
+  clientId: string;
+
+  /** Kafka broker addresses */
+  brokers: string[];
+
+  /** Topic to publish events to */
+  topic: string;
+
+  /** Partitioning strategy (default: 'userId') */
+  partitionStrategy?: PartitionStrategy;
+
+  /** Compression type (default: 'gzip') */
+  compression?: CompressionType;
+
+  /** Enable/disable subscriber */
+  enabled?: boolean;
+
+  /** Maximum buffer size (default: 10000) */
+  maxBufferSize?: number;
+
+  /** Maximum batch size (default: 1000) */
+  maxBatchSize?: number;
+
+  /** Buffer overflow strategy (default: 'block') */
+  bufferOverflowStrategy?: BufferOverflowStrategy;
+
+  /** Flush interval in ms (default: 1000) */
+  flushIntervalMs?: number;
+
+  /** SASL authentication (optional) */
+  sasl?: {
+    mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
+    username: string;
+    password: string;
+  };
+
+  /** SSL/TLS configuration (optional) */
+  ssl?: boolean;
+}
+
+export class KafkaSubscriber extends StreamingEventSubscriber {
+  readonly name = 'KafkaSubscriber';
+  readonly version = '1.0.0';
+
+  private kafka: Kafka;
+  private producer: Producer;
+  private subscriberConfig: Required<Omit<KafkaSubscriberConfig, 'sasl' | 'ssl'>> & {
+    sasl?: KafkaSubscriberConfig['sasl'];
+    ssl?: boolean;
+  };
+  private roundRobinCounter = 0;
+  private isConnected = false;
+
+  constructor(config: KafkaSubscriberConfig) {
+    super({
+      maxBufferSize: config.maxBufferSize ?? 10_000,
+      maxBatchSize: config.maxBatchSize ?? 1000,
+      bufferOverflowStrategy: config.bufferOverflowStrategy ?? 'block',
+      flushIntervalMs: config.flushIntervalMs ?? 1000,
+    });
+
+    // Set config defaults
+    this.adapterConfig = {
+      clientId: config.clientId,
+      brokers: config.brokers,
+      topic: config.topic,
+      partitionStrategy: config.partitionStrategy ?? 'userId',
+      compression: config.compression ?? 'gzip',
+      enabled: config.enabled ?? true,
+      maxBufferSize: config.maxBufferSize ?? 10_000,
+      maxBatchSize: config.maxBatchSize ?? 1000,
+      bufferOverflowStrategy: config.bufferOverflowStrategy ?? 'block',
+      flushIntervalMs: config.flushIntervalMs ?? 1000,
+      sasl: config.sasl,
+      ssl: config.ssl,
+    };
+
+    this.enabled = this.adapterConfig.enabled;
+
+    if (this.enabled) {
+      this.initializeKafka();
+    }
+  }
+
+  private initializeKafka(): void {
+    try {
+      // Initialize Kafka client
+      this.kafka = new Kafka({
+        clientId: this.adapterConfig.clientId,
+        brokers: this.adapterConfig.brokers,
+        sasl: this.adapterConfig.sasl,
+        ssl: this.adapterConfig.ssl,
+        retry: {
+          initialRetryTime: 100,
+          retries: 8,
+          maxRetryTime: 30_000,
+          multiplier: 2,
+        },
+      });
+
+      // Create producer
+      this.producer = this.kafka.producer({
+        allowAutoTopicCreation: false,
+        compression: this.getCompressionType(this.adapterConfig.compression),
+        maxInFlightRequests: 5,
+        idempotent: true, // Exactly-once semantics
+      });
+
+      // Connect asynchronously
+      void this.connect();
+    } catch (error) {
+      console.error('[KafkaSubscriber] Failed to initialize:', error);
+      this.enabled = false;
+    }
+  }
+
+  private async connect(): Promise<void> {
+    try {
+      await this.producer.connect();
+      this.isConnected = true;
+      console.log('[KafkaSubscriber] Connected successfully');
+    } catch (error) {
+      console.error('[KafkaSubscriber] Failed to connect:', error);
+      this.enabled = false;
+      this.isConnected = false;
+    }
+  }
+
+  private getCompressionType(compression: CompressionType): CompressionTypes {
+    switch (compression) {
+      case 'gzip': {
+        return CompressionTypes.GZIP;
+      }
+      case 'snappy': {
+        return CompressionTypes.Snappy;
+      }
+      case 'lz4': {
+        return CompressionTypes.LZ4;
+      }
+      case 'zstd': {
+        return CompressionTypes.ZSTD;
+      }
+      default: {
+        return CompressionTypes.None;
+      }
+    }
+  }
+
+  /**
+   * Get partition key based on configured strategy
+   */
+  protected getPartitionKey(payload: EventPayload): string {
+    switch (this.adapterConfig.partitionStrategy) {
+      case 'userId': {
+        return payload.attributes?.userId?.toString() || 'default';
+      }
+
+      case 'tenantId': {
+        return payload.attributes?.tenantId?.toString() || 'default';
+      }
+
+      case 'eventType': {
+        return payload.type;
+      } // 'event', 'funnel', 'outcome', 'value'
+
+      case 'round-robin': {
+        // Round-robin across partitions
+        this.roundRobinCounter = (this.roundRobinCounter + 1) % 100;
+        return `partition-${this.roundRobinCounter}`;
+      }
+
+      default: {
+        return 'default';
+      }
+    }
+  }
+
+  /**
+   * Send batch of events to Kafka
+   */
+  protected async sendBatch(events: EventPayload[]): Promise<void> {
+    if (!this.isConnected) {
+      throw new Error('[KafkaSubscriber] Producer not connected');
+    }
+
+    // Build Kafka messages
+    const messages = events.map((event) => ({
+      key: this.getPartitionKey(event),
+      value: JSON.stringify(event),
+      headers: {
+        'event-type': event.type,
+        'event-name': event.name,
+        timestamp: event.timestamp,
+      },
+    }));
+
+    // Send to Kafka
+    const record: ProducerRecord = {
+      topic: this.adapterConfig.topic,
+      messages,
+    };
+
+    try {
+      const result = await this.producer.send(record);
+
+      // Log successful send (debug)
+      if (process.env.DEBUG) {
+        console.log(
+          `[KafkaSubscriber] Sent ${messages.length} events to partition ${result[0].partition}`
+        );
+      }
+    } catch (error) {
+      console.error(
+        `[KafkaSubscriber] Failed to send ${messages.length} events:`,
+        error
+      );
+      throw error; // Re-throw for retry logic
+    }
+  }
+
+  /**
+   * Handle errors (override from EventSubscriber)
+   */
+  protected handleError(error: Error, payload: EventPayload): void {
+    console.error(
+      `[KafkaSubscriber] Failed to process ${payload.type} event:`,
+      error,
+      {
+        eventName: payload.name,
+        partitionKey: this.getPartitionKey(payload),
+      }
+    );
+
+    // Check for specific Kafka errors
+    if (error.message.includes('NOT_LEADER_FOR_PARTITION')) {
+      console.error(
+        '[KafkaSubscriber] Partition leadership changed - will retry'
+      );
+    }
+
+    if (error.message.includes('BROKER_NOT_AVAILABLE')) {
+      console.error('[KafkaSubscriber] Broker unavailable - check cluster health');
+    }
+  }
+
+  /**
+   * Graceful shutdown
+   */
+  async shutdown(): Promise<void> {
+    console.log('[KafkaSubscriber] Starting graceful shutdown...');
+
+    // Flush buffer and drain pending requests
+    await super.shutdown();
+
+    // Disconnect producer
+    if (this.isConnected && this.producer) {
+      try {
+        await this.producer.disconnect();
+        this.isConnected = false;
+        console.log('[KafkaSubscriber] Disconnected successfully');
+      } catch (error) {
+        console.error('[KafkaSubscriber] Error during disconnect:', error);
+      }
+    }
+  }
+}