autotel-subscribers 4.0.0

package/examples/subscriber-pubsub.ts

@@ -0,0 +1,336 @@
+/**
+ * Google Cloud Pub/Sub Streaming Subscriber Example
+ *
+ * Production-ready Pub/Sub subscriber for GCP-native event streaming.
+ *
+ * Installation:
+ * ```bash
+ * pnpm add @google-cloud/pubsub
+ * ```
+ *
+ * Features:
+ * - Ordered delivery with ordering keys
+ * - Automatic batching and flow control
+ * - Backpressure handling
+ * - Message deduplication
+ * - Dead letter queue support
+ * - Graceful shutdown
+ *
+ * Setup:
+ * ```bash
+ * # Create topic
+ * gcloud pubsub topics create events-events \
+ *   --message-retention-duration=7d
+ *
+ * # Create subscription (for consumers)
+ * gcloud pubsub subscriptions create events-events-sub \
+ *   --topic=events-events \
+ *   --ack-deadline=60 \
+ *   --message-retention-duration=7d
+ * ```
+ *
+ * Usage:
+ * ```typescript
+ * import { Events } from 'autotel/events';
+ * import { PubSubSubscriber } from './adapter-pubsub';
+ *
+ * const events = new Events('app', {
+ *   subscribers: [
+ *     new PubSubSubscriber({
+ *       projectId: 'my-gcp-project',
+ *       topicName: 'events-events',
+ *       enableMessageOrdering: true,
+ *       partitionStrategy: 'userId',
+ *       maxBufferSize: 10000,
+ *       maxBatchSize: 1000,
+ *       bufferOverflowStrategy: 'block'
+ *     })
+ *   ]
+ * });
+ *
+ * // Events ordered by userId
+ * await events.trackEvent('order.completed', {
+ *   userId: 'user_123',
+ *   amount: 99.99
+ * });
+ * ```
+ */
+
+import {
+  StreamingEventSubscriber,
+  type BufferOverflowStrategy,
+} from '../src/streaming-event-subscriber';
+import type { EventPayload } from '../src/event-subscriber-base';
+import { PubSub, Topic } from '@google-cloud/pubsub';
+
+type PartitionStrategy = 'userId' | 'tenantId' | 'eventType' | 'none';
+
+export interface PubSubSubscriberConfig {
+  /** GCP Project ID */
+  projectId: string;
+
+  /** Pub/Sub topic name */
+  topicName: string;
+
+  /** Enable message ordering (default: true) */
+  enableMessageOrdering?: boolean;
+
+  /** Partitioning strategy for ordering keys (default: 'userId') */
+  partitionStrategy?: PartitionStrategy;
+
+  /** Enable/disable subscriber */
+  enabled?: boolean;
+
+  /** Maximum buffer size (default: 10000) */
+  maxBufferSize?: number;
+
+  /** Maximum batch size (default: 1000) */
+  maxBatchSize?: number;
+
+  /** Buffer overflow strategy (default: 'block') */
+  bufferOverflowStrategy?: BufferOverflowStrategy;
+
+  /** Flush interval in ms (default: 1000) */
+  flushIntervalMs?: number;
+
+  /** Service account key file path (optional, uses GOOGLE_APPLICATION_CREDENTIALS if not set) */
+  keyFilename?: string;
+
+  /** Enable batching settings (default: true) */
+  enableBatching?: boolean;
+
+  /** Max outstanding messages (flow control) (default: 1000) */
+  maxOutstandingMessages?: number;
+
+  /** Max outstanding bytes (flow control) (default: 100MB) */
+  maxOutstandingBytes?: number;
+}
+
+export class PubSubSubscriber extends StreamingEventSubscriber {
+  readonly name = 'PubSubSubscriber';
+  readonly version = '1.0.0';
+
+  private client: PubSub;
+  private topic: Topic;
+  private subscriberConfig: Required<
+    Omit<PubSubSubscriberConfig, 'keyFilename'>
+  > & {
+    keyFilename?: string;
+  };
+
+  constructor(config: PubSubSubscriberConfig) {
+    super({
+      maxBufferSize: config.maxBufferSize ?? 10_000,
+      maxBatchSize: config.maxBatchSize ?? 1000,
+      bufferOverflowStrategy: config.bufferOverflowStrategy ?? 'block',
+      flushIntervalMs: config.flushIntervalMs ?? 1000,
+    });
+
+    // Set config defaults
+    this.adapterConfig = {
+      projectId: config.projectId,
+      topicName: config.topicName,
+      enableMessageOrdering: config.enableMessageOrdering ?? true,
+      partitionStrategy: config.partitionStrategy ?? 'userId',
+      enabled: config.enabled ?? true,
+      maxBufferSize: config.maxBufferSize ?? 10_000,
+      maxBatchSize: config.maxBatchSize ?? 1000,
+      bufferOverflowStrategy: config.bufferOverflowStrategy ?? 'block',
+      flushIntervalMs: config.flushIntervalMs ?? 1000,
+      keyFilename: config.keyFilename,
+      enableBatching: config.enableBatching ?? true,
+      maxOutstandingMessages: config.maxOutstandingMessages ?? 1000,
+      maxOutstandingBytes: config.maxOutstandingBytes ?? 100 * 1024 * 1024, // 100MB
+    };
+
+    this.enabled = this.adapterConfig.enabled;
+
+    if (this.enabled) {
+      this.initializePubSub();
+    }
+  }
+
+  private initializePubSub(): void {
+    try {
+      const options: any = {
+        projectId: this.adapterConfig.projectId,
+      };
+
+      if (this.adapterConfig.keyFilename) {
+        options.keyFilename = this.adapterConfig.keyFilename;
+      }
+
+      this.client = new PubSub(options);
+
+      // Get topic reference
+      this.topic = this.client.topic(this.adapterConfig.topicName);
+
+      // Configure topic settings (combine all options in single call to avoid overwriting)
+      const publishOptions: any = {};
+
+      if (this.adapterConfig.enableMessageOrdering) {
+        publishOptions.messageOrdering = true;
+      }
+
+      if (this.adapterConfig.enableBatching) {
+        publishOptions.batching = {
+          maxMessages: this.adapterConfig.maxBatchSize,
+          maxMilliseconds: this.adapterConfig.flushIntervalMs,
+        };
+        publishOptions.flowControlOptions = {
+          maxOutstandingMessages: this.adapterConfig.maxOutstandingMessages,
+          maxOutstandingBytes: this.adapterConfig.maxOutstandingBytes,
+        };
+      }
+
+      // Set all options at once to avoid overwriting previous settings
+      if (Object.keys(publishOptions).length > 0) {
+        this.topic.setPublishOptions(publishOptions);
+      }
+
+      console.log('[PubSubSubscriber] Initialized successfully');
+    } catch (error) {
+      console.error('[PubSubSubscriber] Failed to initialize:', error);
+      this.enabled = false;
+    }
+  }
+
+  /**
+   * Get ordering key (partition key) based on configured strategy
+   *
+   * Pub/Sub uses ordering keys to ensure messages with same key
+   * are delivered in order to subscribers.
+   */
+  protected getPartitionKey(payload: EventPayload): string {
+    switch (this.adapterConfig.partitionStrategy) {
+      case 'userId': {
+        return payload.attributes?.userId?.toString() || '';
+      }
+
+      case 'tenantId': {
+        return payload.attributes?.tenantId?.toString() || '';
+      }
+
+      case 'eventType': {
+        return payload.type;
+      } // 'event', 'funnel', 'outcome', 'value'
+
+      case 'none': {
+        return '';
+      } // No ordering
+
+      default: {
+        return '';
+      }
+    }
+  }
+
+  /**
+   * Send batch of events to Pub/Sub
+   */
+  protected async sendBatch(events: EventPayload[]): Promise<void> {
+    // Publish all messages concurrently
+    const publishPromises = events.map((event) => {
+      const data = Buffer.from(JSON.stringify(event));
+      const orderingKey = this.getPartitionKey(event);
+
+      const message: any = {
+        data,
+        attributes: {
+          eventType: event.type,
+          eventName: event.name,
+          timestamp: event.timestamp,
+        },
+      };
+
+      // Add ordering key if message ordering is enabled
+      if (
+        this.adapterConfig.enableMessageOrdering &&
+        orderingKey
+      ) {
+        message.orderingKey = orderingKey;
+      }
+
+      return this.topic.publishMessage(message);
+    });
+
+    try {
+      // Wait for all publishes to complete
+      const messageIds = await Promise.all(publishPromises);
+
+      // Success - log metrics
+      if (process.env.DEBUG) {
+        console.log(
+          `[PubSubSubscriber] Published ${events.length} messages (IDs: ${messageIds.slice(0, 3).join(', ')}${events.length > 3 ? '...' : ''})`
+        );
+      }
+    } catch (error: any) {
+      console.error(
+        `[PubSubSubscriber] Failed to publish ${events.length} messages:`,
+        error
+      );
+
+      // Handle specific Pub/Sub errors
+      if (error.code === 10) {
+        console.error(
+          '[PubSubSubscriber] Flow control limits exceeded - reduce rate or increase limits'
+        );
+      }
+
+      if (error.code === 5) {
+        console.error(
+          `[PubSubSubscriber] Topic not found: ${this.adapterConfig.topicName}`
+        );
+      }
+
+      throw error;
+    }
+  }
+
+  /**
+   * Handle errors (override from EventSubscriber)
+   */
+  protected handleError(error: Error, payload: EventPayload): void {
+    console.error(
+      `[PubSubSubscriber] Failed to process ${payload.type} event:`,
+      error,
+      {
+        eventName: payload.name,
+        orderingKey: this.getPartitionKey(payload),
+        topicName: this.adapterConfig.topicName,
+      }
+    );
+  }
+
+  /**
+   * Graceful shutdown
+   */
+  async shutdown(): Promise<void> {
+    console.log('[PubSubSubscriber] Starting graceful shutdown...');
+
+    // Flush buffer and drain pending requests
+    await super.shutdown();
+
+    // Flush any remaining messages in topic's internal buffer
+    if (this.topic) {
+      try {
+        await this.topic.flush();
+        console.log('[PubSubSubscriber] Flushed topic buffer');
+      } catch (error) {
+        console.error('[PubSubSubscriber] Error flushing topic:', error);
+      }
+    }
+
+    // Close Pub/Sub client
+    if (this.client) {
+      try {
+        await this.client.close();
+        console.log('[PubSubSubscriber] Closed Pub/Sub client');
+      } catch (error) {
+        console.error('[PubSubSubscriber] Error closing client:', error);
+      }
+    }
+
+    console.log('[PubSubSubscriber] Shutdown complete');
+  }
+}

package/examples/subscriber-snowflake.ts

@@ -0,0 +1,232 @@
+/**
+ * Snowflake Subscriber Example
+ *
+ * Sends events events to Snowflake data warehouse.
+ * This is a complete, production-ready implementation.
+ *
+ * Installation:
+ * ```bash
+ * pnpm add snowflake-sdk
+ * ```
+ *
+ * Setup Snowflake table:
+ * ```sql
+ * CREATE TABLE events_events (
+ *   event_id VARCHAR(36) PRIMARY KEY,
+ *   event_type VARCHAR(50) NOT NULL,
+ *   event_name VARCHAR(255) NOT NULL,
+ *   attributes VARIANT,
+ *   funnel VARCHAR(100),
+ *   step VARCHAR(50),
+ *   operation VARCHAR(100),
+ *   outcome VARCHAR(50),
+ *   value DECIMAL(18,2),
+ *   timestamp TIMESTAMP_NTZ NOT NULL,
+ *   created_at TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP()
+ * );
+ *
+ * CREATE INDEX idx_event_type ON events_events(event_type);
+ * CREATE INDEX idx_event_name ON events_events(event_name);
+ * CREATE INDEX idx_timestamp ON events_events(timestamp);
+ * ```
+ *
+ * Usage:
+ * ```typescript
+ * import { Events } from 'autotel/events';
+ * import { SnowflakeSubscriber } from './adapter-snowflake';
+ *
+ * const events = new Events('app', {
+ *   subscribers: [
+ *     new SnowflakeSubscriber({
+ *       account: 'xy12345.us-east-1',
+ *       username: process.env.SNOWFLAKE_USER!,
+ *       password: process.env.SNOWFLAKE_PASS!,
+ *       database: 'ANALYTICS',
+ *       schema: 'PUBLIC',
+ *       warehouse: 'COMPUTE_WH',
+ *       table: 'events_events'
+ *     })
+ *   ]
+ * });
+ *
+ * events.trackEvent('order.completed', { orderId: 'ord_123', amount: 99.99 });
+ * ```
+ */
+
+import {
+  EventSubscriber,
+  type EventPayload,
+} from '../src/event-subscriber-base';
+import snowflake from 'snowflake-sdk';
+
+export interface SnowflakeSubscriberConfig {
+  /** Snowflake account (e.g., 'xy12345.us-east-1') */
+  account: string;
+  /** Username */
+  username: string;
+  /** Password */
+  password: string;
+  /** Database name */
+  database: string;
+  /** Schema name (default: 'PUBLIC') */
+  schema?: string;
+  /** Warehouse name (default: 'COMPUTE_WH') */
+  warehouse?: string;
+  /** Table name (default: 'events_events') */
+  table?: string;
+  /** Enable/disable subscriber */
+  enabled?: boolean;
+  /** Batch size (default: 100) */
+  batchSize?: number;
+  /** Flush interval in ms (default: 10000) */
+  flushInterval?: number;
+}
+
+export class SnowflakeSubscriber extends EventSubscriber {
+  readonly name = 'SnowflakeSubscriber';
+  readonly version = '1.0.0';
+
+  private connection: snowflake.Connection;
+  private config: Required<SnowflakeSubscriberConfig>;
+  private buffer: EventPayload[] = [];
+  private flushIntervalHandle: NodeJS.Timeout | null = null;
+
+  constructor(config: SnowflakeSubscriberConfig) {
+    super();
+
+    // Set defaults
+    this.config = {
+      schema: 'PUBLIC',
+      warehouse: 'COMPUTE_WH',
+      table: 'events_events',
+      enabled: true,
+      batchSize: 100,
+      flushInterval: 10_000,
+      ...config,
+    };
+
+    this.enabled = this.config.enabled;
+
+    if (this.enabled) {
+      this.initializeConnection();
+      this.startFlushInterval();
+    }
+  }
+
+  private initializeConnection(): void {
+    this.connection = snowflake.createConnection({
+      account: this.config.account,
+      username: this.config.username,
+      password: this.config.password,
+      database: this.config.database,
+      schema: this.config.schema,
+      warehouse: this.config.warehouse,
+    });
+
+    // Connect asynchronously
+    this.connection.connect((err) => {
+      if (err) {
+        console.error('[SnowflakeSubscriber] Failed to connect:', err);
+        this.enabled = false;
+      } else {
+        console.log('[SnowflakeSubscriber] Connected successfully');
+      }
+    });
+  }
+
+  private startFlushInterval(): void {
+    this.flushIntervalHandle = setInterval(() => {
+      void this.flushBuffer();
+    }, this.config.flushInterval);
+  }
+
+  protected async sendToDestination(payload: EventPayload): Promise<void> {
+    this.buffer.push(payload);
+
+    // Auto-flush at batch size
+    if (this.buffer.length >= this.config.batchSize) {
+      await this.flushBuffer();
+    }
+  }
+
+  private async flushBuffer(): Promise<void> {
+    if (this.buffer.length === 0) return;
+
+    const batch = [...this.buffer];
+    this.buffer = [];
+
+    try {
+      await this.insertBatch(batch);
+    } catch (error) {
+      console.error('[SnowflakeSubscriber] Failed to flush batch:', error);
+      // Re-add to buffer for retry
+      this.buffer.unshift(...batch);
+    }
+  }
+
+  private async insertBatch(events: EventPayload[]): Promise<void> {
+    const sql = `
+      INSERT INTO ${this.config.table}
+      (event_id, event_type, event_name, attributes, funnel, step, operation, outcome, value, timestamp)
+      SELECT
+        column1 as event_id,
+        column2 as event_type,
+        column3 as event_name,
+        PARSE_JSON(column4) as attributes,
+        column5 as funnel,
+        column6 as step,
+        column7 as operation,
+        column8 as outcome,
+        column9 as value,
+        TO_TIMESTAMP_NTZ(column10) as timestamp
+      FROM VALUES ${events.map(() => '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').join(', ')}
+    `;
+
+    const binds = events.flatMap((event) => [
+      crypto.randomUUID(),
+      event.type,
+      event.name,
+      JSON.stringify(event.attributes || {}),
+      event.funnel || null,
+      event.step || null,
+      event.operation || null,
+      event.outcome || null,
+      event.value || null,
+      event.timestamp,
+    ]);
+
+    return new Promise((resolve, reject) => {
+      this.connection.execute({
+        sqlText: sql,
+        binds,
+        complete: (err, _stmt, _rows) => {
+          if (err) reject(err);
+          else resolve();
+        },
+      });
+    });
+  }
+
+  async shutdown(): Promise<void> {
+    // Clear flush interval
+    if (this.flushIntervalHandle) {
+      clearInterval(this.flushIntervalHandle);
+      this.flushIntervalHandle = null;
+    }
+
+    // Flush remaining events
+    await this.flushBuffer();
+
+    // Wait for pending requests
+    await super.shutdown();
+
+    // Close connection
+    if (this.connection) {
+      this.connection.destroy((err) => {
+        if (err) {
+          console.error('[SnowflakeSubscriber] Error closing connection:', err);
+        }
+      });
+    }
+  }
+}

package/package.json

@@ -0,0 +1,141 @@
+{
+  "name": "autotel-subscribers",
+  "version": "4.0.0",
+  "description": "Write Once, Observe Anywhere - Event subscribers for autotel (PostHog, Mixpanel, Amplitude, Segment)",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    },
+    "./posthog": {
+      "types": "./dist/posthog.d.ts",
+      "import": "./dist/posthog.js",
+      "require": "./dist/posthog.cjs"
+    },
+    "./mixpanel": {
+      "types": "./dist/mixpanel.d.ts",
+      "import": "./dist/mixpanel.js",
+      "require": "./dist/mixpanel.cjs"
+    },
+    "./segment": {
+      "types": "./dist/segment.d.ts",
+      "import": "./dist/segment.js",
+      "require": "./dist/segment.cjs"
+    },
+    "./amplitude": {
+      "types": "./dist/amplitude.d.ts",
+      "import": "./dist/amplitude.js",
+      "require": "./dist/amplitude.cjs"
+    },
+    "./webhook": {
+      "types": "./dist/webhook.d.ts",
+      "import": "./dist/webhook.js",
+      "require": "./dist/webhook.cjs"
+    },
+    "./slack": {
+      "types": "./dist/slack.d.ts",
+      "import": "./dist/slack.js",
+      "require": "./dist/slack.cjs"
+    },
+    "./factories": {
+      "types": "./dist/factories.d.ts",
+      "import": "./dist/factories.js",
+      "require": "./dist/factories.cjs"
+    },
+    "./middleware": {
+      "types": "./dist/middleware.d.ts",
+      "import": "./dist/middleware.js",
+      "require": "./dist/middleware.cjs"
+    },
+    "./testing": {
+      "types": "./dist/testing/index.d.ts",
+      "import": "./dist/testing/index.js",
+      "require": "./dist/testing/index.cjs"
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "examples",
+    "README.md"
+  ],
+  "keywords": [
+    "autotel",
+    "opentelemetry",
+    "analytics",
+    "posthog",
+    "mixpanel",
+    "amplitude",
+    "segment",
+    "webhook",
+    "adapters",
+    "business-metrics",
+    "product-analytics",
+    "telemetry",
+    "observability"
+  ],
+  "author": "Jag Reehal<jag@jagreehal.com> (https://jagreehal.com)",
+  "license": "MIT",
+  "peerDependencies": {
+    "autotel": "2.1.0"
+  },
+  "peerDependenciesMeta": {
+    "posthog-node": {
+      "optional": true
+    },
+    "mixpanel": {
+      "optional": true
+    },
+    "@segment/analytics-node": {
+      "optional": true
+    },
+    "@amplitude/analytics-node": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@amplitude/analytics-node": "^1.5.26",
+    "@arethetypeswrong/cli": "^0.18.2",
+    "@cloudflare/workers-types": "^4.20251121.0",
+    "@prisma/client": "^7.0.0",
+    "@segment/analytics-node": "^2.3.0",
+    "@types/node": "^24.10.1",
+    "@typescript-eslint/eslint-plugin": "^8.47.0",
+    "@typescript-eslint/parser": "^8.47.0",
+    "drizzle-orm": "^0.44.7",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-unicorn": "^62.0.0",
+    "mixpanel": "^0.19.1",
+    "mongoose": "^9.0.0",
+    "posthog-node": "^5.13.2",
+    "prettier": "^3.6.2",
+    "rimraf": "^6.1.2",
+    "tsup": "^8.5.1",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.47.0",
+    "vitest": "^4.0.13",
+    "autotel": "2.1.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jagreehal/autotel"
+  },
+  "bugs": {
+    "url": "https://github.com/jagreehal/autotel/issues"
+  },
+  "homepage": "https://github.com/jagreehal/autotel#readme",
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "lint": "npx eslint 'src/**/*.ts' 'examples/**/*.ts'",
+    "lint:fix": "npx eslint 'src/**/*.ts' 'examples/**/*.ts' --fix",
+    "type-check": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}