@dotdo/postgres 0.1.0

package/src/cdc/types.ts

@@ -0,0 +1,346 @@
+/**
+ * CDC (Change Data Capture) Type Definitions
+ * Task: postgres-lp00 - CDC/Change Data Capture streaming
+ *
+ * Core types for CDC streaming, filtering, and transport.
+ */
+
+// ============================================================================
+// Core CDC Types
+// ============================================================================
+
+/**
+ * CDC operation types matching PostgreSQL logical replication
+ */
+export type CDCOperation = 'INSERT' | 'UPDATE' | 'DELETE' | 'TRUNCATE'
+
+/**
+ * Resume token for exactly-once delivery
+ * Contains LSN and metadata for stream resumption
+ */
+export interface ResumeToken {
+  /** Encoded token data */
+  _data: string
+  /** Decoded token info (cached) */
+  _decoded?: {
+    lsn: string
+    timestamp: number
+    shardId?: string | undefined
+    sequence?: number | undefined
+    checksum?: string | undefined
+  } | undefined
+}
+
+/**
+ * Change event representing a database change
+ */
+export interface ChangeEvent<T = Record<string, unknown>> {
+  /** Unique event identifier */
+  id: string
+  /** PostgreSQL Log Sequence Number */
+  lsn: string
+  /** Type of database operation */
+  operation: CDCOperation
+  /** Database schema name */
+  schema: string
+  /** Table name */
+  table: string
+  /** New row data (for INSERT and UPDATE) */
+  newRow?: T
+  /** Old row data (for UPDATE and DELETE) */
+  oldRow?: T
+  /** Event timestamp (unix milliseconds) */
+  timestamp: number
+  /** Resume token for this event */
+  resumeToken: ResumeToken
+  /** Transaction ID (optional) */
+  transactionId?: string
+  /** Whether this is the last event in a transaction */
+  isLastInTransaction?: boolean
+  /** Changed columns (for UPDATE) */
+  changedColumns?: string[]
+}
+
+// ============================================================================
+// Filter Types
+// ============================================================================
+
+/**
+ * Filter configuration for change streams
+ */
+export interface ChangeStreamFilter {
+  /** Filter by operation types */
+  operations?: CDCOperation[]
+  /** Filter by exact table names */
+  tables?: string[]
+  /** Filter by glob patterns (e.g., 'user*', '*_log') */
+  tablePatterns?: string[]
+  /** Filter by schema names */
+  schemas?: string[]
+  /** SQL-like WHERE clause for row filtering */
+  where?: string
+  /** Projection to include/exclude columns */
+  projection?: {
+    include?: string[]
+    exclude?: string[]
+  }
+}
+
+/**
+ * Pipeline stage for transforming events
+ * Inspired by MongoDB change stream pipelines
+ */
+export interface ChangeStreamPipelineStage {
+  /** Match stage - filter events */
+  $match?: ChangeStreamFilter
+  /** Project stage - include/exclude fields */
+  $project?: Record<string, boolean>
+  /** Add fields stage - add computed fields */
+  $addFields?: Record<string, unknown>
+  /** Unset stage - remove fields */
+  $unset?: string[]
+}
+
+// ============================================================================
+// Subscription Types
+// ============================================================================
+
+/**
+ * Subscription state
+ */
+export type SubscriptionState = 'pending' | 'active' | 'paused' | 'closed' | 'error'
+
+/**
+ * Subscription configuration
+ */
+export interface SubscriptionConfig {
+  /** Unique subscription identifier */
+  id: string
+  /** Tables to subscribe to */
+  tables: string[]
+  /** Optional filter */
+  filter?: ChangeStreamFilter
+  /** Pipeline stages */
+  pipeline?: ChangeStreamPipelineStage[]
+  /** Resume from this token */
+  resumeAfter?: ResumeToken
+  /** Maximum batch size */
+  batchSize?: number
+  /** Heartbeat interval in milliseconds */
+  heartbeatInterval?: number
+}
+
+/**
+ * CDC subscription interface
+ */
+export interface CDCSubscription<T = Record<string, unknown>> {
+  /** Subscription ID */
+  readonly id: string
+  /** Tables being watched */
+  readonly tables: string[]
+  /** Current state */
+  readonly state: SubscriptionState
+  /** Current resume token */
+  readonly resumeToken: ResumeToken | null
+  /** Pause the subscription */
+  pause(): Promise<void>
+  /** Resume the subscription */
+  resume(): Promise<void>
+  /** Close the subscription */
+  close(): Promise<void>
+  /** Get next event */
+  next(): Promise<ChangeEvent<T> | null>
+  /** Try to get next event (non-blocking) */
+  tryNext(): ChangeEvent<T> | null
+  /** Async iterator support */
+  [Symbol.asyncIterator](): AsyncIterableIterator<ChangeEvent<T>>
+}
+
+/**
+ * CDC stream manager interface
+ */
+export interface CDCStreamManager {
+  /** Create a new subscription */
+  subscribe(config: SubscriptionConfig): Promise<CDCSubscription>
+  /** Unsubscribe by ID */
+  unsubscribe(subscriptionId: string): Promise<void>
+  /** Get all active subscriptions */
+  getSubscriptions(): CDCSubscription[]
+  /** Get a subscription by ID */
+  getSubscription(id: string): CDCSubscription | undefined
+  /** Close all subscriptions */
+  closeAll(): Promise<void>
+}
+
+/**
+ * CDC stream statistics
+ */
+export interface CDCStreamStats {
+  /** Current state */
+  state: SubscriptionState
+  /** Total events received */
+  eventsReceived: number
+  /** Total events emitted (after filtering) */
+  eventsEmitted: number
+  /** Events currently queued */
+  queuedEvents: number
+  /** Time of last event (unix ms) */
+  lastEventTime?: number
+  /** Current lag in milliseconds */
+  lagMs?: number
+  /** Bytes received */
+  bytesReceived?: number
+}
+
+// ============================================================================
+// Transport Types
+// ============================================================================
+
+/**
+ * Base transport options
+ */
+export interface CDCTransportOptions {
+  /** Resume from this token */
+  resumeToken?: ResumeToken
+  /** Custom headers */
+  headers?: Record<string, string>
+  /** API key for authentication */
+  apiKey?: string
+  /** Database name */
+  database?: string
+}
+
+/**
+ * SSE transport configuration
+ */
+export interface SSETransportOptions extends CDCTransportOptions {
+  /** SSE endpoint URL */
+  url: string
+  /** Use fetch instead of EventSource */
+  useFetch?: boolean
+  /** Connection timeout in ms */
+  connectTimeout?: number
+  /** Reconnection configuration */
+  reconnect?: {
+    /** Enable automatic reconnection */
+    enabled?: boolean
+    /** Maximum reconnection attempts */
+    maxAttempts?: number
+    /** Initial delay in ms */
+    initialDelay?: number
+    /** Maximum delay in ms */
+    maxDelay?: number
+    /** Backoff multiplier */
+    backoffMultiplier?: number
+  }
+}
+
+/**
+ * WebSocket transport configuration
+ */
+export interface WebSocketTransportOptions extends CDCTransportOptions {
+  /** WebSocket endpoint URL */
+  url: string
+  /** Connection timeout in ms */
+  connectTimeout?: number
+  /** Ping interval in ms */
+  pingInterval?: number
+  /** Custom WebSocket implementation */
+  WebSocket?: typeof WebSocket
+  /** Reconnection configuration */
+  reconnect?: {
+    /** Enable automatic reconnection */
+    enabled?: boolean
+    /** Maximum reconnection attempts */
+    maxAttempts?: number
+    /** Initial delay in ms */
+    initialDelay?: number
+    /** Maximum delay in ms */
+    maxDelay?: number
+    /** Backoff multiplier */
+    backoffMultiplier?: number
+  }
+}
+
+/**
+ * CDC transport interface
+ */
+export interface CDCTransport {
+  /** Connect to the CDC stream */
+  connect(options?: CDCTransportOptions): Promise<void>
+  /** Disconnect from the CDC stream */
+  disconnect(): Promise<void>
+  /** Subscribe to tables */
+  subscribe(config: SubscriptionConfig): Promise<void>
+  /** Unsubscribe from a subscription */
+  unsubscribe(subscriptionId: string): Promise<void>
+  /** Acknowledge an event (commit LSN) */
+  ack(lsn: string): Promise<void>
+  /** Check if connected */
+  isConnected(): boolean
+  /** Add event listener */
+  on(event: 'change', handler: (event: ChangeEvent) => void): void
+  on(event: 'error', handler: (error: Error) => void): void
+  on(event: 'close', handler: () => void): void
+  on(event: 'reconnect', handler: () => void): void
+  /** Remove event listener */
+  off(event: string, handler: (...args: unknown[]) => void): void
+}
+
+// ============================================================================
+// Electric Integration Types
+// ============================================================================
+
+/**
+ * Electric bridge configuration
+ */
+export interface ElectricBridgeConfig {
+  /** Electric sync server URL */
+  syncUrl: string
+  /** Database connection URL */
+  databaseUrl?: string
+  /** API key */
+  apiKey?: string
+  /** Shapes to sync */
+  shapes: ElectricShapeDefinition[]
+}
+
+/**
+ * Electric shape definition (subset of data to sync)
+ */
+export interface ElectricShapeDefinition {
+  /** Table name */
+  table: string
+  /** Schema name (default: public) */
+  schema?: string
+  /** Where clause for filtering */
+  where?: string
+  /** Columns to include */
+  columns?: string[]
+  /** Related tables to include */
+  include?: ElectricShapeDefinition[]
+}
+
+// ============================================================================
+// Change Stream Options
+// ============================================================================
+
+/**
+ * Options for creating a change stream
+ */
+export interface ChangeStreamOptions {
+  /** Resume from this token */
+  resumeAfter?: ResumeToken | string
+  /** Filter configuration */
+  filter?: ChangeStreamFilter
+  /** Pipeline stages */
+  pipeline?: ChangeStreamPipelineStage[]
+  /** Maximum batch size */
+  batchSize?: number
+  /** Buffer size (max events to queue) */
+  bufferSize?: number
+  /** Transport to use (default: SSE) */
+  transport?: 'sse' | 'websocket'
+  /** Transport configuration */
+  transportConfig?: SSETransportOptions | WebSocketTransportOptions
+}

package/src/config/index.ts

@@ -0,0 +1,25 @@
+/**
+ * Centralized Configuration Constants
+ *
+ * This module consolidates related constants from across the packages/postgres
+ * codebase into coherent groups. Constants are organized by domain:
+ *
+ * - **Memory**: Worker memory limits and pressure thresholds
+ * - **Streaming**: Stream buffer sizes and water marks
+ * - **Storage**: Tiered storage configuration defaults
+ * - **Timeouts**: Standard timeout values for various operations
+ *
+ * ## Design Principles
+ *
+ * 1. **Single Source of Truth**: Each constant is defined once and imported elsewhere
+ * 2. **Documented Defaults**: Each constant includes rationale for its value
+ * 3. **Grouped by Domain**: Related constants are co-located for discoverability
+ * 4. **Cloudflare Workers Aware**: Defaults respect 128MB Worker memory limit
+ *
+ * @module config
+ */
+
+export * from './memory'
+export * from './streaming'
+export * from './storage'
+export * from './timeouts'

package/src/config/memory.ts

@@ -0,0 +1,177 @@
+/**
+ * Memory Configuration Constants
+ *
+ * Centralized memory-related constants for Cloudflare Workers environment.
+ * These constants define the memory limits, pressure thresholds, and WASM
+ * characteristics used throughout the postgres package.
+ *
+ * ## Cloudflare Workers Memory Budget
+ *
+ * ```
+ * 128 MB (Worker limit)
+ * - 15 MB (static: WASM + data + JS)
+ * - 16 MB (shared_buffers)
+ * - 16 MB (WAL + buffers)
+ * - 10 MB (PostgreSQL catalog/metadata)
+ * = ~71 MB available for queries, results, and runtime
+ * ```
+ *
+ * @module config/memory
+ */
+
+// =============================================================================
+// Worker Memory Limits
+// =============================================================================
+
+/**
+ * Cloudflare Worker memory limit in bytes (128MB).
+ *
+ * This is the hard limit enforced by the Workers runtime. Exceeding this
+ * causes the isolate to be terminated with an OOM error.
+ *
+ * @see https://developers.cloudflare.com/workers/platform/limits/
+ */
+export const WORKER_MEMORY_LIMIT_BYTES = 128 * 1024 * 1024
+
+/**
+ * WASM memory page size in bytes (64KB).
+ *
+ * WebAssembly allocates memory in 64KB pages. Used for heap size calculations.
+ *
+ * @see https://webassembly.github.io/spec/core/exec/runtime.html#page-size
+ */
+export const WASM_PAGE_SIZE_BYTES = 64 * 1024
+
+/**
+ * Estimated PGLite baseline memory footprint (55MB).
+ *
+ * Typical memory usage after PGLite initialization:
+ * - ~15MB static (WASM binary + data bundle + JS runtime)
+ * - ~40MB PostgreSQL runtime (shared_buffers, WAL, catalog)
+ */
+export const PGLITE_BASELINE_BYTES = (15 + 40) * 1024 * 1024
+
+// =============================================================================
+// Memory Pressure Thresholds (Percentages)
+// =============================================================================
+
+/**
+ * Default warning threshold as percentage of memory limit (70%).
+ *
+ * At this level, the system begins taking protective actions:
+ * - Suggests cache reduction
+ * - May trigger GC hints
+ * - Logs warnings
+ */
+export const DEFAULT_WARNING_THRESHOLD_PERCENT = 70
+
+/**
+ * Default critical threshold as percentage of memory limit (85%).
+ *
+ * At this level, aggressive protective actions are taken:
+ * - Pauses new queries
+ * - Triggers aggressive GC
+ * - May unload extensions
+ * - Disables non-essential features
+ */
+export const DEFAULT_CRITICAL_THRESHOLD_PERCENT = 85
+
+/**
+ * Default OOM threshold as percentage of memory limit (95%).
+ *
+ * At this level, emergency actions are taken:
+ * - Rejects new queries
+ * - Clears all caches
+ * - May trigger DO restart recommendation
+ */
+export const DEFAULT_OOM_THRESHOLD_PERCENT = 95
+
+// =============================================================================
+// Memory Pressure Thresholds (Ratios for observability/memory-metrics.ts)
+// =============================================================================
+
+/**
+ * Default warning threshold as ratio (0.7 = 70%).
+ *
+ * Used by MemoryMetricsCollector for compatibility with ratio-based APIs.
+ */
+export const DEFAULT_WARNING_THRESHOLD_RATIO = 0.7
+
+/**
+ * Default critical threshold as ratio (0.9 = 90%).
+ *
+ * Note: This is higher than the percentage-based threshold because
+ * the observability module uses a more conservative definition.
+ */
+export const DEFAULT_CRITICAL_THRESHOLD_RATIO = 0.9
+
+// =============================================================================
+// Memory Spike Detection
+// =============================================================================
+
+/**
+ * Threshold for detecting memory spikes (10MB).
+ *
+ * A sudden increase of this magnitude between samples triggers a spike event.
+ * Used to detect potential memory leaks or unexpectedly large allocations.
+ */
+export const MEMORY_SPIKE_THRESHOLD_BYTES = 10 * 1024 * 1024
+
+// =============================================================================
+// Index Memory Management
+// =============================================================================
+
+/**
+ * Estimated bytes per storage index entry (120 bytes).
+ *
+ * Breakdown:
+ * - Map entry overhead: ~40 bytes
+ * - StorageIndexEntry object: ~80 bytes (8 fields x 8-10 bytes each)
+ *
+ * Used for memory budgeting of the tiered storage index.
+ */
+export const INDEX_ENTRY_MEMORY_OVERHEAD_BYTES = 120
+
+/**
+ * Map internal structure overhead estimate (10%).
+ *
+ * JavaScript Maps have additional overhead for internal hash tables
+ * and bucket structures beyond the per-entry cost.
+ */
+export const MAP_OVERHEAD_ESTIMATE_PERCENT = 0.1
+
+/**
+ * Default memory warning threshold for storage index (80%).
+ *
+ * Warns when index memory usage reaches this percentage of the configured
+ * maximum to allow proactive eviction before hitting hard limits.
+ */
+export const DEFAULT_INDEX_MEMORY_WARNING_THRESHOLD = 0.8
+
+/**
+ * Target percentage of max memory when evicting due to memory pressure (80%).
+ *
+ * Eviction continues until usage drops to this percentage of the limit
+ * to prevent immediate re-eviction on the next write.
+ */
+export const INDEX_MEMORY_EVICTION_TARGET = 0.8
+
+// =============================================================================
+// Memory History Tracking
+// =============================================================================
+
+/**
+ * Default number of metrics samples to retain in history.
+ *
+ * Used for trend analysis, growth rate calculation, and OOM prediction.
+ * Higher values provide better trend data but use more memory.
+ */
+export const DEFAULT_METRICS_HISTORY_SIZE = 100
+
+/**
+ * Average key length assumption for memory estimation (20 characters).
+ *
+ * Used when calculating memory requirements for index capacity planning
+ * when actual key lengths are unknown.
+ */
+export const DEFAULT_AVG_KEY_LENGTH_CHARS = 20