@dotdo/postgres 0.1.0

package/src/index.ts

@@ -0,0 +1,212 @@
+/**
+ * @module @dotdo/postgres
+ *
+ * PostgreSQL server for Cloudflare Workers/DOs with PGLite WASM and tiered storage.
+ *
+ * This package provides a complete PostgreSQL-compatible database solution
+ * optimized for Cloudflare's edge infrastructure:
+ *
+ * - **Worker Module**: PostgresDO Durable Object with HTTP/WebSocket APIs
+ * - **PGLite Module**: Tiered storage system (Cache/DO/R2) for PGLite
+ * - **Extensions Module**: PGLite extension management and pgvector support
+ * - **Routing Module**: Intelligent query routing based on CPU cost estimation
+ * - **Read-Only Module**: Security-focused read-only deployment mode
+ * - **Middleware Module**: Rate limiting with sliding window and token bucket algorithms
+ *
+ * @example
+ * ```typescript
+ * import {
+ *   PostgresDO,
+ *   createRoutes,
+ *   WebSocketHandler,
+ * } from '@dotdo/postgres'
+ *
+ * // Export the Durable Object
+ * export { PostgresDO }
+ *
+ * // Create Hono routes
+ * const app = new Hono()
+ * app.route('/api/sql', createRoutes(env))
+ *
+ * export default {
+ *   fetch: app.fetch,
+ * }
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Use tiered storage for PGLite
+ * import { createDOVFS, TieredVFS, CacheLayer } from '@dotdo/postgres/pglite'
+ *
+ * const vfs = createDOVFS(ctx.storage)
+ * const pglite = await PGlite.create({ vfs })
+ * ```
+ */
+
+// ============================================================================
+// Configuration Module - Centralized Constants
+// ============================================================================
+
+export * from './config'
+
+// ============================================================================
+// Worker Module - PostgresDO Durable Object
+// ============================================================================
+
+export {
+  createTenantId,
+  createPageNumber,
+  createMigrationVersion,
+  createQueryId,
+  createSnapshotId,
+  createBlockId,
+  createShardId,
+  createIngestId,
+  createReplicaId,
+  createBatchId,
+  isTenantId,
+  isPageNumber,
+  isMigrationVersion,
+  isQueryId,
+  isSnapshotId,
+  isBlockId,
+  isShardId,
+  isIngestId,
+  isReplicaId,
+  isBatchId,
+  unwrap,
+  unsafeTenantId,
+  unsafePageNumber,
+  unsafeMigrationVersion,
+  unsafeQueryId,
+  unsafeSnapshotId,
+  unsafeBlockId,
+  unsafeShardId,
+  unsafeIngestId,
+  unsafeReplicaId,
+  unsafeBatchId,
+  TENANT_ID_MAX_LENGTH,
+  TENANT_ID_PATTERN,
+  MAX_PAGE_NUMBER,
+  QUERY_ID_PATTERN,
+  SNAPSHOT_ID_PATTERN,
+  MAX_SNAPSHOT_ID,
+  BLOCK_ID_PATTERN,
+  SHARD_ID_PATTERN,
+  SHARD_ID_MAX_LENGTH,
+  INGEST_ID_PATTERN,
+  INGEST_ID_MAX_LENGTH,
+  REPLICA_ID_PATTERN,
+  REPLICA_ID_MAX_LENGTH,
+  BATCH_ID_PATTERN,
+  BATCH_ID_MAX_LENGTH,
+} from './types/branded'
+
+export type {
+  TenantId,
+  PageNumber,
+  MigrationVersion,
+  QueryId,
+  SnapshotId,
+  BlockId,
+  ShardId,
+  IngestId,
+  ReplicaId,
+  BatchId,
+  Branded,
+} from './types/branded'
+
+// ============================================================================
+// Type Utilities - Reusable type patterns
+// ============================================================================
+
+export {
+  // Utility functions
+  isDefined,
+  isString,
+  isNumber,
+  isBoolean,
+  isPlainObject,
+  isArray,
+  isFunction,
+  isPromise,
+  isError,
+  assertDefined,
+  assert,
+  assertNever,
+  success,
+  failure,
+  isSuccess,
+  isFailure,
+  // Web API helpers
+  toBodyInit,
+  toArrayBuffer,
+  isArrayBufferView,
+  isUint8Array,
+  // Storage helpers
+  narrowStorageValue,
+  assertStorageValue,
+  // Discriminated union helpers
+  createDiscriminatorGuard,
+  // Safe casting utilities
+  unsafeCast,
+  safeCast,
+  // Object property helpers
+  getProperty,
+  hasProperty,
+  hasProperties,
+} from './types/utilities'
+
+export type {
+  // Nullable types
+  Nullable,
+  Optional,
+  Maybe,
+  NonNullish,
+  // Constructor types
+  Constructor,
+  ConstructorWithArgs,
+  AbstractConstructor,
+  // Result types
+  Success,
+  Failure,
+  Result,
+  AsyncResult,
+  // Deep types
+  DeepPartial,
+  DeepRequired,
+  DeepReadonly,
+  DeepMutable,
+  // Async types
+  Awaited,
+  MaybePromise,
+  Awaitable,
+  AsyncFunction,
+  // Record utilities
+  StringRecord,
+  KeysOfType,
+  PickByType,
+  OmitByType,
+  RequireKeys,
+  OptionalKeys,
+  ValueOf,
+  // Function utilities
+  AnyFunction,
+  VoidFunction,
+  Thunk,
+  Predicate,
+  TypeGuard,
+  FirstParameter,
+  LastParameter,
+  // String utilities
+  Capitalize,
+  Uncapitalize,
+  Join,
+  // Tuple utilities
+  Head,
+  Tail,
+  Last,
+  Length,
+  // Discriminated union
+  DiscriminatedMessage,
+} from './types/utilities'

package/src/lineage/index.ts

@@ -0,0 +1,42 @@
+/**
+ * Data Lineage Tracking Module
+ * Task: postgres-zce - Data lineage tracking
+ *
+ * Comprehensive data lineage tracking for PostgresDO with WAL and Iceberg integration.
+ */
+
+// Export types
+export type {
+  LineageNodeType,
+  LineageOperationType,
+  LineageJoinType,
+  LineageAggregateFunction,
+  ColumnLineage,
+  JoinDependency,
+  AggregateDependency,
+  LineageNode,
+  LineageEdge,
+  LineageGraph,
+  RowLineage,
+  LineageQueryResult,
+  LineageTrackerConfig,
+  LineageStats,
+  LineageVisualizationGraph,
+  LineageVisualizationNode,
+  LineageVisualizationEdge,
+  IcebergLineageMetadata,
+  LineageParseResult,
+} from './types'
+
+// Export tracker
+export { LineageTracker, createLineageTracker, parseLineage } from './tracker'
+
+// Export integration
+export {
+  LineageIntegration,
+  createLineageIntegration,
+  DOLineageCatalog,
+  createDOLineageCatalog,
+} from './integration'
+
+export type { LineageIntegrationConfig, LineageCatalogStorage } from './integration'

package/src/lineage/integration.ts

@@ -0,0 +1,334 @@
+/**
+ * Lineage Integration with WAL and Iceberg
+ * Task: postgres-zce - Data lineage tracking
+ *
+ * Integrates lineage tracking with WAL capture and Iceberg storage,
+ * enabling automatic lineage collection and persistence.
+ */
+
+import type { WALManager, SerializedWALEntry } from '../worker/wal'
+import type { IcebergTableMetadata } from '../iceberg/types'
+import type { LineageTrackerConfig, IcebergLineageMetadata } from './types'
+import { LineageTracker } from './tracker'
+
+/**
+ * Lineage integration configuration
+ */
+export interface LineageIntegrationConfig {
+  /** Lineage tracker configuration */
+  lineageConfig?: LineageTrackerConfig
+  /** Enable automatic lineage tracking from WAL */
+  autoTrackWAL?: boolean
+  /** Enable automatic persistence to Iceberg */
+  autoSyncIceberg?: boolean
+  /** Sync interval in milliseconds (default: 60000 - 1 minute) */
+  syncIntervalMs?: number
+}
+
+/**
+ * Default configuration
+ */
+const DEFAULT_INTEGRATION_CONFIG: Required<LineageIntegrationConfig> = {
+  lineageConfig: {},
+  autoTrackWAL: true,
+  autoSyncIceberg: true,
+  syncIntervalMs: 60000,
+}
+
+/**
+ * Lineage Integration Manager
+ *
+ * Automatically tracks lineage from WAL entries and syncs to Iceberg metadata.
+ */
+export class LineageIntegration {
+  private tracker: LineageTracker
+  private config: Required<LineageIntegrationConfig>
+  /** @internal Reserved for future WAL manager reference */
+  private _walManager: WALManager | null = null
+  private syncInterval: number | null = null
+  private unsubscribeWAL: (() => void) | null = null
+
+  constructor(config: LineageIntegrationConfig = {}) {
+    this.config = { ...DEFAULT_INTEGRATION_CONFIG, ...config }
+    this.tracker = new LineageTracker(this.config.lineageConfig)
+    void this._walManager // Reserved for future WAL manager reference
+  }
+
+  /**
+   * Get the lineage tracker
+   */
+  getTracker(): LineageTracker {
+    return this.tracker
+  }
+
+  /**
+   * Attach to a WAL manager for automatic tracking
+   */
+  attachWALManager(walManager: WALManager): void {
+    // Detach from previous manager if any
+    this.detachWALManager()
+
+    this._walManager = walManager
+
+    if (this.config.autoTrackWAL) {
+      // Subscribe to WAL entries
+      this.unsubscribeWAL = walManager.onEntry((entry) => {
+        // Convert WALEntry to SerializedWALEntry
+        const serializedEntry: SerializedWALEntry = {
+          lsn: entry.lsn.toString(),
+          operation: entry.operation,
+          schema: entry.schema,
+          table: entry.table,
+          primaryKey: entry.primaryKey,
+          newRow: entry.newRow,
+          oldRow: entry.oldRow,
+          changedColumns: entry.changedColumns,
+          transactionId: entry.transactionId,
+          timestamp: entry.timestamp,
+          sql: entry.sql,
+        }
+
+        this.tracker.trackWALEntry(serializedEntry)
+      })
+    }
+  }
+
+  /**
+   * Detach from the current WAL manager
+   */
+  detachWALManager(): void {
+    if (this.unsubscribeWAL) {
+      this.unsubscribeWAL()
+      this.unsubscribeWAL = null
+    }
+    this._walManager = null
+  }
+
+  /**
+   * Start automatic sync to Iceberg
+   */
+  startAutoSync(): void {
+    if (this.syncInterval !== null) {
+      return // Already running
+    }
+
+    if (!this.config.autoSyncIceberg) {
+      return
+    }
+
+    // Use setInterval in a way that's compatible with both Node and browser
+    this.syncInterval = setInterval(() => {
+      void this.syncToIceberg().catch((error) => {
+        console.error('Lineage auto-sync error:', error)
+      })
+    }, this.config.syncIntervalMs) as unknown as number
+  }
+
+  /**
+   * Stop automatic sync to Iceberg
+   */
+  stopAutoSync(): void {
+    if (this.syncInterval !== null) {
+      clearInterval(this.syncInterval)
+      this.syncInterval = null
+    }
+  }
+
+  /**
+   * Manually sync lineage to Iceberg metadata
+   *
+   * This updates the Iceberg table properties with lineage information.
+   * Note: This requires access to the Iceberg catalog to update metadata.
+   */
+  async syncToIceberg(): Promise<void> {
+    // Get all tracked tables
+    const stats = this.tracker.getStats()
+    if (stats.totalNodes === 0) {
+      return // Nothing to sync
+    }
+
+    // Export metadata for each table
+    const graph = this.tracker.getGraph()
+    for (const [_nodeId, node] of graph.nodes) {
+      if (node.type !== 'table') continue
+
+      // Export lineage metadata
+      const metadata = this.tracker.exportIcebergMetadata(node.schema, node.name)
+
+      // Store in a lineage catalog (implementation would depend on storage backend)
+      await this.storeLineageMetadata(node.schema, node.name, metadata)
+    }
+  }
+
+  /**
+   * Store lineage metadata
+   *
+   * This is a placeholder - actual implementation would store in:
+   * 1. Iceberg table properties (preferred)
+   * 2. Separate lineage catalog table
+   * 3. DO storage
+   */
+  private async storeLineageMetadata(
+    schema: string,
+    table: string,
+    metadata: IcebergLineageMetadata
+  ): Promise<void> {
+    // For now, just log - actual implementation would update Iceberg metadata
+    console.log(`Storing lineage for ${schema}.${table}:`, metadata)
+
+    // In a real implementation, this would:
+    // 1. Load the Iceberg table metadata
+    // 2. Update the properties section
+    // 3. Write back the metadata
+    // 4. Or store in a separate lineage catalog table
+  }
+
+  /**
+   * Load lineage metadata from Iceberg
+   */
+  async loadFromIceberg(schema: string, table: string): Promise<void> {
+    // Placeholder - would load from Iceberg table properties
+    // For now, we'll just skip
+    console.log(`Loading lineage for ${schema}.${table}`)
+  }
+
+  /**
+   * Update Iceberg table metadata with lineage
+   */
+  updateIcebergMetadata(metadata: IcebergTableMetadata, schema: string, table: string): IcebergTableMetadata {
+    const lineageMetadata = this.tracker.exportIcebergMetadata(schema, table)
+
+    // Build lineage properties, only including defined values
+    const lineageProperties: Record<string, string> = {
+      'lineage.version': lineageMetadata['lineage.version'],
+      'lineage.sources': lineageMetadata['lineage.sources'],
+      'lineage.operations': lineageMetadata['lineage.operations'],
+      'lineage.updated-at': lineageMetadata['lineage.updated-at'],
+    }
+
+    // Only add graph property if it's defined
+    if (lineageMetadata['lineage.graph'] !== undefined) {
+      lineageProperties['lineage.graph'] = lineageMetadata['lineage.graph']
+    }
+
+    // Merge lineage metadata into Iceberg properties
+    return {
+      ...metadata,
+      properties: {
+        ...metadata.properties,
+        ...lineageProperties,
+      },
+    }
+  }
+
+  /**
+   * Extract lineage from Iceberg table metadata
+   */
+  extractFromIcebergMetadata(metadata: IcebergTableMetadata, schema: string, table: string): void {
+    const properties = metadata.properties || {}
+
+    // Check if lineage metadata exists
+    if (properties['lineage.version']) {
+      const lineageMetadata: IcebergLineageMetadata = {
+        'lineage.version': properties['lineage.version'],
+        'lineage.sources': properties['lineage.sources'] || '',
+        'lineage.operations': properties['lineage.operations'] || '',
+        'lineage.updated-at': properties['lineage.updated-at'] || '',
+        'lineage.graph': properties['lineage.graph'],
+      }
+
+      this.tracker.importIcebergMetadata(schema, table, lineageMetadata)
+    }
+  }
+
+  /**
+   * Get lineage statistics
+   */
+  getStats() {
+    return this.tracker.getStats()
+  }
+
+  /**
+   * Shutdown the integration
+   */
+  shutdown(): void {
+    this.stopAutoSync()
+    this.detachWALManager()
+  }
+}
+
+/**
+ * Create a lineage integration instance
+ */
+export function createLineageIntegration(config?: LineageIntegrationConfig): LineageIntegration {
+  return new LineageIntegration(config)
+}
+
+/**
+ * Storage interface for lineage catalog
+ */
+export interface LineageCatalogStorage {
+  /** Store lineage metadata for a table */
+  storeLineage(schema: string, table: string, metadata: IcebergLineageMetadata): Promise<void>
+
+  /** Load lineage metadata for a table */
+  loadLineage(schema: string, table: string): Promise<IcebergLineageMetadata | null>
+
+  /** List all tables with lineage metadata */
+  listTrackedTables(): Promise<{ schema: string; table: string }[]>
+
+  /** Delete lineage metadata for a table */
+  deleteLineage(schema: string, table: string): Promise<void>
+}
+
+/**
+ * Durable Object Storage implementation for lineage catalog
+ */
+export class DOLineageCatalog implements LineageCatalogStorage {
+  private state: DurableObjectState
+  private prefix: string
+
+  constructor(state: DurableObjectState, prefix = 'lineage:') {
+    this.state = state
+    this.prefix = prefix
+  }
+
+  async storeLineage(schema: string, table: string, metadata: IcebergLineageMetadata): Promise<void> {
+    const key = `${this.prefix}${schema}.${table}`
+    await this.state.storage.put(key, metadata)
+  }
+
+  async loadLineage(schema: string, table: string): Promise<IcebergLineageMetadata | null> {
+    const key = `${this.prefix}${schema}.${table}`
+    return (await this.state.storage.get<IcebergLineageMetadata>(key)) || null
+  }
+
+  async listTrackedTables(): Promise<{ schema: string; table: string }[]> {
+    const keys = await this.state.storage.list<IcebergLineageMetadata>({
+      prefix: this.prefix,
+    })
+
+    const tables: { schema: string; table: string }[] = []
+    for (const [key] of keys) {
+      const tableId = key.slice(this.prefix.length)
+      const [schema, table] = tableId.split('.')
+      if (schema && table) {
+        tables.push({ schema, table })
+      }
+    }
+
+    return tables
+  }
+
+  async deleteLineage(schema: string, table: string): Promise<void> {
+    const key = `${this.prefix}${schema}.${table}`
+    await this.state.storage.delete(key)
+  }
+}
+
+/**
+ * Create a DO lineage catalog
+ */
+export function createDOLineageCatalog(state: DurableObjectState): DOLineageCatalog {
+  return new DOLineageCatalog(state)
+}