@dotdo/postgres 0.1.0

package/src/iceberg/schema.ts

@@ -0,0 +1,616 @@
+/**
+ * Iceberg Schema Management
+ * Task: postgres-6qa - Apache Iceberg table format support
+ *
+ * Provides schema definition and evolution support for WAL Iceberg tables.
+ */
+
+import type {
+  IcebergSchema,
+  IcebergField,
+  IcebergPartitionSpec,
+  IcebergSortOrder,
+  IcebergType,
+} from './types'
+
+/**
+ * Field IDs for the WAL schema (stable across evolution)
+ */
+export const WAL_FIELD_IDS = {
+  DO_ID: 1,
+  LSN: 2,
+  TIMESTAMP: 3,
+  OPERATION: 4,
+  SCHEMA_NAME: 5,
+  TABLE_NAME: 6,
+  PRIMARY_KEY: 7,
+  NEW_ROW: 8,
+  OLD_ROW: 9,
+  CHANGED_COLUMNS: 10,
+  TRANSACTION_ID: 11,
+  SQL: 12,
+  // Reserved for future fields: 13-99
+  // Per-table row data starts at 100
+} as const
+
+/**
+ * Create the base WAL schema for Iceberg
+ *
+ * This schema captures all WAL entries from PostgresDO instances.
+ */
+export function createWALSchema(schemaId = 0): IcebergSchema {
+  return {
+    type: 'struct',
+    'schema-id': schemaId,
+    'identifier-field-ids': [WAL_FIELD_IDS.DO_ID, WAL_FIELD_IDS.LSN],
+    fields: [
+      {
+        id: WAL_FIELD_IDS.DO_ID,
+        name: 'do_id',
+        required: true,
+        type: 'string',
+        doc: 'Durable Object identifier (partition key)',
+      },
+      {
+        id: WAL_FIELD_IDS.LSN,
+        name: 'lsn',
+        required: true,
+        type: 'string',
+        doc: 'Log Sequence Number (unique within DO)',
+      },
+      {
+        id: WAL_FIELD_IDS.TIMESTAMP,
+        name: 'timestamp',
+        required: true,
+        type: 'timestamptz',
+        doc: 'When the change occurred (sort key, hidden partition)',
+      },
+      {
+        id: WAL_FIELD_IDS.OPERATION,
+        name: 'operation',
+        required: true,
+        type: 'string',
+        doc: 'Operation type: INSERT, UPDATE, DELETE',
+      },
+      {
+        id: WAL_FIELD_IDS.SCHEMA_NAME,
+        name: 'schema_name',
+        required: true,
+        type: 'string',
+        doc: 'PostgreSQL schema name',
+      },
+      {
+        id: WAL_FIELD_IDS.TABLE_NAME,
+        name: 'table_name',
+        required: true,
+        type: 'string',
+        doc: 'PostgreSQL table name',
+      },
+      {
+        id: WAL_FIELD_IDS.PRIMARY_KEY,
+        name: 'primary_key',
+        required: false,
+        type: 'string',
+        doc: 'JSON-encoded primary key values',
+      },
+      {
+        id: WAL_FIELD_IDS.NEW_ROW,
+        name: 'new_row',
+        required: false,
+        type: 'string',
+        doc: 'JSON-encoded new row data (INSERT/UPDATE)',
+      },
+      {
+        id: WAL_FIELD_IDS.OLD_ROW,
+        name: 'old_row',
+        required: false,
+        type: 'string',
+        doc: 'JSON-encoded old row data (UPDATE/DELETE)',
+      },
+      {
+        id: WAL_FIELD_IDS.CHANGED_COLUMNS,
+        name: 'changed_columns',
+        required: false,
+        type: 'string',
+        doc: 'JSON-encoded list of changed columns (delta mode)',
+      },
+      {
+        id: WAL_FIELD_IDS.TRANSACTION_ID,
+        name: 'transaction_id',
+        required: false,
+        type: 'string',
+        doc: 'Transaction identifier (if within transaction)',
+      },
+      {
+        id: WAL_FIELD_IDS.SQL,
+        name: 'sql',
+        required: false,
+        type: 'string',
+        doc: 'Original SQL statement (debug mode only)',
+      },
+    ],
+  }
+}
+
+/**
+ * Create the default partition spec for WAL data
+ *
+ * Partitions by:
+ * 1. DO ID (identity) - enables efficient filtering by DO
+ * 2. Timestamp (day) - enables efficient time travel queries
+ */
+export function createWALPartitionSpec(specId = 0): IcebergPartitionSpec {
+  return {
+    'spec-id': specId,
+    fields: [
+      {
+        'source-id': WAL_FIELD_IDS.DO_ID,
+        'field-id': 1000,
+        name: 'do_id_partition',
+        transform: 'identity',
+      },
+      {
+        'source-id': WAL_FIELD_IDS.TIMESTAMP,
+        'field-id': 1001,
+        name: 'timestamp_day',
+        transform: 'day',
+      },
+    ],
+  }
+}
+
+/**
+ * Create an hourly partition spec for high-volume scenarios
+ */
+export function createWALPartitionSpecHourly(specId = 1): IcebergPartitionSpec {
+  return {
+    'spec-id': specId,
+    fields: [
+      {
+        'source-id': WAL_FIELD_IDS.DO_ID,
+        'field-id': 1000,
+        name: 'do_id_partition',
+        transform: 'identity',
+      },
+      {
+        'source-id': WAL_FIELD_IDS.TIMESTAMP,
+        'field-id': 1002,
+        name: 'timestamp_hour',
+        transform: 'hour',
+      },
+    ],
+  }
+}
+
+/**
+ * Create the default sort order for WAL data
+ *
+ * Sorts by timestamp descending for efficient time travel queries.
+ */
+export function createWALSortOrder(orderId = 0): IcebergSortOrder {
+  return {
+    'order-id': orderId,
+    fields: [
+      {
+        'source-id': WAL_FIELD_IDS.TIMESTAMP,
+        transform: 'identity',
+        direction: 'desc',
+        'null-order': 'nulls-last',
+      },
+      {
+        'source-id': WAL_FIELD_IDS.LSN,
+        transform: 'identity',
+        direction: 'desc',
+        'null-order': 'nulls-last',
+      },
+    ],
+  }
+}
+
+/**
+ * Schema evolution operation types
+ */
+export type SchemaEvolutionOp =
+  | { type: 'add-column'; field: IcebergField; parentId?: number }
+  | { type: 'drop-column'; fieldId: number }
+  | { type: 'rename-column'; fieldId: number; newName: string }
+  | { type: 'update-column-doc'; fieldId: number; newDoc: string }
+  | { type: 'make-optional'; fieldId: number }
+  | { type: 'make-required'; fieldId: number }
+  | { type: 'update-column-type'; fieldId: number; newType: IcebergType }
+
+/**
+ * Schema evolution result
+ */
+export interface SchemaEvolutionResult {
+  success: boolean
+  newSchema: IcebergSchema
+  changes: string[]
+  errors: string[]
+}
+
+/**
+ * Schema evolution manager for WAL Iceberg tables
+ */
+export class SchemaEvolutionManager {
+  private lastColumnId: number
+  private schemas: IcebergSchema[]
+
+  constructor(initialSchema: IcebergSchema, lastColumnId: number) {
+    this.schemas = [initialSchema]
+    this.lastColumnId = lastColumnId
+  }
+
+  /**
+   * Get the current schema
+   */
+  getCurrentSchema(): IcebergSchema {
+    const schema = this.schemas[this.schemas.length - 1]
+    if (!schema) {
+      throw new Error('No schema found')
+    }
+    return schema
+  }
+
+  /**
+   * Get all schema versions
+   */
+  getAllSchemas(): IcebergSchema[] {
+    return [...this.schemas]
+  }
+
+  /**
+   * Get the last column ID
+   */
+  getLastColumnId(): number {
+    return this.lastColumnId
+  }
+
+  /**
+   * Apply schema evolution operations
+   */
+  evolve(operations: SchemaEvolutionOp[]): SchemaEvolutionResult {
+    const currentSchema = this.getCurrentSchema()
+    const newFields = [...currentSchema.fields]
+    const changes: string[] = []
+    const errors: string[] = []
+
+    for (const op of operations) {
+      switch (op.type) {
+        case 'add-column': {
+          // Check if field ID already exists
+          if (newFields.some((f) => f.id === op.field.id)) {
+            errors.push(`Field ID ${op.field.id} already exists`)
+            continue
+          }
+          // Check if field name already exists
+          if (newFields.some((f) => f.name === op.field.name)) {
+            errors.push(`Field name '${op.field.name}' already exists`)
+            continue
+          }
+          newFields.push(op.field)
+          if (op.field.id > this.lastColumnId) {
+            this.lastColumnId = op.field.id
+          }
+          changes.push(`Added column '${op.field.name}' (id: ${op.field.id})`)
+          break
+        }
+
+        case 'drop-column': {
+          const index = newFields.findIndex((f) => f.id === op.fieldId)
+          if (index === -1) {
+            errors.push(`Field ID ${op.fieldId} not found`)
+            continue
+          }
+          const droppedField = newFields[index]
+          // Don't actually remove, just mark as dropped (Iceberg doesn't delete fields)
+          // Instead, we leave it in the schema but it won't be written to new files
+          if (droppedField) {
+            changes.push(`Marked column '${droppedField.name}' (id: ${op.fieldId}) for deletion`)
+          }
+          break
+        }
+
+        case 'rename-column': {
+          const field = newFields.find((f) => f.id === op.fieldId)
+          if (!field) {
+            errors.push(`Field ID ${op.fieldId} not found`)
+            continue
+          }
+          if (newFields.some((f) => f.name === op.newName && f.id !== op.fieldId)) {
+            errors.push(`Field name '${op.newName}' already exists`)
+            continue
+          }
+          const oldName = field.name
+          field.name = op.newName
+          changes.push(`Renamed column '${oldName}' to '${op.newName}'`)
+          break
+        }
+
+        case 'update-column-doc': {
+          const field = newFields.find((f) => f.id === op.fieldId)
+          if (!field) {
+            errors.push(`Field ID ${op.fieldId} not found`)
+            continue
+          }
+          field.doc = op.newDoc
+          changes.push(`Updated documentation for column '${field.name}'`)
+          break
+        }
+
+        case 'make-optional': {
+          const field = newFields.find((f) => f.id === op.fieldId)
+          if (!field) {
+            errors.push(`Field ID ${op.fieldId} not found`)
+            continue
+          }
+          if (!field.required) {
+            errors.push(`Field '${field.name}' is already optional`)
+            continue
+          }
+          field.required = false
+          changes.push(`Made column '${field.name}' optional`)
+          break
+        }
+
+        case 'make-required': {
+          const field = newFields.find((f) => f.id === op.fieldId)
+          if (!field) {
+            errors.push(`Field ID ${op.fieldId} not found`)
+            continue
+          }
+          if (field.required) {
+            errors.push(`Field '${field.name}' is already required`)
+            continue
+          }
+          // Note: Making a field required is only allowed if all existing data has non-null values
+          field.required = true
+          changes.push(`Made column '${field.name}' required (verify data compatibility)`)
+          break
+        }
+
+        case 'update-column-type': {
+          const field = newFields.find((f) => f.id === op.fieldId)
+          if (!field) {
+            errors.push(`Field ID ${op.fieldId} not found`)
+            continue
+          }
+          if (!isTypeEvolutionAllowed(field.type, op.newType)) {
+            errors.push(
+              `Type evolution from ${JSON.stringify(field.type)} to ${JSON.stringify(op.newType)} is not allowed`
+            )
+            continue
+          }
+          const oldType = field.type
+          field.type = op.newType
+          changes.push(`Changed type of '${field.name}' from ${JSON.stringify(oldType)} to ${JSON.stringify(op.newType)}`)
+          break
+        }
+      }
+    }
+
+    if (errors.length > 0 && changes.length === 0) {
+      return {
+        success: false,
+        newSchema: currentSchema,
+        changes,
+        errors,
+      }
+    }
+
+    // Create new schema version
+    const newSchema: IcebergSchema = {
+      type: 'struct',
+      'schema-id': currentSchema['schema-id'] + 1,
+      fields: newFields,
+    }
+
+    // Preserve identifier fields if they exist
+    if (currentSchema['identifier-field-ids']) {
+      newSchema['identifier-field-ids'] = currentSchema['identifier-field-ids']
+    }
+
+    this.schemas.push(newSchema)
+
+    return {
+      success: true,
+      newSchema,
+      changes,
+      errors,
+    }
+  }
+
+  /**
+   * Add a new column to the schema
+   */
+  addColumn(
+    name: string,
+    type: IcebergType,
+    options?: {
+      required?: boolean
+      doc?: string
+      fieldId?: number
+    }
+  ): SchemaEvolutionResult {
+    const fieldId = options?.fieldId ?? this.lastColumnId + 1
+    const field: IcebergField = {
+      id: fieldId,
+      name,
+      required: options?.required ?? false,
+      type,
+    }
+    if (options?.doc) {
+      field.doc = options.doc
+    }
+    return this.evolve([
+      {
+        type: 'add-column',
+        field,
+      },
+    ])
+  }
+
+  /**
+   * Rename a column
+   */
+  renameColumn(fieldId: number, newName: string): SchemaEvolutionResult {
+    return this.evolve([{ type: 'rename-column', fieldId, newName }])
+  }
+
+  /**
+   * Make a column optional
+   */
+  makeOptional(fieldId: number): SchemaEvolutionResult {
+    return this.evolve([{ type: 'make-optional', fieldId }])
+  }
+}
+
+/**
+ * Check if type evolution is allowed (Iceberg rules)
+ */
+function isTypeEvolutionAllowed(from: IcebergType, to: IcebergType): boolean {
+  // Primitive type promotions allowed by Iceberg
+  const promotions: Record<string, string[]> = {
+    int: ['long'],
+    float: ['double'],
+    decimal: ['decimal'], // Width can increase
+  }
+
+  if (typeof from === 'string' && typeof to === 'string') {
+    return from === to || (promotions[from]?.includes(to) ?? false)
+  }
+
+  // Struct/list/map evolution requires recursive checks
+  // For simplicity, we don't allow nested type evolution in this implementation
+  return false
+}
+
+/**
+ * Create a schema evolution manager initialized with the WAL schema
+ */
+export function createWALSchemaEvolutionManager(): SchemaEvolutionManager {
+  return new SchemaEvolutionManager(createWALSchema(), WAL_FIELD_IDS.SQL)
+}
+
+/**
+ * Validate that a partition spec is compatible with a schema
+ */
+export function validatePartitionSpec(
+  schema: IcebergSchema,
+  partitionSpec: IcebergPartitionSpec
+): { valid: boolean; errors: string[] } {
+  const errors: string[] = []
+  const fieldIds = new Set(schema.fields.map((f) => f.id))
+
+  for (const field of partitionSpec.fields) {
+    if (!fieldIds.has(field['source-id'])) {
+      errors.push(
+        `Partition field '${field.name}' references non-existent source field ID ${field['source-id']}`
+      )
+    }
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  }
+}
+
+/**
+ * Get the partition value for a WAL entry based on the partition spec
+ */
+export function getPartitionValue(
+  record: { do_id: string; timestamp: number },
+  partitionSpec: IcebergPartitionSpec
+): Record<string, unknown> {
+  const result: Record<string, unknown> = {}
+
+  for (const field of partitionSpec.fields) {
+    switch (field['source-id']) {
+      case WAL_FIELD_IDS.DO_ID:
+        result[field.name] = applyTransform(record.do_id, field.transform)
+        break
+      case WAL_FIELD_IDS.TIMESTAMP:
+        result[field.name] = applyTransform(record.timestamp, field.transform)
+        break
+    }
+  }
+
+  return result
+}
+
+/**
+ * Apply an Iceberg transform to a value
+ */
+function applyTransform(value: unknown, transform: string): unknown {
+  if (transform === 'identity') {
+    return value
+  }
+
+  if (typeof value === 'number') {
+    const date = new Date(value)
+    switch (transform) {
+      case 'year':
+        return date.getUTCFullYear()
+      case 'month':
+        // Days since epoch / 30 (approximate)
+        return Math.floor(value / (1000 * 60 * 60 * 24 * 30))
+      case 'day':
+        // Days since epoch
+        return Math.floor(value / (1000 * 60 * 60 * 24))
+      case 'hour':
+        // Hours since epoch
+        return Math.floor(value / (1000 * 60 * 60))
+    }
+  }
+
+  if (typeof value === 'string') {
+    const bucketMatch = transform.match(/^bucket\[(\d+)\]$/)
+    if (bucketMatch && bucketMatch[1]) {
+      const numBuckets = parseInt(bucketMatch[1], 10)
+      return hashString(value) % numBuckets
+    }
+
+    const truncateMatch = transform.match(/^truncate\[(\d+)\]$/)
+    if (truncateMatch && truncateMatch[1]) {
+      const width = parseInt(truncateMatch[1], 10)
+      return value.slice(0, width)
+    }
+  }
+
+  return value
+}
+
+/**
+ * Simple string hash function for bucket transform
+ */
+function hashString(str: string): number {
+  let hash = 0
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i)
+    hash = (hash << 5) - hash + char
+    hash = hash & hash // Convert to 32-bit integer
+  }
+  return Math.abs(hash)
+}
+
+/**
+ * Generate partition path from partition values
+ */
+export function generatePartitionPath(partition: Record<string, unknown>): string {
+  const parts: string[] = []
+  for (const [key, value] of Object.entries(partition)) {
+    parts.push(`${key}=${encodePartitionValue(value)}`)
+  }
+  return parts.join('/')
+}
+
+/**
+ * Encode a partition value for use in a path
+ */
+function encodePartitionValue(value: unknown): string {
+  if (value === null || value === undefined) {
+    return '__HIVE_DEFAULT_PARTITION__'
+  }
+  return String(value).replace(/[/\\?%*:|"<>]/g, '_')
+}