@dotdo/postgres 0.1.0

package/src/iceberg/metadata.ts

@@ -0,0 +1,649 @@
+/**
+ * Iceberg Metadata Writer
+ * Task: postgres-6qa - Apache Iceberg table format support
+ *
+ * Manages Iceberg metadata files including manifest files, manifest lists,
+ * and table metadata.json files following the Iceberg v2 specification.
+ */
+
+import type {
+  IcebergTableMetadata,
+  IcebergSchema,
+  IcebergPartitionSpec,
+  IcebergSortOrder,
+  IcebergSnapshot,
+  IcebergSnapshotSummary,
+  IcebergManifestFile,
+  IcebergManifestEntry,
+  IcebergDataFile,
+  IcebergSnapshotLogEntry,
+  IcebergMetadataLogEntry,
+} from './types'
+
+/**
+ * Generate a UUID v4
+ */
+function generateUUID(): string {
+  const hex = '0123456789abcdef'
+  let uuid = ''
+  for (let i = 0; i < 36; i++) {
+    if (i === 8 || i === 13 || i === 18 || i === 23) {
+      uuid += '-'
+    } else if (i === 14) {
+      uuid += '4'
+    } else if (i === 19) {
+      uuid += hex[(Math.random() * 4) | 8]
+    } else {
+      uuid += hex[(Math.random() * 16) | 0]
+    }
+  }
+  return uuid
+}
+
+/**
+ * Generate a unique snapshot ID
+ */
+function generateSnapshotId(): number {
+  // Use timestamp + random for uniqueness
+  return Date.now() * 1000 + Math.floor(Math.random() * 1000)
+}
+
+/**
+ * Iceberg table metadata builder
+ */
+export class IcebergMetadataBuilder {
+  private tableUuid: string
+  private location: string
+  private lastSequenceNumber: number = 0
+  private lastColumnId: number
+  private currentSchemaId: number = 0
+  private schemas: IcebergSchema[]
+  private defaultSpecId: number = 0
+  private partitionSpecs: IcebergPartitionSpec[]
+  private lastPartitionId: number
+  private defaultSortOrderId: number = 0
+  private sortOrders: IcebergSortOrder[]
+  private properties: Record<string, string>
+  private currentSnapshotId: number | undefined
+  private snapshots: IcebergSnapshot[] = []
+  private snapshotLog: IcebergSnapshotLogEntry[] = []
+  private metadataLog: IcebergMetadataLogEntry[] = []
+
+  constructor(options: {
+    location: string
+    schema: IcebergSchema
+    partitionSpec: IcebergPartitionSpec
+    sortOrder?: IcebergSortOrder
+    properties?: Record<string, string>
+    tableUuid?: string
+  }) {
+    this.tableUuid = options.tableUuid || generateUUID()
+    this.location = options.location
+    this.schemas = [options.schema]
+    this.currentSchemaId = options.schema['schema-id']
+    this.lastColumnId = Math.max(...options.schema.fields.map((f) => f.id))
+    this.partitionSpecs = [options.partitionSpec]
+    this.defaultSpecId = options.partitionSpec['spec-id']
+    this.lastPartitionId = Math.max(
+      ...options.partitionSpec.fields.map((f) => f['field-id']),
+      0
+    )
+    this.sortOrders = options.sortOrder
+      ? [options.sortOrder]
+      : [{ 'order-id': 0, fields: [] }]
+    this.defaultSortOrderId = options.sortOrder?.['order-id'] ?? 0
+    this.properties = options.properties || {}
+  }
+
+  /**
+   * Get the table UUID
+   */
+  getTableUuid(): string {
+    return this.tableUuid
+  }
+
+  /**
+   * Get the table location
+   */
+  getLocation(): string {
+    return this.location
+  }
+
+  /**
+   * Get current snapshot ID
+   */
+  getCurrentSnapshotId(): number | undefined {
+    return this.currentSnapshotId
+  }
+
+  /**
+   * Get the current sequence number
+   */
+  getSequenceNumber(): number {
+    return this.lastSequenceNumber
+  }
+
+  /**
+   * Increment and return the next sequence number
+   */
+  nextSequenceNumber(): number {
+    this.lastSequenceNumber++
+    return this.lastSequenceNumber
+  }
+
+  /**
+   * Get current schema
+   */
+  getCurrentSchema(): IcebergSchema {
+    return this.schemas.find((s) => s['schema-id'] === this.currentSchemaId)!
+  }
+
+  /**
+   * Get current partition spec
+   */
+  getCurrentPartitionSpec(): IcebergPartitionSpec {
+    return this.partitionSpecs.find((s) => s['spec-id'] === this.defaultSpecId)!
+  }
+
+  /**
+   * Add a new schema version
+   */
+  addSchema(schema: IcebergSchema): void {
+    this.schemas.push(schema)
+    this.currentSchemaId = schema['schema-id']
+    this.lastColumnId = Math.max(this.lastColumnId, ...schema.fields.map((f) => f.id))
+  }
+
+  /**
+   * Add a new partition spec
+   */
+  addPartitionSpec(spec: IcebergPartitionSpec): void {
+    this.partitionSpecs.push(spec)
+    this.defaultSpecId = spec['spec-id']
+    this.lastPartitionId = Math.max(
+      this.lastPartitionId,
+      ...spec.fields.map((f) => f['field-id'])
+    )
+  }
+
+  /**
+   * Create a new snapshot
+   */
+  createSnapshot(options: {
+    manifestListPath: string
+    summary: IcebergSnapshotSummary
+    schemaId?: number
+  }): IcebergSnapshot {
+    const snapshotId = generateSnapshotId()
+    const sequenceNumber = this.nextSequenceNumber()
+    const timestamp = Date.now()
+
+    const snapshot: IcebergSnapshot = {
+      'snapshot-id': snapshotId,
+      ...(this.currentSnapshotId !== undefined && { 'parent-snapshot-id': this.currentSnapshotId }),
+      'sequence-number': sequenceNumber,
+      'timestamp-ms': timestamp,
+      'manifest-list': options.manifestListPath,
+      summary: options.summary,
+      'schema-id': options.schemaId ?? this.currentSchemaId,
+    }
+
+    this.snapshots.push(snapshot)
+    this.currentSnapshotId = snapshotId
+    this.snapshotLog.push({
+      'timestamp-ms': timestamp,
+      'snapshot-id': snapshotId,
+    })
+
+    return snapshot
+  }
+
+  /**
+   * Add metadata log entry
+   */
+  addMetadataLogEntry(metadataFile: string): void {
+    this.metadataLog.push({
+      'timestamp-ms': Date.now(),
+      'metadata-file': metadataFile,
+    })
+  }
+
+  /**
+   * Set a table property
+   */
+  setProperty(key: string, value: string): void {
+    this.properties[key] = value
+  }
+
+  /**
+   * Build the table metadata object
+   */
+  build(): IcebergTableMetadata {
+    const result: IcebergTableMetadata = {
+      'format-version': 2,
+      'table-uuid': this.tableUuid,
+      location: this.location,
+      'last-sequence-number': this.lastSequenceNumber,
+      'last-updated-ms': Date.now(),
+      'last-column-id': this.lastColumnId,
+      'current-schema-id': this.currentSchemaId,
+      schemas: this.schemas,
+      'default-spec-id': this.defaultSpecId,
+      'partition-specs': this.partitionSpecs,
+      'last-partition-id': this.lastPartitionId,
+      'default-sort-order-id': this.defaultSortOrderId,
+      'sort-orders': this.sortOrders,
+      properties: this.properties,
+    }
+
+    if (this.currentSnapshotId !== undefined) {
+      result['current-snapshot-id'] = this.currentSnapshotId
+    }
+    if (this.snapshots.length > 0) {
+      result.snapshots = this.snapshots
+    }
+    if (this.snapshotLog.length > 0) {
+      result['snapshot-log'] = this.snapshotLog
+    }
+    if (this.metadataLog.length > 0) {
+      result['metadata-log'] = this.metadataLog
+    }
+
+    return result
+  }
+
+  /**
+   * Build and serialize to JSON
+   */
+  toJSON(): string {
+    return JSON.stringify(this.build(), null, 2)
+  }
+
+  /**
+   * Load from existing metadata
+   */
+  static fromMetadata(metadata: IcebergTableMetadata): IcebergMetadataBuilder {
+    const sortOrder = metadata['sort-orders'].find(
+      (s) => s['order-id'] === metadata['default-sort-order-id']
+    )
+
+    const options: {
+      location: string
+      schema: IcebergSchema
+      partitionSpec: IcebergPartitionSpec
+      sortOrder?: IcebergSortOrder
+      properties?: Record<string, string>
+      tableUuid?: string
+    } = {
+      location: metadata.location,
+      schema: metadata.schemas.find((s) => s['schema-id'] === metadata['current-schema-id'])!,
+      partitionSpec: metadata['partition-specs'].find(
+        (s) => s['spec-id'] === metadata['default-spec-id']
+      )!,
+      tableUuid: metadata['table-uuid'],
+    }
+
+    if (sortOrder) {
+      options.sortOrder = sortOrder
+    }
+    if (metadata.properties) {
+      options.properties = metadata.properties
+    }
+
+    const builder = new IcebergMetadataBuilder(options)
+
+    // Restore state
+    builder.lastSequenceNumber = metadata['last-sequence-number']
+    builder.lastColumnId = metadata['last-column-id']
+    builder.lastPartitionId = metadata['last-partition-id']
+    builder.schemas = [...metadata.schemas]
+    builder.partitionSpecs = [...metadata['partition-specs']]
+    builder.sortOrders = [...metadata['sort-orders']]
+    builder.currentSnapshotId = metadata['current-snapshot-id']
+    builder.snapshots = metadata.snapshots ? [...metadata.snapshots] : []
+    builder.snapshotLog = metadata['snapshot-log'] ? [...metadata['snapshot-log']] : []
+    builder.metadataLog = metadata['metadata-log'] ? [...metadata['metadata-log']] : []
+
+    return builder
+  }
+}
+
+/**
+ * Manifest file builder
+ */
+export class ManifestBuilder {
+  private entries: IcebergManifestEntry[] = []
+  private partitionSpecId: number
+  private sequenceNumber: number
+  private snapshotId: number
+  private content: 0 | 1
+
+  constructor(options: {
+    partitionSpecId: number
+    sequenceNumber: number
+    snapshotId: number
+    content?: 0 | 1
+  }) {
+    this.partitionSpecId = options.partitionSpecId
+    this.sequenceNumber = options.sequenceNumber
+    this.snapshotId = options.snapshotId
+    this.content = options.content ?? 0
+  }
+
+  /**
+   * Add a data file to the manifest
+   */
+  addDataFile(dataFile: IcebergDataFile, status: 0 | 1 | 2 = 1): void {
+    const entry: IcebergManifestEntry = {
+      status,
+      'file-sequence-number': this.sequenceNumber,
+      'data-file': dataFile,
+    }
+    if (status === 1) {
+      entry['snapshot-id'] = this.snapshotId
+      entry['sequence-number'] = this.sequenceNumber
+    }
+    this.entries.push(entry)
+  }
+
+  /**
+   * Add an existing data file (for compaction)
+   */
+  addExistingFile(dataFile: IcebergDataFile, originalSequenceNumber: number): void {
+    this.entries.push({
+      status: 0, // EXISTING
+      'sequence-number': originalSequenceNumber,
+      'file-sequence-number': originalSequenceNumber,
+      'data-file': dataFile,
+    })
+  }
+
+  /**
+   * Mark a data file as deleted
+   */
+  deleteDataFile(dataFile: IcebergDataFile): void {
+    this.entries.push({
+      status: 2, // DELETED
+      'snapshot-id': this.snapshotId,
+      'sequence-number': this.sequenceNumber,
+      'file-sequence-number': this.sequenceNumber,
+      'data-file': dataFile,
+    })
+  }
+
+  /**
+   * Get manifest entries
+   */
+  getEntries(): IcebergManifestEntry[] {
+    return this.entries
+  }
+
+  /**
+   * Get entry counts by status
+   */
+  getEntryCounts(): {
+    added: number
+    existing: number
+    deleted: number
+  } {
+    let added = 0
+    let existing = 0
+    let deleted = 0
+
+    for (const entry of this.entries) {
+      switch (entry.status) {
+        case 0:
+          existing++
+          break
+        case 1:
+          added++
+          break
+        case 2:
+          deleted++
+          break
+      }
+    }
+
+    return { added, existing, deleted }
+  }
+
+  /**
+   * Get row counts by status
+   */
+  getRowCounts(): {
+    addedRows: number
+    existingRows: number
+    deletedRows: number
+  } {
+    let addedRows = 0
+    let existingRows = 0
+    let deletedRows = 0
+
+    for (const entry of this.entries) {
+      const rows = entry['data-file']['record-count']
+      switch (entry.status) {
+        case 0:
+          existingRows += rows
+          break
+        case 1:
+          addedRows += rows
+          break
+        case 2:
+          deletedRows += rows
+          break
+      }
+    }
+
+    return { addedRows, existingRows, deletedRows }
+  }
+
+  /**
+   * Build the manifest as Avro-compatible JSON
+   *
+   * Note: Iceberg manifests are typically Avro files, but for simplicity
+   * we use JSON format here. Production implementations should use Avro.
+   */
+  build(): { schema: object; entries: IcebergManifestEntry[] } {
+    return {
+      schema: MANIFEST_AVRO_SCHEMA,
+      entries: this.entries,
+    }
+  }
+
+  /**
+   * Build and serialize to JSON
+   */
+  toJSON(): string {
+    return JSON.stringify(this.build(), null, 2)
+  }
+
+  /**
+   * Create manifest file metadata
+   */
+  createManifestFileMetadata(manifestPath: string, manifestLength: number): IcebergManifestFile {
+    const counts = this.getEntryCounts()
+    const rowCounts = this.getRowCounts()
+
+    return {
+      'manifest-path': manifestPath,
+      'manifest-length': manifestLength,
+      'partition-spec-id': this.partitionSpecId,
+      content: this.content,
+      'sequence-number': this.sequenceNumber,
+      'min-sequence-number': Math.min(
+        ...this.entries.map((e) => e['sequence-number'] ?? this.sequenceNumber)
+      ),
+      'added-snapshot-id': this.snapshotId,
+      'added-files-count': counts.added,
+      'existing-files-count': counts.existing,
+      'deleted-files-count': counts.deleted,
+      'added-rows-count': rowCounts.addedRows,
+      'existing-rows-count': rowCounts.existingRows,
+      'deleted-rows-count': rowCounts.deletedRows,
+    }
+  }
+}
+
+/**
+ * Manifest list builder
+ */
+export class ManifestListBuilder {
+  private manifests: IcebergManifestFile[] = []
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  constructor(_snapshotId: number, _sequenceNumber: number) {
+    // snapshotId and sequenceNumber reserved for future manifest list metadata
+  }
+
+  /**
+   * Add a manifest file to the list
+   */
+  addManifest(manifest: IcebergManifestFile): void {
+    this.manifests.push(manifest)
+  }
+
+  /**
+   * Get all manifest files
+   */
+  getManifests(): IcebergManifestFile[] {
+    return this.manifests
+  }
+
+  /**
+   * Build the manifest list as Avro-compatible JSON
+   */
+  build(): { schema: object; manifests: IcebergManifestFile[] } {
+    return {
+      schema: MANIFEST_LIST_AVRO_SCHEMA,
+      manifests: this.manifests,
+    }
+  }
+
+  /**
+   * Build and serialize to JSON
+   */
+  toJSON(): string {
+    return JSON.stringify(this.build(), null, 2)
+  }
+
+  /**
+   * Calculate snapshot summary
+   */
+  calculateSummary(operation: 'append' | 'overwrite' | 'delete' | 'replace'): IcebergSnapshotSummary {
+    let addedDataFiles = 0
+    let deletedDataFiles = 0
+    let addedRecords = 0
+    let deletedRecords = 0
+    let addedFilesSize = 0
+    let removedFilesSize = 0
+
+    for (const manifest of this.manifests) {
+      addedDataFiles += manifest['added-files-count']
+      deletedDataFiles += manifest['deleted-files-count']
+      addedRecords += manifest['added-rows-count']
+      deletedRecords += manifest['deleted-rows-count']
+      // Note: File sizes would need to be tracked separately
+    }
+
+    const totalRecords = addedRecords - deletedRecords
+    // totalFiles would be used for size tracking in full implementation
+    // const totalFiles = addedDataFiles - deletedDataFiles
+
+    return {
+      operation,
+      'added-data-files': String(addedDataFiles),
+      'deleted-data-files': String(deletedDataFiles),
+      'added-records': String(addedRecords),
+      'deleted-records': String(deletedRecords),
+      'added-files-size': String(addedFilesSize),
+      'removed-files-size': String(removedFilesSize),
+      'total-records': String(totalRecords),
+      'total-files-size': String(addedFilesSize - removedFilesSize),
+    }
+  }
+}
+
+/**
+ * Avro schema for manifest files (simplified)
+ */
+const MANIFEST_AVRO_SCHEMA = {
+  type: 'record',
+  name: 'manifest_entry',
+  fields: [
+    { name: 'status', type: 'int' },
+    { name: 'snapshot_id', type: ['null', 'long'], default: null },
+    { name: 'sequence_number', type: ['null', 'long'], default: null },
+    { name: 'file_sequence_number', type: ['null', 'long'], default: null },
+    {
+      name: 'data_file',
+      type: {
+        type: 'record',
+        name: 'r2',
+        fields: [
+          { name: 'content', type: 'int' },
+          { name: 'file_path', type: 'string' },
+          { name: 'file_format', type: 'string' },
+          { name: 'partition', type: { type: 'record', name: 'r3', fields: [] } },
+          { name: 'record_count', type: 'long' },
+          { name: 'file_size_in_bytes', type: 'long' },
+        ],
+      },
+    },
+  ],
+}
+
+/**
+ * Avro schema for manifest list files (simplified)
+ */
+const MANIFEST_LIST_AVRO_SCHEMA = {
+  type: 'record',
+  name: 'manifest_file',
+  fields: [
+    { name: 'manifest_path', type: 'string' },
+    { name: 'manifest_length', type: 'long' },
+    { name: 'partition_spec_id', type: 'int' },
+    { name: 'content', type: 'int' },
+    { name: 'sequence_number', type: 'long' },
+    { name: 'min_sequence_number', type: 'long' },
+    { name: 'added_snapshot_id', type: 'long' },
+    { name: 'added_files_count', type: 'int' },
+    { name: 'existing_files_count', type: 'int' },
+    { name: 'deleted_files_count', type: 'int' },
+    { name: 'added_rows_count', type: 'long' },
+    { name: 'existing_rows_count', type: 'long' },
+    { name: 'deleted_rows_count', type: 'long' },
+  ],
+}
+
+/**
+ * Generate a metadata file name
+ */
+export function generateMetadataFileName(version: number): string {
+  // Version is used; timestamp would be for additional uniqueness if needed
+  return `v${version}.metadata.json`
+}
+
+/**
+ * Generate a manifest file name
+ */
+export function generateManifestFileName(snapshotId: number, attempt: number = 0): string {
+  const uuid = generateUUID()
+  return `${snapshotId}-m${attempt}-${uuid}.avro`
+}
+
+/**
+ * Generate a manifest list file name
+ */
+export function generateManifestListFileName(snapshotId: number): string {
+  const uuid = generateUUID()
+  return `snap-${snapshotId}-${uuid}.avro`
+}
+
+/**
+ * Generate a data file name
+ */
+export function generateDataFileName(partitionPath: string): string {
+  const uuid = generateUUID()
+  const timestamp = Date.now()
+  return `${partitionPath}/data-${timestamp}-${uuid}.parquet`
+}