@dotdo/postgres 0.1.0

package/src/iceberg/catalog-api.ts

@@ -0,0 +1,520 @@
+/**
+ * Iceberg Catalog REST API
+ * Task: postgres-b44 - Iceberg metadata storage strategy
+ *
+ * Implements the Iceberg REST Catalog API specification for external tool compatibility.
+ * Compatible with:
+ * - Apache Spark
+ * - Trino/Presto
+ * - DuckDB
+ * - PyIceberg
+ * - Any Iceberg REST catalog client
+ *
+ * @see https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml
+ */
+
+import { Hono } from 'hono'
+import type { Context } from 'hono'
+import {
+  IcebergCatalog,
+  NamespaceNotFoundError,
+  TableNotFoundError,
+  CommitConflictError,
+  type TableIdentifier,
+  type NamespaceIdentifier,
+  type CreateNamespaceRequest,
+  type CreateTableRequest,
+  type CommitTableRequest,
+} from './catalog'
+import type { IcebergSchema, IcebergPartitionSpec, IcebergSortOrder } from './types'
+import { IcebergMetadataBuilder } from './metadata'
+import { createLogger, LogLevel, type ILogger } from '@dotdo/postgres-shared'
+
+// Module-level logger for Catalog API errors
+const catalogApiLogger: ILogger = createLogger({
+  level: LogLevel.ERROR,
+  prefix: '[CatalogAPI]',
+})
+
+/**
+ * API configuration
+ */
+export interface CatalogApiConfig {
+  /** The Iceberg catalog instance */
+  catalog: IcebergCatalog
+  /** API prefix (default: '/api/catalog/v1') */
+  prefix?: string
+  /** Enable CORS (default: true) */
+  cors?: boolean
+  /** Authentication handler (optional) */
+  authenticate?: (c: Context) => Promise<boolean>
+}
+
+/**
+ * Standard error response
+ */
+interface ErrorResponse {
+  error: {
+    message: string
+    type: string
+    code: number
+  }
+}
+
+/**
+ * Create error response
+ */
+function errorResponse(message: string, type: string, code: number): ErrorResponse {
+  return {
+    error: {
+      message,
+      type,
+      code,
+    },
+  }
+}
+
+/**
+ * Parse namespace from URL path
+ */
+function parseNamespace(namespace: string): NamespaceIdentifier {
+  // URL-encoded multipart namespace (e.g., "tenant%1Fapp" for tenant.app)
+  const decoded = decodeURIComponent(namespace)
+  const levels = decoded.includes('\x1F') ? decoded.split('\x1F') : decoded.split('.')
+  return { levels }
+}
+
+
+/**
+ * Create the Iceberg Catalog REST API router
+ */
+export function createCatalogApi(config: CatalogApiConfig): Hono {
+  const { catalog, prefix = '/api/catalog/v1', cors = true, authenticate } = config
+
+  const app = new Hono()
+
+  // CORS middleware
+  if (cors) {
+    app.use('*', async (c, next) => {
+      c.res.headers.set('Access-Control-Allow-Origin', '*')
+      c.res.headers.set('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS')
+      c.res.headers.set('Access-Control-Allow-Headers', 'Content-Type, Authorization, X-Iceberg-Access-Delegation')
+
+      if (c.req.method === 'OPTIONS') {
+        return new Response(null, { status: 204 })
+      }
+
+      return next()
+    })
+  }
+
+  // Authentication middleware
+  if (authenticate) {
+    app.use('*', async (c, next) => {
+      const authenticated = await authenticate(c)
+      if (!authenticated) {
+        return c.json(errorResponse('Unauthorized', 'UnauthorizedError', 401), 401)
+      }
+      return next()
+    })
+  }
+
+  // Error handling middleware
+  app.onError((err, c) => {
+    catalogApiLogger.error('Error', { error: err instanceof Error ? err.message : String(err) })
+
+    if (err instanceof NamespaceNotFoundError) {
+      return c.json(errorResponse(err.message, 'NoSuchNamespaceException', 404), 404)
+    }
+
+    if (err instanceof TableNotFoundError) {
+      return c.json(errorResponse(err.message, 'NoSuchTableException', 404), 404)
+    }
+
+    if (err instanceof CommitConflictError) {
+      return c.json(errorResponse(err.message, 'CommitFailedException', 409), 409)
+    }
+
+    return c.json(
+      errorResponse(err.message || 'Internal server error', 'InternalServerError', 500),
+      500
+    )
+  })
+
+  // ========== Config Endpoint ==========
+
+  /**
+   * GET /config
+   * Get catalog configuration
+   */
+  app.get(`${prefix}/config`, async (c) => {
+    return c.json({
+      defaults: {},
+      overrides: {},
+    })
+  })
+
+  // ========== Namespace Endpoints ==========
+
+  /**
+   * GET /namespaces
+   * List namespaces
+   */
+  app.get(`${prefix}/namespaces`, async (c) => {
+    const parent = c.req.query('parent')
+    let parentNs: NamespaceIdentifier | undefined
+
+    if (parent) {
+      parentNs = parseNamespace(parent)
+    }
+
+    const namespaces = await catalog.listNamespaces(parentNs)
+
+    return c.json({
+      namespaces: namespaces.map((ns) => ns.levels),
+    })
+  })
+
+  /**
+   * POST /namespaces
+   * Create a namespace
+   */
+  app.post(`${prefix}/namespaces`, async (c) => {
+    const body = await c.req.json<CreateNamespaceRequest>()
+
+    if (!body.namespace || !Array.isArray(body.namespace) || body.namespace.length === 0) {
+      return c.json(errorResponse('Invalid namespace', 'BadRequestException', 400), 400)
+    }
+
+    const namespace: NamespaceIdentifier = { levels: body.namespace }
+    const metadata = await catalog.createNamespace(namespace, body.properties || {})
+
+    return c.json({
+      namespace: metadata.namespace.levels,
+      properties: metadata.properties,
+    })
+  })
+
+  /**
+   * GET /namespaces/:namespace
+   * Get namespace metadata
+   */
+  app.get(`${prefix}/namespaces/:namespace`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const metadata = await catalog.getNamespace(namespace)
+
+    return c.json({
+      namespace: metadata.namespace.levels,
+      properties: metadata.properties,
+    })
+  })
+
+  /**
+   * HEAD /namespaces/:namespace (via GET with HEAD method check)
+   * Check if namespace exists
+   */
+  app.on('HEAD', `${prefix}/namespaces/:namespace`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const exists = await catalog.namespaceExists(namespace)
+
+    if (!exists) {
+      return new Response(null, { status: 404 })
+    }
+
+    return new Response(null, { status: 204 })
+  })
+
+  /**
+   * POST /namespaces/:namespace/properties
+   * Update namespace properties
+   */
+  app.post(`${prefix}/namespaces/:namespace/properties`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const body = await c.req.json<{
+      updates?: Record<string, string>
+      removals?: string[]
+    }>()
+
+    await catalog.updateNamespaceProperties(
+      namespace,
+      body.updates || {},
+      body.removals || []
+    )
+
+    return c.json({
+      updated: Object.keys(body.updates || {}),
+      removed: body.removals || [],
+      missing: [], // We don't track this
+    })
+  })
+
+  /**
+   * DELETE /namespaces/:namespace
+   * Drop a namespace
+   */
+  app.delete(`${prefix}/namespaces/:namespace`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    await catalog.dropNamespace(namespace)
+    return new Response(null, { status: 204 })
+  })
+
+  // ========== Table Endpoints ==========
+
+  /**
+   * GET /namespaces/:namespace/tables
+   * List tables in a namespace
+   */
+  app.get(`${prefix}/namespaces/:namespace/tables`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const tables = await catalog.listTables(namespace)
+
+    return c.json({
+      identifiers: tables.map((t) => ({
+        namespace: t.namespace.levels,
+        name: t.name,
+      })),
+    })
+  })
+
+  /**
+   * POST /namespaces/:namespace/tables
+   * Create a table
+   */
+  app.post(`${prefix}/namespaces/:namespace/tables`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const body = await c.req.json<CreateTableRequest>()
+
+    if (!body.name) {
+      return c.json(errorResponse('Table name is required', 'BadRequestException', 400), 400)
+    }
+
+    if (!body.schema) {
+      return c.json(errorResponse('Schema is required', 'BadRequestException', 400), 400)
+    }
+
+    const identifier: TableIdentifier = {
+      namespace,
+      name: body.name,
+    }
+
+    // Build initial metadata
+    const schema = body.schema as IcebergSchema
+    const partitionSpec = (body['partition-spec'] as IcebergPartitionSpec) || {
+      'spec-id': 0,
+      fields: [],
+    }
+    const sortOrder = (body['write-order'] as IcebergSortOrder) || {
+      'order-id': 0,
+      fields: [],
+    }
+
+    const builder = new IcebergMetadataBuilder({
+      location: '', // Will be set by catalog
+      schema,
+      partitionSpec,
+      sortOrder,
+      ...(body.properties && { properties: body.properties }),
+    })
+
+    const metadata = builder.build()
+    const entry = await catalog.createTable(identifier, metadata, body.properties || {})
+
+    // Load the full metadata to return
+    const fullMetadata = await catalog.loadTable(identifier)
+
+    return c.json({
+      'metadata-location': entry.metadataLocation,
+      metadata: fullMetadata,
+    })
+  })
+
+  /**
+   * GET /namespaces/:namespace/tables/:table
+   * Load a table
+   */
+  app.get(`${prefix}/namespaces/:namespace/tables/:table`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const tableName = c.req.param('table')
+
+    const identifier: TableIdentifier = {
+      namespace,
+      name: tableName,
+    }
+
+    const entry = await catalog.getTableEntry(identifier)
+    if (!entry) {
+      throw new TableNotFoundError(identifier)
+    }
+
+    const metadata = await catalog.loadTable(identifier)
+
+    return c.json({
+      'metadata-location': entry.metadataLocation,
+      metadata,
+    })
+  })
+
+  /**
+   * HEAD /namespaces/:namespace/tables/:table
+   * Check if table exists
+   */
+  app.on('HEAD', `${prefix}/namespaces/:namespace/tables/:table`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const tableName = c.req.param('table')
+
+    const identifier: TableIdentifier = {
+      namespace,
+      name: tableName,
+    }
+
+    const exists = await catalog.tableExists(identifier)
+
+    if (!exists) {
+      return new Response(null, { status: 404 })
+    }
+
+    return new Response(null, { status: 204 })
+  })
+
+  /**
+   * POST /namespaces/:namespace/tables/:table
+   * Commit table updates
+   */
+  app.post(`${prefix}/namespaces/:namespace/tables/:table`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const tableName = c.req.param('table')
+    const body = await c.req.json<CommitTableRequest>()
+
+    const identifier: TableIdentifier = {
+      namespace,
+      name: tableName,
+    }
+
+    // Get current metadata
+    const currentVersion = await catalog.getCurrentVersion(identifier)
+    const currentMetadata = await catalog.loadTable(identifier)
+
+    // Apply updates to metadata
+    const builder = IcebergMetadataBuilder.fromMetadata(currentMetadata)
+
+    for (const update of body.updates) {
+      switch (update.action) {
+        case 'add-snapshot':
+          builder.createSnapshot({
+            manifestListPath: update.snapshot['manifest-list'],
+            summary: update.snapshot.summary,
+            ...(update.snapshot['schema-id'] !== undefined && {
+              schemaId: update.snapshot['schema-id'],
+            }),
+          })
+          break
+
+        case 'set-properties':
+          for (const [key, value] of Object.entries(update.updates)) {
+            builder.setProperty(key, value)
+          }
+          break
+
+        case 'add-schema':
+          builder.addSchema(update.schema as IcebergSchema)
+          break
+
+        case 'add-partition-spec':
+          builder.addPartitionSpec(update.spec as IcebergPartitionSpec)
+          break
+
+        // Additional update types can be added here
+      }
+    }
+
+    // Commit the update
+    const newMetadata = builder.build()
+    await catalog.commitTableUpdate(identifier, newMetadata, currentVersion)
+
+    const entry = await catalog.getTableEntry(identifier)
+
+    return c.json({
+      'metadata-location': entry?.metadataLocation,
+      metadata: newMetadata,
+    })
+  })
+
+  /**
+   * DELETE /namespaces/:namespace/tables/:table
+   * Drop a table
+   */
+  app.delete(`${prefix}/namespaces/:namespace/tables/:table`, async (c) => {
+    const namespace = parseNamespace(c.req.param('namespace'))
+    const tableName = c.req.param('table')
+    const purge = c.req.query('purgeRequested') === 'true'
+
+    const identifier: TableIdentifier = {
+      namespace,
+      name: tableName,
+    }
+
+    await catalog.dropTable(identifier, purge)
+    return new Response(null, { status: 204 })
+  })
+
+  /**
+   * POST /tables/rename
+   * Rename a table
+   */
+  app.post(`${prefix}/tables/rename`, async (c) => {
+    const body = await c.req.json<{
+      source: { namespace: string[]; name: string }
+      destination: { namespace: string[]; name: string }
+    }>()
+
+    const from: TableIdentifier = {
+      namespace: { levels: body.source.namespace },
+      name: body.source.name,
+    }
+
+    const to: TableIdentifier = {
+      namespace: { levels: body.destination.namespace },
+      name: body.destination.name,
+    }
+
+    await catalog.renameTable(from, to)
+    return new Response(null, { status: 204 })
+  })
+
+  // ========== Metrics Endpoint ==========
+
+  /**
+   * POST /namespaces/:namespace/tables/:table/metrics
+   * Report table metrics (Iceberg clients report metrics here)
+   */
+  app.post(`${prefix}/namespaces/:namespace/tables/:table/metrics`, async () => {
+    // Accept metrics but don't process them in this implementation
+    // Could be extended to store/aggregate metrics
+    return new Response(null, { status: 204 })
+  })
+
+  // ========== OAuth Token Endpoint (if needed) ==========
+
+  /**
+   * POST /oauth/tokens
+   * OAuth token endpoint for authentication
+   */
+  app.post(`${prefix}/oauth/tokens`, async (c) => {
+    // This is a placeholder for OAuth implementation
+    // In production, implement proper OAuth2 token handling
+    return c.json({
+      access_token: 'anonymous-token',
+      token_type: 'bearer',
+      expires_in: 3600,
+    })
+  })
+
+  return app
+}
+
+/**
+ * Create a Hono middleware that adds catalog API routes
+ */
+export function catalogApiMiddleware(config: CatalogApiConfig): Hono {
+  return createCatalogApi(config)
+}