@dotdo/postgres 0.1.0

package/src/iceberg/time-travel-api.ts

@@ -0,0 +1,731 @@
+/**
+ * Time Travel Query REST API
+ * Task: postgres-3kc - Time travel query API (query as-of timestamp)
+ *
+ * Provides REST endpoints for time travel queries:
+ * - GET /v1/do/{doId}/query?as_of=<timestamp>
+ * - POST /v1/do/{doId}/sql with as_of parameter
+ * - GET /v1/do/{doId}/snapshots - list available snapshots
+ * - GET /v1/do/{doId}/time-range - get available time range
+ */
+
+import { Hono } from 'hono'
+import type { Context } from 'hono'
+import {
+  IcebergTimeTravelReader,
+  parseTimestamp,
+  parseTimeTravelSQL,
+  formatTimestamp,
+  TimestampOutOfRangeError,
+  NoSnapshotFoundError,
+  InvalidTimestampError,
+  type TimeTravelQueryOptions,
+  type TimeTravelMetadata,
+} from './time-travel'
+import { IcebergCatalog, TableNotFoundError, type TableIdentifier } from './catalog'
+import { createHistoricalAnalytics } from './analytics'
+import { createLogger, LogLevel, type ILogger } from '@dotdo/postgres-shared'
+
+// Re-export analytics types for external use
+export type { HistoricalAnalytics, ChangeRateAnalysis, GrowthMetrics, SnapshotComparison } from './analytics'
+
+// Module-level logger for Time Travel API errors
+const timeTravelApiLogger: ILogger = createLogger({
+  level: LogLevel.ERROR,
+  prefix: '[TimeTravelAPI]',
+})
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Time travel API configuration
+ */
+export interface TimeTravelApiConfig {
+  /** Iceberg catalog instance */
+  catalog: IcebergCatalog
+  /** R2 bucket for reading data */
+  bucket: R2Bucket
+  /** API prefix (default: '/v1') */
+  prefix?: string
+  /** Enable CORS (default: true) */
+  cors?: boolean
+  /** Authentication handler (optional) */
+  authenticate?: (c: Context) => Promise<boolean>
+  /** Default retention window in milliseconds */
+  retentionWindowMs?: number
+}
+
+/**
+ * Time travel query request body
+ */
+export interface TimeTravelQueryRequest {
+  /** SQL query to execute */
+  query: string
+  /** Query parameters */
+  params?: unknown[]
+  /** Timestamp for AS OF query (ISO string, epoch ms, or relative) */
+  as_of?: string | number
+  /** Start timestamp for range queries */
+  from_timestamp?: string | number
+  /** End timestamp for range queries */
+  to_timestamp?: string | number
+}
+
+/**
+ * Time travel query response
+ */
+export interface TimeTravelQueryResponse {
+  /** Whether the query was successful */
+  success: boolean
+  /** Query results (if applicable) */
+  data?: unknown
+  /** Time travel metadata */
+  timeTravel: TimeTravelMetadata
+  /** Execution time in milliseconds */
+  executionTimeMs: number
+  /** Timestamp of the query */
+  timestamp: string
+}
+
+/**
+ * Snapshot list response
+ */
+export interface SnapshotListResponse {
+  /** List of available snapshots */
+  snapshots: Array<{
+    snapshotId: number
+    timestampMs: number
+    timestamp: string
+    summary: Record<string, string>
+  }>
+  /** Current snapshot ID */
+  currentSnapshotId: number | null
+  /** Total number of snapshots */
+  count: number
+}
+
+/**
+ * Time range response
+ */
+export interface TimeRangeResponse {
+  /** Oldest available timestamp (ms) */
+  oldestTimestampMs: number | null
+  /** Newest available timestamp (ms) */
+  newestTimestampMs: number | null
+  /** Oldest as ISO string */
+  oldest: string | null
+  /** Newest as ISO string */
+  newest: string | null
+  /** Retention window in milliseconds */
+  retentionWindowMs: number
+}
+
+/**
+ * Error response structure
+ */
+interface ErrorResponse {
+  error: {
+    message: string
+    type: string
+    code: number
+    details?: Record<string, unknown>
+  }
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/**
+ * Create error response
+ */
+function errorResponse(
+  message: string,
+  type: string,
+  code: number,
+  details?: Record<string, unknown>
+): ErrorResponse {
+  return {
+    error: {
+      message,
+      type,
+      code,
+      ...(details && { details }),
+    },
+  }
+}
+
+/**
+ * Parse DO ID from path parameter
+ * Supports formats: do_id, namespace.do_id, namespace/do_id
+ */
+function parseDoIdentifier(doIdParam: string): TableIdentifier {
+  // Handle URL-encoded separators
+  const decoded = decodeURIComponent(doIdParam)
+
+  // Split by common separators
+  let parts: string[]
+  if (decoded.includes('/')) {
+    parts = decoded.split('/')
+  } else if (decoded.includes('.')) {
+    parts = decoded.split('.')
+  } else {
+    // Single DO ID - use default namespace
+    return {
+      namespace: { levels: ['default'] },
+      name: decoded,
+    }
+  }
+
+  if (parts.length === 1) {
+    return {
+      namespace: { levels: ['default'] },
+      name: parts[0]!,
+    }
+  }
+
+  // Last part is the table name, rest is namespace
+  const tableName = parts.pop()!
+  return {
+    namespace: { levels: parts },
+    name: tableName,
+  }
+}
+
+// ============================================================================
+// API Routes
+// ============================================================================
+
+/**
+ * Create the Time Travel Query API router
+ */
+export function createTimeTravelApi(config: TimeTravelApiConfig): Hono {
+  const {
+    catalog,
+    bucket,
+    prefix = '/v1',
+    cors = true,
+    authenticate,
+    retentionWindowMs = 30 * 24 * 60 * 60 * 1000,
+  } = config
+
+  const app = new Hono()
+  const reader = new IcebergTimeTravelReader({
+    bucket,
+    retentionWindowMs,
+  })
+  const analytics = createHistoricalAnalytics({
+    bucket,
+    catalog,
+    retentionWindowMs,
+  })
+
+  // CORS middleware
+  if (cors) {
+    app.use('*', async (c, next) => {
+      c.res.headers.set('Access-Control-Allow-Origin', '*')
+      c.res.headers.set(
+        'Access-Control-Allow-Methods',
+        'GET, POST, OPTIONS'
+      )
+      c.res.headers.set(
+        'Access-Control-Allow-Headers',
+        'Content-Type, Authorization, X-As-Of-Timestamp'
+      )
+
+      if (c.req.method === 'OPTIONS') {
+        // Return a 204 No Content response with CORS headers
+        return new Response(null, {
+          status: 204,
+          headers: {
+            'Access-Control-Allow-Origin': '*',
+            'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+            'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-As-Of-Timestamp',
+          },
+        })
+      }
+
+      return next()
+    })
+  }
+
+  // Authentication middleware
+  if (authenticate) {
+    app.use('*', async (c, next) => {
+      const authenticated = await authenticate(c)
+      if (!authenticated) {
+        return c.json(
+          errorResponse('Unauthorized', 'UnauthorizedError', 401),
+          401
+        )
+      }
+      return next()
+    })
+  }
+
+  // Error handling middleware
+  app.onError((err, c) => {
+    timeTravelApiLogger.error('Error', { error: err instanceof Error ? err.message : String(err) })
+
+    if (err instanceof TimestampOutOfRangeError) {
+      return c.json(
+        errorResponse(err.message, 'TimestampOutOfRangeError', 400, {
+          requestedTimestamp: err.requestedTimestamp,
+          oldestAvailable: err.oldestAvailable,
+          newestAvailable: err.newestAvailable,
+        }),
+        400
+      )
+    }
+
+    if (err instanceof NoSnapshotFoundError) {
+      return c.json(
+        errorResponse(err.message, 'NoSnapshotFoundError', 404, {
+          requestedTimestamp: err.requestedTimestamp,
+        }),
+        404
+      )
+    }
+
+    if (err instanceof InvalidTimestampError) {
+      return c.json(
+        errorResponse(err.message, 'InvalidTimestampError', 400, {
+          input: String(err.input),
+        }),
+        400
+      )
+    }
+
+    if (err instanceof TableNotFoundError) {
+      return c.json(
+        errorResponse(err.message, 'TableNotFoundError', 404),
+        404
+      )
+    }
+
+    return c.json(
+      errorResponse(
+        err.message || 'Internal server error',
+        'InternalServerError',
+        500
+      ),
+      500
+    )
+  })
+
+  // ========== Query Endpoints ==========
+
+  /**
+   * GET /v1/do/{doId}/query
+   * Execute a time travel query with query parameters
+   */
+  app.get(`${prefix}/do/:doId/query`, async (c) => {
+    const startTime = performance.now()
+
+    const doIdParam = c.req.param('doId')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    // Parse query parameters
+    const sql = c.req.query('sql')
+    const asOf = c.req.query('as_of') ?? c.req.header('X-As-Of-Timestamp')
+
+    if (!sql) {
+      return c.json(
+        errorResponse(
+          'Missing required query parameter: sql',
+          'BadRequestError',
+          400
+        ),
+        400
+      )
+    }
+
+    // Load table metadata
+    const metadata = await catalog.loadTable(tableId)
+
+    // Build query options
+    const options: TimeTravelQueryOptions = {}
+    if (asOf) {
+      options.asOf = asOf
+    }
+
+    // Resolve data files
+    const resolved = await reader.resolveDataFiles(metadata, options)
+
+    const executionTimeMs = performance.now() - startTime
+
+    const response: TimeTravelQueryResponse = {
+      success: true,
+      data: {
+        sql,
+        dataFiles: resolved.dataFiles.map((f) => ({
+          path: f['file-path'],
+          recordCount: f['record-count'],
+          fileSizeBytes: f['file-size-in-bytes'],
+          partition: f.partition,
+        })),
+      },
+      timeTravel: resolved.metadata,
+      executionTimeMs,
+      timestamp: new Date().toISOString(),
+    }
+
+    return c.json(response)
+  })
+
+  /**
+   * POST /v1/do/{doId}/sql
+   * Execute a time travel SQL query
+   */
+  app.post(`${prefix}/do/:doId/sql`, async (c) => {
+    const startTime = performance.now()
+
+    const doIdParam = c.req.param('doId')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    const body = await c.req.json<TimeTravelQueryRequest>()
+
+    if (!body.query) {
+      return c.json(
+        errorResponse(
+          'Missing required field: query',
+          'BadRequestError',
+          400
+        ),
+        400
+      )
+    }
+
+    // Parse SQL for AS OF clause
+    const parsedSql = parseTimeTravelSQL(body.query)
+
+    // Determine the timestamp to use
+    let asOfTimestamp: string | number | undefined = body.as_of
+
+    // SQL AS OF clause takes precedence
+    if (parsedSql.hasTimeTravelClause && parsedSql.timestamp) {
+      asOfTimestamp = parsedSql.timestamp.timestampMs
+    }
+
+    // Load table metadata
+    const metadata = await catalog.loadTable(tableId)
+
+    // Build query options
+    const options: TimeTravelQueryOptions = {}
+    if (asOfTimestamp) {
+      options.asOf = asOfTimestamp
+    }
+    if (body.from_timestamp) {
+      options.fromTimestamp = body.from_timestamp
+      options.mode = 'range'
+    }
+    if (body.to_timestamp) {
+      options.toTimestamp = body.to_timestamp
+      options.mode = 'range'
+    }
+
+    // Resolve data files
+    const resolved = await reader.resolveDataFiles(metadata, options)
+
+    const executionTimeMs = performance.now() - startTime
+
+    const response: TimeTravelQueryResponse = {
+      success: true,
+      data: {
+        originalQuery: body.query,
+        cleanQuery: parsedSql.cleanSql,
+        params: body.params,
+        dataFiles: resolved.dataFiles.map((f) => ({
+          path: f['file-path'],
+          recordCount: f['record-count'],
+          fileSizeBytes: f['file-size-in-bytes'],
+          partition: f.partition,
+        })),
+      },
+      timeTravel: resolved.metadata,
+      executionTimeMs,
+      timestamp: new Date().toISOString(),
+    }
+
+    return c.json(response)
+  })
+
+  // ========== Snapshot Endpoints ==========
+
+  /**
+   * GET /v1/do/{doId}/snapshots
+   * List all available snapshots for a DO
+   */
+  app.get(`${prefix}/do/:doId/snapshots`, async (c) => {
+    const doIdParam = c.req.param('doId')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    // Load table metadata
+    const metadata = await catalog.loadTable(tableId)
+
+    const snapshots = metadata.snapshots ?? []
+    const snapshotList = reader.listSnapshots(metadata)
+
+    const response: SnapshotListResponse = {
+      snapshots: snapshotList.map((s) => {
+        const full = snapshots.find((snap) => snap['snapshot-id'] === s.snapshotId)
+        return {
+          snapshotId: s.snapshotId,
+          timestampMs: s.timestampMs,
+          timestamp: s.timestamp,
+          summary: full?.summary ?? {},
+        }
+      }),
+      currentSnapshotId: metadata['current-snapshot-id'] ?? null,
+      count: snapshotList.length,
+    }
+
+    return c.json(response)
+  })
+
+  /**
+   * GET /v1/do/{doId}/snapshots/:snapshotId
+   * Get details for a specific snapshot
+   */
+  app.get(`${prefix}/do/:doId/snapshots/:snapshotId`, async (c) => {
+    const doIdParam = c.req.param('doId')
+    const snapshotIdParam = c.req.param('snapshotId')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    const snapshotId = parseInt(snapshotIdParam, 10)
+    if (isNaN(snapshotId)) {
+      return c.json(
+        errorResponse(
+          'Invalid snapshot ID',
+          'BadRequestError',
+          400
+        ),
+        400
+      )
+    }
+
+    // Load table metadata
+    const metadata = await catalog.loadTable(tableId)
+
+    const snapshot = metadata.snapshots?.find(
+      (s) => s['snapshot-id'] === snapshotId
+    )
+
+    if (!snapshot) {
+      return c.json(
+        errorResponse(
+          `Snapshot ${snapshotId} not found`,
+          'SnapshotNotFoundError',
+          404
+        ),
+        404
+      )
+    }
+
+    // Get data files for this snapshot
+    const resolved = await reader.getDataFilesForSnapshot(metadata, snapshot)
+
+    return c.json({
+      snapshotId: snapshot['snapshot-id'],
+      parentSnapshotId: snapshot['parent-snapshot-id'] ?? null,
+      sequenceNumber: snapshot['sequence-number'],
+      timestampMs: snapshot['timestamp-ms'],
+      timestamp: formatTimestamp(snapshot['timestamp-ms']),
+      manifestList: snapshot['manifest-list'],
+      summary: snapshot.summary,
+      schemaId: snapshot['schema-id'],
+      dataFiles: {
+        count: resolved.dataFiles.length,
+        totalRecords: resolved.metadata.totalRecords,
+        files: resolved.dataFiles.map((f) => ({
+          path: f['file-path'],
+          recordCount: f['record-count'],
+          fileSizeBytes: f['file-size-in-bytes'],
+          partition: f.partition,
+        })),
+      },
+    })
+  })
+
+  // ========== Time Range Endpoints ==========
+
+  /**
+   * GET /v1/do/{doId}/time-range
+   * Get the available time range for queries
+   */
+  app.get(`${prefix}/do/:doId/time-range`, async (c) => {
+    const doIdParam = c.req.param('doId')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    // Load table metadata
+    const metadata = await catalog.loadTable(tableId)
+
+    const range = reader.getAvailableTimeRange(metadata)
+
+    const response: TimeRangeResponse = {
+      oldestTimestampMs: range.oldest,
+      newestTimestampMs: range.newest,
+      oldest: range.oldest ? formatTimestamp(range.oldest) : null,
+      newest: range.newest ? formatTimestamp(range.newest) : null,
+      retentionWindowMs,
+    }
+
+    return c.json(response)
+  })
+
+  /**
+   * GET /v1/do/{doId}/resolve-timestamp
+   * Resolve a timestamp and find the corresponding snapshot
+   */
+  app.get(`${prefix}/do/:doId/resolve-timestamp`, async (c) => {
+    const doIdParam = c.req.param('doId')
+    const timestampParam = c.req.query('timestamp')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    if (!timestampParam) {
+      return c.json(
+        errorResponse(
+          'Missing required query parameter: timestamp',
+          'BadRequestError',
+          400
+        ),
+        400
+      )
+    }
+
+    // Load table metadata
+    const metadata = await catalog.loadTable(tableId)
+
+    // Parse and resolve timestamp
+    const parsed = parseTimestamp(timestampParam)
+    const resolved = await reader.findSnapshotAtTimestamp(metadata, timestampParam)
+
+    return c.json({
+      input: timestampParam,
+      parsed: {
+        timestampMs: parsed.timestampMs,
+        timestamp: formatTimestamp(parsed.timestampMs),
+        isRelative: parsed.isRelative,
+      },
+      resolved: {
+        snapshotId: resolved.snapshot['snapshot-id'],
+        snapshotTimestampMs: resolved.snapshot['timestamp-ms'],
+        snapshotTimestamp: formatTimestamp(resolved.snapshot['timestamp-ms']),
+        exactMatch: resolved.exactMatch,
+        timeDifferenceMs: resolved.timeDifferenceMs,
+      },
+    })
+  })
+
+  // ========== Analytics Endpoints ==========
+
+  /**
+   * GET /v1/do/{doId}/analytics/change-rate
+   * Get change rate analysis for a time period
+   */
+  app.get(`${prefix}/do/:doId/analytics/change-rate`, async (c) => {
+    const doIdParam = c.req.param('doId')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    const startTime = c.req.query('start_time')
+    const endTime = c.req.query('end_time')
+
+    if (!startTime || !endTime) {
+      return c.json(
+        errorResponse(
+          'Missing required query parameters: start_time, end_time',
+          'BadRequestError',
+          400
+        ),
+        400
+      )
+    }
+
+    try {
+      const analysis = await analytics.getChangeRate(tableId, startTime, endTime)
+      return c.json(analysis)
+    } catch (error) {
+      throw error
+    }
+  })
+
+  /**
+   * GET /v1/do/{doId}/analytics/growth
+   * Get data growth metrics
+   */
+  app.get(`${prefix}/do/:doId/analytics/growth`, async (c) => {
+    const doIdParam = c.req.param('doId')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    const samplingIntervalParam = c.req.query('sampling_interval')
+    const samplingInterval = samplingIntervalParam
+      ? parseInt(samplingIntervalParam, 10)
+      : undefined
+
+    try {
+      const metrics = await analytics.getGrowthMetrics(tableId, samplingInterval)
+      return c.json(metrics)
+    } catch (error) {
+      throw error
+    }
+  })
+
+  /**
+   * GET /v1/do/{doId}/analytics/snapshot-comparison
+   * Compare two snapshots
+   */
+  app.get(`${prefix}/do/:doId/analytics/snapshot-comparison`, async (c) => {
+    const doIdParam = c.req.param('doId')
+    const tableId = parseDoIdentifier(doIdParam)
+
+    const snapshot1Param = c.req.query('snapshot1')
+    const snapshot2Param = c.req.query('snapshot2')
+
+    if (!snapshot1Param || !snapshot2Param) {
+      return c.json(
+        errorResponse(
+          'Missing required query parameters: snapshot1, snapshot2',
+          'BadRequestError',
+          400
+        ),
+        400
+      )
+    }
+
+    const snapshot1Id = parseInt(snapshot1Param, 10)
+    const snapshot2Id = parseInt(snapshot2Param, 10)
+
+    if (isNaN(snapshot1Id) || isNaN(snapshot2Id)) {
+      return c.json(
+        errorResponse(
+          'Invalid snapshot IDs',
+          'BadRequestError',
+          400
+        ),
+        400
+      )
+    }
+
+    try {
+      const comparison = await analytics.getSnapshotComparison(
+        tableId,
+        snapshot1Id,
+        snapshot2Id
+      )
+      return c.json(comparison)
+    } catch (error) {
+      throw error
+    }
+  })
+
+  return app
+}
+
+/**
+ * Create a Hono middleware that adds time travel API routes
+ */
+export function timeTravelApiMiddleware(config: TimeTravelApiConfig): Hono {
+  return createTimeTravelApi(config)
+}