@dotdo/postgres 0.1.0

package/dist/recovery/disaster-recovery.d.ts

@@ -0,0 +1,326 @@
+/**
+ * Disaster Recovery from Iceberg Snapshots
+ * Task: postgres-90v - Disaster recovery from Iceberg
+ *
+ * Provides disaster recovery capabilities by restoring Durable Objects from
+ * Iceberg snapshots. When a DO is lost or corrupted, this module can:
+ * - Restore to a specific point in time using Iceberg snapshots
+ * - Perform full database restoration
+ * - Perform partial restoration (specific tables)
+ * - Orchestrate recovery across multiple DOs
+ * - Verify recovery integrity
+ *
+ * Recovery Process:
+ * 1. Find Iceberg snapshot at or before target timestamp
+ * 2. Read all data files for the snapshot
+ * 3. Replay data into new/existing DO
+ * 4. Verify row counts and checksums
+ *
+ * @example
+ * ```typescript
+ * const recovery = createDisasterRecovery({
+ *   bucket: env.ICEBERG_BUCKET,
+ *   tablePath: 'iceberg/wal',
+ * })
+ *
+ * await recovery.initialize()
+ *
+ * // Restore DO to specific point in time
+ * const result = await recovery.recover('do-123', '2024-01-15T10:30:00Z')
+ *
+ * // Restore specific table
+ * await recovery.recoverTable('do-123', 'users', '2024-01-15T10:30:00Z')
+ *
+ * // List available recovery points
+ * const points = await recovery.listRecoveryPoints('do-123')
+ *
+ * // Validate recovery
+ * const validation = await recovery.validateRecovery('do-123')
+ * ```
+ */
+import type { IcebergTableMetadata } from '../iceberg/types';
+import { IcebergTimeTravelReader, type TimestampSpec } from '../iceberg/time-travel';
+import type { SerializedWALEntry } from '../worker/wal';
+export type { IcebergSnapshot, IcebergManifestFile, IcebergManifestEntry } from '../iceberg/types';
+export type { ParsedTimestamp } from '../iceberg/time-travel';
+export { TimestampOutOfRangeError, NoSnapshotFoundError } from '../iceberg/time-travel';
+export type { WALOperation } from '../worker/wal';
+import type { ReplayExecutor } from '../worker/wal-replay';
+/**
+ * Recovery point metadata
+ */
+export interface RecoveryPoint {
+    /** Snapshot ID */
+    snapshotId: number;
+    /** Snapshot timestamp */
+    timestampMs: number;
+    /** Formatted timestamp */
+    timestamp: string;
+    /** Number of data files */
+    dataFileCount: number;
+    /** Total records in snapshot */
+    totalRecords: number;
+    /** Total size in bytes */
+    totalSizeBytes: number;
+    /** DO ID (if filtered by DO) */
+    doId?: string;
+}
+/**
+ * Recovery target specification
+ */
+export interface RecoveryTarget {
+    /** Target timestamp */
+    timestamp: TimestampSpec;
+    /** DO ID to recover */
+    doId: string;
+    /** Tables to recover (empty = all tables) */
+    tables?: string[];
+    /** Schemas to recover (empty = all schemas) */
+    schemas?: string[];
+}
+/**
+ * Recovery progress event
+ */
+export interface RecoveryProgressEvent {
+    /** Current phase */
+    phase: 'initializing' | 'loading_snapshot' | 'loading_data' | 'replaying' | 'verifying' | 'completed' | 'failed';
+    /** Current progress (0-100) */
+    progress: number;
+    /** Current file being processed */
+    currentFile: number;
+    /** Total files to process */
+    totalFiles: number;
+    /** Records processed */
+    recordsProcessed: number;
+    /** Total records to process */
+    totalRecords: number;
+    /** Elapsed time in milliseconds */
+    elapsedMs: number;
+    /** Estimated remaining time in milliseconds */
+    estimatedRemainingMs: number | null;
+    /** Error message if failed */
+    error?: string;
+}
+/**
+ * Recovery progress listener
+ */
+export type RecoveryProgressListener = (event: RecoveryProgressEvent) => void;
+/**
+ * Recovery statistics
+ */
+export interface RecoveryStats {
+    /** Snapshot used for recovery */
+    snapshotId: number;
+    /** Snapshot timestamp */
+    snapshotTimestampMs: number;
+    /** Requested timestamp */
+    requestedTimestampMs: number;
+    /** Files read */
+    filesRead: number;
+    /** Records recovered */
+    recordsRecovered: number;
+    /** Records skipped (filtered) */
+    recordsSkipped: number;
+    /** Records failed */
+    recordsFailed: number;
+    /** Tables recovered */
+    tablesRecovered: string[];
+    /** Total duration in milliseconds */
+    durationMs: number;
+    /** Throughput (records per second) */
+    throughputRecordsPerSec: number;
+}
+/**
+ * Recovery result
+ */
+export interface RecoveryResult {
+    /** Whether recovery succeeded */
+    success: boolean;
+    /** Recovery statistics */
+    stats: RecoveryStats;
+    /** Error message if failed */
+    error?: string;
+    /** Failed records with errors */
+    failedRecords?: Array<{
+        record: SerializedWALEntry;
+        error: string;
+    }>;
+}
+/**
+ * Recovery validation result
+ */
+export interface RecoveryValidationResult {
+    /** Whether validation passed */
+    valid: boolean;
+    /** Validation checks performed */
+    checks: {
+        /** Row count matches expected */
+        rowCountMatch: boolean;
+        /** Expected row count from snapshot */
+        expectedRowCount: number;
+        /** Actual row count in DO */
+        actualRowCount: number;
+        /** Tables verified */
+        tablesVerified: string[];
+        /** Tables with mismatches */
+        tablesMismatch: string[];
+    };
+    /** Error message if validation failed */
+    error?: string;
+}
+/**
+ * Disaster recovery configuration
+ */
+export interface DisasterRecoveryConfig {
+    /** R2 bucket for Iceberg data */
+    bucket: R2Bucket;
+    /** Iceberg table path */
+    tablePath: string;
+    /** Batch size for reading data files (default: 100) */
+    batchSize?: number;
+    /** Continue on error (default: false) */
+    continueOnError?: boolean;
+    /** Dry run mode (validate but don't execute) */
+    dryRun?: boolean;
+    /** Progress callback */
+    onProgress?: RecoveryProgressListener;
+    /** Progress report interval in ms (default: 1000) */
+    progressIntervalMs?: number;
+}
+/**
+ * Multi-DO recovery configuration
+ */
+export interface MultiDORecoveryConfig {
+    /** DO IDs to recover */
+    doIds: string[];
+    /** Target timestamp for recovery */
+    timestamp: TimestampSpec;
+    /** Maximum concurrent recoveries (default: 5) */
+    maxConcurrent?: number;
+    /** Continue if one DO fails (default: true) */
+    continueOnError?: boolean;
+    /** Progress callback */
+    onProgress?: (doId: string, event: RecoveryProgressEvent) => void;
+}
+/**
+ * Multi-DO recovery result
+ */
+export interface MultiDORecoveryResult {
+    /** Overall success */
+    success: boolean;
+    /** Results per DO */
+    results: Map<string, RecoveryResult>;
+    /** DOs that failed */
+    failedDOs: string[];
+    /** Total duration in milliseconds */
+    durationMs: number;
+}
+/**
+ * Disaster Recovery Manager
+ *
+ * Provides disaster recovery capabilities from Iceberg snapshots.
+ */
+export declare class DisasterRecovery {
+    private bucket;
+    private tablePath;
+    /** @internal Reserved for future batch processing */
+    private _batchSize;
+    private continueOnError;
+    private dryRun;
+    /** @internal Reserved for future progress reporting */
+    private _progressIntervalMs;
+    private onProgress?;
+    private timeTravelReader;
+    private metadata;
+    private initialized;
+    private parquetParser;
+    constructor(config: DisasterRecoveryConfig);
+    /**
+     * Initialize the disaster recovery manager
+     */
+    initialize(): Promise<void>;
+    /**
+     * Ensure initialized
+     */
+    private ensureInitialized;
+    /**
+     * Load table metadata from R2
+     */
+    private loadMetadata;
+    /**
+     * Recover a DO to a specific point in time
+     */
+    recover(doId: string, timestamp: TimestampSpec, executor: ReplayExecutor): Promise<RecoveryResult>;
+    /**
+     * Recover a specific table
+     */
+    recoverTable(doId: string, table: string, timestamp: TimestampSpec, executor: ReplayExecutor): Promise<RecoveryResult>;
+    /**
+     * List available recovery points for a DO
+     */
+    listRecoveryPoints(doId?: string): Promise<RecoveryPoint[]>;
+    /**
+     * Validate recovery integrity
+     */
+    validateRecovery(doId: string, timestamp: TimestampSpec, executor: ReplayExecutor): Promise<RecoveryValidationResult>;
+    /**
+     * Recover multiple DOs in parallel
+     */
+    recoverMultipleDOs(config: MultiDORecoveryConfig, executorFactory: (doId: string) => ReplayExecutor): Promise<MultiDORecoveryResult>;
+    /**
+     * Check if a data file matches a DO ID
+     */
+    private fileMatchesDoId;
+    /**
+     * Check if a data file matches a table
+     */
+    private fileMatchesTable;
+    /**
+     * Extract table name from data file
+     */
+    private extractTableFromFile;
+    /**
+     * Replay data files into DO
+     */
+    private replayDataFiles;
+    /**
+     * Parse Parquet file and extract WAL entries
+     *
+     * Uses the ParquetParser to:
+     * 1. Validate magic bytes (PAR1 header and footer)
+     * 2. Extract footer metadata
+     * 3. Parse data region (supports JSON-encoded test format and production format)
+     * 4. Decompress if needed (GZIP, Snappy, etc.)
+     * 5. Convert to SerializedWALEntry format
+     *
+     * @param data - The Parquet file contents as an ArrayBuffer
+     * @returns Array of SerializedWALEntry records
+     * @throws {ParquetParseError} If the file is invalid or corrupted
+     */
+    private parseParquetFile;
+    /**
+     * Replay a single WAL operation
+     */
+    private replayOperation;
+    /**
+     * Report progress
+     */
+    private reportProgress;
+    /**
+     * Create empty stats object
+     */
+    private createEmptyStats;
+    /**
+     * Get the Iceberg metadata
+     */
+    getMetadata(): IcebergTableMetadata | null;
+    /**
+     * Get the time travel reader
+     */
+    getTimeTravelReader(): IcebergTimeTravelReader;
+}
+/**
+ * Create a disaster recovery manager
+ */
+export declare function createDisasterRecovery(config: DisasterRecoveryConfig): DisasterRecovery;
+//# sourceMappingURL=disaster-recovery.d.ts.map

package/dist/recovery/disaster-recovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"disaster-recovery.d.ts","sourceRoot":"","sources":["../../src/recovery/disaster-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAAmB,MAAM,kBAAkB,CAAA;AAC7E,OAAO,EACL,uBAAuB,EAEvB,KAAK,aAAa,EAGnB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAKvD,YAAY,EAAE,eAAe,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AAClG,YAAY,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AAC7D,OAAO,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAA;AACvF,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAEjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AAgC1D;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAA;IACnB,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAA;IACjB,2BAA2B;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,gCAAgC;IAChC,YAAY,EAAE,MAAM,CAAA;IACpB,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAA;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,uBAAuB;IACvB,SAAS,EAAE,aAAa,CAAA;IACxB,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,6CAA6C;IAC7C,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oBAAoB;IACpB,KAAK,EAAE,cAAc,GAAG,kBAAkB,GAAG,cAAc,GAAG,WAAW,GAAG,WAAW,GAAG,WAAW,GAAG,QAAQ,CAAA;IAChH,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,mCAAmC;IACnC,WAAW,EAAE,MAAM,CAAA;IACnB,6BAA6B;IAC7B,UAAU,EAAE,MAAM,CAAA;IAClB,wBAAwB;IACxB,gBAAgB,EAAE,MAAM,CAAA;IACxB,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAA;IACjB,+CAA+C;IAC/C,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAA;IACnC,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,KAAK,EAAE,qBAAqB,KAAK,IAAI,CAAA;AAE7E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAA;IAClB,yBAAyB;IACzB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,0BAA0B;IAC1B,oBAAoB,EAAE,MAAM,CAAA;IAC5B,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,wBAAwB;IACxB,gBAAgB,EAAE,MAAM,CAAA;IACxB,iCAAiC;IACjC,cAAc,EAAE,MAAM,CAAA;IACtB,qBAAqB;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,uBAAuB;IACvB,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAA;IAClB,sCAAsC;IACtC,uBAAuB,EAAE,MAAM,CAAA;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,iCAAiC;IACjC,OAAO,EAAE,OAAO,CAAA;IAChB,0BAA0B;IAC1B,KAAK,EAAE,aAAa,CAAA;IACpB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,kBAAkB,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACrE;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAA;IACd,kCAAkC;IAClC,MAAM,EAAE;QACN,iCAAiC;QACjC,aAAa,EAAE,OAAO,CAAA;QACtB,uCAAuC;QACvC,gBAAgB,EAAE,MAAM,CAAA;QACxB,6BAA6B;QAC7B,cAAc,EAAE,MAAM,CAAA;QACtB,sBAAsB;QACtB,cAAc,EAAE,MAAM,EAAE,CAAA;QACxB,6BAA6B;QAC7B,cAAc,EAAE,MAAM,EAAE,CAAA;KACzB,CAAA;IACD,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,iCAAiC;IACjC,MAAM,EAAE,QAAQ,CAAA;IAChB,yBAAyB;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,yCAAyC;IACzC,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,gDAAgD;IAChD,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,wBAAwB;IACxB,UAAU,CAAC,EAAE,wBAAwB,CAAA;IACrC,qDAAqD;IACrD,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,wBAAwB;IACxB,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,oCAAoC;IACpC,SAAS,EAAE,aAAa,CAAA;IACxB,iDAAiD;IACjD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,+CAA+C;IAC/C,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,wBAAwB;IACxB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,qBAAqB,KAAK,IAAI,CAAA;CAClE;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,sBAAsB;IACtB,OAAO,EAAE,OAAO,CAAA;IAChB,qBAAqB;IACrB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IACpC,sBAAsB;IACtB,SAAS,EAAE,MAAM,EAAE,CAAA;IACnB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAA;CACnB;AAsBD;;;;GAIG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,SAAS,CAAQ;IACzB,qDAAqD;IACrD,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,MAAM,CAAS;IACvB,uDAAuD;IACvD,OAAO,CAAC,mBAAmB,CAAQ;IACnC,OAAO,CAAC,UAAU,CAAC,CAA0B;IAE7C,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,QAAQ,CAAoC;IACpD,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,aAAa,CAAe;gBAExB,MAAM,EAAE,sBAAsB;IAsB1C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IASjC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAMzB;;OAEG;YACW,YAAY;IA4C1B;;OAEG;IACG,OAAO,CACX,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,aAAa,EACxB,QAAQ,EAAE,cAAc,GACvB,OAAO,CAAC,cAAc,CAAC;IA2J1B;;OAEG;IACG,YAAY,CAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,aAAa,EACxB,QAAQ,EAAE,cAAc,GACvB,OAAO,CAAC,cAAc,CAAC;IAuE1B;;OAEG;IACG,kBAAkB,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA0CjE;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,aAAa,EACxB,QAAQ,EAAE,cAAc,GACvB,OAAO,CAAC,wBAAwB,CAAC;IA+FpC;;OAEG;IACG,kBAAkB,CACtB,MAAM,EAAE,qBAAqB,EAC7B,eAAe,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,cAAc,GAChD,OAAO,CAAC,qBAAqB,CAAC;IA6EjC;;OAEG;IACH,OAAO,CAAC,eAAe;IAcvB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAW5B;;OAEG;YACW,eAAe;IAoI7B;;;;;;;;;;;;;OAaG;YACW,gBAAgB;IAY9B;;OAEG;YACW,eAAe;IA2C7B;;OAEG;IACH,OAAO,CAAC,cAAc;IAMtB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAoBxB;;OAEG;IACH,WAAW,IAAI,oBAAoB,GAAG,IAAI;IAI1C;;OAEG;IACH,mBAAmB,IAAI,uBAAuB;CAG/C;AAMD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,sBAAsB,GAAG,gBAAgB,CAEvF"}