@dotdo/do 0.1.0

package/dist/db/cdc/replay.d.ts

@@ -0,0 +1,388 @@
+/**
+ * CDC Event Replay and Recovery
+ *
+ * Provides cursor-based replay of CDC events from any point in time.
+ * Supports checkpoint management for reliable event processing.
+ *
+ * @module cdc/replay
+ */
+import type { CDCEvent, CDCCursor, CDCBatch, CDCOptions } from '../../types/storage';
+import type { CDCStorage } from './storage';
+/**
+ * Checkpoint representing a subscriber's progress
+ */
+export interface Checkpoint {
+    /**
+     * Unique subscriber identifier
+     */
+    subscriberId: string;
+    /**
+     * Current cursor position
+     */
+    cursor: CDCCursor;
+    /**
+     * When the checkpoint was created
+     */
+    createdAt: number;
+    /**
+     * When the checkpoint was last updated
+     */
+    updatedAt: number;
+    /**
+     * Optional metadata about the subscriber
+     */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Options for replay operations
+ */
+export interface ReplayOptions extends CDCOptions {
+    /**
+     * Subscriber ID for checkpoint management
+     */
+    subscriberId?: string;
+    /**
+     * Whether to auto-commit checkpoints after each batch
+     * @default true
+     */
+    autoCommit?: boolean;
+    /**
+     * Callback for progress updates
+     */
+    onProgress?: (cursor: CDCCursor, eventsProcessed: number) => void;
+    /**
+     * Maximum time in ms for the entire replay
+     */
+    timeout?: number;
+    /**
+     * Whether to include events from cold storage (R2)
+     * @default true
+     */
+    includeCold?: boolean;
+    /**
+     * Whether to include events from hot storage (SQLite)
+     * @default true
+     */
+    includeHot?: boolean;
+    /**
+     * Start from this timestamp (inclusive)
+     */
+    fromTimestamp?: number;
+    /**
+     * Start from this sequence number (inclusive)
+     */
+    fromSequence?: number;
+}
+/**
+ * Replay progress information
+ */
+export interface ReplayProgress {
+    /**
+     * Current cursor position
+     */
+    cursor: CDCCursor;
+    /**
+     * Total events replayed so far
+     */
+    eventsReplayed: number;
+    /**
+     * Estimated total events (may be approximate)
+     */
+    estimatedTotal?: number;
+    /**
+     * Percentage complete (0-100)
+     */
+    percentComplete?: number;
+    /**
+     * Current storage tier being read
+     */
+    currentTier: 'hot' | 'cold';
+    /**
+     * Whether replay is complete
+     */
+    isComplete: boolean;
+}
+/**
+ * EventReplayer - Replays CDC events from any point in time
+ *
+ * @example
+ * ```typescript
+ * const replayer = new EventReplayer(storage, hotEvents)
+ *
+ * // Replay from 24 hours ago
+ * const cursor = {
+ *   sequence: 0,
+ *   timestamp: Date.now() - 86400000,
+ * }
+ *
+ * for await (const batch of replayer.replay({ fromCursor: cursor })) {
+ *   for (const event of batch.events) {
+ *     await processEvent(event)
+ *   }
+ * }
+ * ```
+ */
+/**
+ * Options for EventReplayer constructor
+ */
+export interface EventReplayerOptions {
+    /**
+     * Cold storage (R2/Iceberg)
+     */
+    coldStorage: CDCStorage;
+    /**
+     * Hot storage (DO SQLite)
+     */
+    hotStorage: HotStorageInterface;
+}
+export declare class EventReplayer {
+    private coldStorage;
+    private hotStorage;
+    /**
+     * Creates a new event replayer
+     *
+     * @param options - Configuration options including storage interfaces
+     */
+    constructor(options: EventReplayerOptions);
+    /**
+     * Replays events as an async iterator
+     *
+     * Merges hot and cold storage transparently, maintaining order.
+     *
+     * @param options - Replay options
+     * @yields Batches of events
+     *
+     * @example
+     * ```typescript
+     * for await (const batch of replayer.replay({
+     *   fromCursor: checkpoint.cursor,
+     *   collections: ['users', 'orders'],
+     *   batchSize: 100,
+     * })) {
+     *   await processBatch(batch)
+     *   console.log(`Processed up to sequence ${batch.cursor.sequence}`)
+     * }
+     * ```
+     */
+    replay(options?: ReplayOptions): AsyncGenerator<CDCBatch>;
+    /**
+     * Replays events with a callback (non-iterator pattern)
+     *
+     * @param handler - Callback for each batch
+     * @param options - Replay options
+     * @returns Final cursor position
+     *
+     * @example
+     * ```typescript
+     * const finalCursor = await replayer.replayWithHandler(
+     *   async (batch) => {
+     *     for (const event of batch.events) {
+     *       await processEvent(event)
+     *     }
+     *   },
+     *   { subscriberId: 'my-processor' }
+     * )
+     * ```
+     */
+    replayWithHandler(handler: (batch: CDCBatch) => Promise<void>, options?: ReplayOptions): Promise<CDCCursor>;
+    /**
+     * Gets a single batch of events
+     *
+     * @param options - Query options
+     * @returns Single batch with cursor for next page
+     */
+    getBatch(options?: ReplayOptions): Promise<CDCBatch>;
+    /**
+     * Gets current replay progress
+     *
+     * @param cursor - Current cursor position
+     * @returns Progress information
+     */
+    getProgress(cursor: CDCCursor): Promise<ReplayProgress>;
+    /**
+     * Estimates time to complete replay from cursor
+     *
+     * @param fromCursor - Starting cursor
+     * @param options - Options including events per second
+     * @returns Estimated replay time
+     */
+    estimateReplayTime(fromCursor: CDCCursor, options: {
+        eventsPerSecond: number;
+    }): Promise<{
+        estimatedSeconds: number;
+    }>;
+}
+/**
+ * Interface for hot storage (DO SQLite) access
+ */
+export interface HotStorageInterface {
+    /**
+     * Gets events from hot storage
+     */
+    getEvents(options: CDCOptions): Promise<CDCEvent[]>;
+    /**
+     * Gets the current sequence number
+     */
+    getCurrentSequence(): Promise<number>;
+    /**
+     * Gets the oldest sequence still in hot storage
+     */
+    getOldestSequence(): Promise<number>;
+}
+/**
+ * CheckpointManager - Manages subscriber checkpoints
+ *
+ * @example
+ * ```typescript
+ * const checkpoints = new CheckpointManager(storage)
+ *
+ * // Save progress
+ * await checkpoints.save('my-processor', batch.cursor)
+ *
+ * // Resume from checkpoint
+ * const checkpoint = await checkpoints.get('my-processor')
+ * for await (const batch of replayer.replay({
+ *   fromCursor: checkpoint?.cursor,
+ * })) {
+ *   await processBatch(batch)
+ *   await checkpoints.save('my-processor', batch.cursor)
+ * }
+ * ```
+ */
+export declare class CheckpointManager {
+    private storage;
+    /**
+     * Creates a new checkpoint manager
+     *
+     * @param storage - Storage interface for checkpoints
+     */
+    constructor(storage: CheckpointStorage);
+    /**
+     * Saves a checkpoint for a subscriber
+     *
+     * @param subscriberId - Unique subscriber ID
+     * @param cursor - Current cursor position
+     * @param metadata - Optional metadata
+     * @returns The saved checkpoint
+     */
+    save(subscriberId: string, cursor: CDCCursor, metadata?: Record<string, unknown>): Promise<Checkpoint>;
+    /**
+     * Gets a checkpoint for a subscriber
+     *
+     * @param subscriberId - Subscriber ID
+     * @returns The checkpoint or null if not found
+     */
+    get(subscriberId: string): Promise<Checkpoint | null>;
+    /**
+     * Deletes a checkpoint
+     *
+     * @param subscriberId - Subscriber ID
+     * @returns Whether the checkpoint existed
+     */
+    delete(subscriberId: string): Promise<boolean>;
+    /**
+     * Lists all checkpoints
+     *
+     * @returns All checkpoints
+     */
+    list(): Promise<Checkpoint[]>;
+    /**
+     * Gets checkpoints that are behind by more than threshold
+     *
+     * @param currentCursor - Current position
+     * @param options - Options including minimum lag sequence
+     * @returns Checkpoints that are lagging
+     */
+    getLagging(currentCursor: CDCCursor, options: {
+        minLagSequence: number;
+    }): Promise<Checkpoint[]>;
+}
+/**
+ * Interface for checkpoint storage
+ */
+export interface CheckpointStorage {
+    get(subscriberId: string): Promise<Checkpoint | null>;
+    put(checkpoint: Checkpoint): Promise<Checkpoint>;
+    delete(subscriberId: string): Promise<boolean>;
+    list(): Promise<Checkpoint[]>;
+}
+/**
+ * Subscription handle returned by createSubscription
+ */
+export interface SubscriptionHandle {
+    /**
+     * Subscriber ID
+     */
+    subscriberId: string;
+    /**
+     * Current status
+     */
+    status: 'running' | 'paused' | 'stopped';
+    /**
+     * Starts the subscription
+     */
+    start: () => Promise<void>;
+    /**
+     * Stops the subscription
+     */
+    stop: () => Promise<void>;
+    /**
+     * Pauses the subscription
+     */
+    pause: () => Promise<void>;
+    /**
+     * Resumes the subscription
+     */
+    resume: () => Promise<void>;
+}
+/**
+ * Creates a replay subscription that handles backpressure
+ *
+ * @param options - Subscription options including handler
+ * @returns Subscription handle
+ *
+ * @example
+ * ```typescript
+ * const subscription = createSubscription({
+ *   subscriberId: 'analytics-worker',
+ *   handler: async (batch) => {
+ *     await sendToAnalytics(batch.events)
+ *   },
+ *   replayer,
+ * })
+ *
+ * await subscription.start()
+ * // Later...
+ * await subscription.stop()
+ * ```
+ */
+export declare function createSubscription(options: {
+    subscriberId: string;
+    handler: (batch: CDCBatch) => Promise<void>;
+    replayer?: EventReplayer;
+    checkpointManager?: CheckpointManager;
+    concurrency?: number;
+    onError?: (error: Error, batch?: CDCBatch) => Promise<void>;
+}): SubscriptionHandle;
+/**
+ * Merges events from hot and cold storage in sequence order
+ *
+ * @param hotEvents - Events from hot storage
+ * @param coldEvents - Events from cold storage
+ * @returns Merged events in sequence order
+ */
+export declare function mergeEventStreams(hotEvents: CDCEvent[], coldEvents: CDCEvent[]): CDCEvent[];
+/**
+ * Validates that events are in correct sequence order
+ *
+ * @param events - Events to validate
+ * @returns Validation result with any gaps found
+ */
+export declare function validateSequence(events: CDCEvent[]): {
+    valid: boolean;
+    gaps: Array<{
+        start: number;
+        end: number;
+    }>;
+};
+//# sourceMappingURL=replay.d.ts.map

package/dist/db/cdc/replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../../../db/cdc/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAmB,MAAM,qBAAqB,CAAA;AAErG,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,WAAW,CAAA;AAE7D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,MAAM,EAAE,SAAS,CAAA;IAEjB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,aAAc,SAAQ,UAAU;IAC/C;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,KAAK,IAAI,CAAA;IAEjE;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;IAErB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,MAAM,EAAE,SAAS,CAAA;IAEjB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;IAEtB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;IAEvB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IAExB;;OAEG;IACH,WAAW,EAAE,KAAK,GAAG,MAAM,CAAA;IAE3B;;OAEG;IACH,UAAU,EAAE,OAAO,CAAA;CACpB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,WAAW,EAAE,UAAU,CAAA;IAEvB;;OAEG;IACH,UAAU,EAAE,mBAAmB,CAAA;CAChC;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,UAAU,CAAqB;IAEvC;;;;OAIG;gBACS,OAAO,EAAE,oBAAoB;IAKzC;;;;;;;;;;;;;;;;;;;OAmBG;IACI,MAAM,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,cAAc,CAAC,QAAQ,CAAC;IAyBhE;;;;;;;;;;;;;;;;;;OAkBG;IACG,iBAAiB,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;IAiBjH;;;;;OAKG;IACG,QAAQ,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC;IAoG1D;;;;;OAKG;IACG,WAAW,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC;IAsB7D;;;;;;OAMG;IACG,kBAAkB,CAAC,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE;QAAE,eAAe,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,gBAAgB,EAAE,MAAM,CAAA;KAAE,CAAC;CAO7H;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAA;IAEnD;;OAEG;IACH,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;IAErC;;OAEG;IACH,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;CACrC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAmB;IAElC;;;;OAIG;gBACS,OAAO,EAAE,iBAAiB;IAItC;;;;;;;OAOG;IACG,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAe5G;;;;;OAKG;IACG,GAAG,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAI3D;;;;;OAKG;IACG,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIpD;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IAInC;;;;;;OAMG;IACG,UAAU,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;CAIvG;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,GAAG,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAA;IACrD,GAAG,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IAChD,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAC9C,IAAI,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IAEpB;;OAEG;IACH,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAA;IAExC;;OAEG;IACH,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAE1B;;OAEG;IACH,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAEzB;;OAEG;IACH,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAE1B;;OAEG;IACH,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE;IAC1C,YAAY,EAAE,MAAM,CAAA;IACpB,OAAO,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC3C,QAAQ,CAAC,EAAE,aAAa,CAAA;IACxB,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;IACrC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAC5D,GAAG,kBAAkB,CA0FrB;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAuB3F;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG;IACpD,KAAK,EAAE,OAAO,CAAA;IACd,IAAI,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAC5C,CA0BA"}