lakesync 0.1.0

package/dist/compactor.d.ts

@@ -0,0 +1,342 @@
+import { H as HLCTimestamp, R as Result, L as LakeSyncError, F as FlushError } from './result-CojzlFE2.js';
+import { T as TableSchema } from './types-V_jVu2sA.js';
+import { L as LakeAdapter } from './types-DSC_EiwR.js';
+
+/** Configuration for checkpoint generation */
+interface CheckpointConfig {
+    /** Max raw proto bytes per chunk. Tune to serving runtime memory budget. */
+    chunkBytes: number;
+}
+/** Default checkpoint configuration (16 MB chunks for 128 MB DO) */
+declare const DEFAULT_CHECKPOINT_CONFIG: CheckpointConfig;
+/** Result of a checkpoint generation operation */
+interface CheckpointResult {
+    /** Number of chunk files written */
+    chunksWritten: number;
+    /** Total bytes written across all chunks */
+    bytesWritten: number;
+    /** Snapshot HLC timestamp */
+    snapshotHlc: HLCTimestamp;
+}
+/** Manifest stored alongside checkpoint chunks */
+interface CheckpointManifest {
+    /** Snapshot HLC as decimal string (JSON-safe bigint) */
+    snapshotHlc: string;
+    /** ISO 8601 generation timestamp */
+    generatedAt: string;
+    /** Number of chunks */
+    chunkCount: number;
+    /** Total deltas across all chunks */
+    totalDeltas: number;
+    /** Ordered list of chunk file names */
+    chunks: string[];
+}
+/**
+ * Generates checkpoint files from base Parquet files.
+ *
+ * Reads compacted base files, encodes ALL rows as proto SyncResponse chunks
+ * sized to a configurable byte budget, and writes them to storage. Chunks
+ * contain all rows (not per-user); filtering happens at serve time.
+ */
+declare class CheckpointGenerator {
+    private readonly adapter;
+    private readonly gatewayId;
+    private readonly config;
+    constructor(adapter: LakeAdapter, _schema: TableSchema, gatewayId: string, config?: CheckpointConfig);
+    /**
+     * Generate checkpoint chunks from base Parquet files.
+     *
+     * Reads each base file sequentially, accumulates deltas, and flushes
+     * chunks when the estimated byte size exceeds the configured threshold.
+     *
+     * @param baseFileKeys - Storage keys of the base Parquet files
+     * @param snapshotHlc - The HLC timestamp representing this snapshot point
+     * @returns A Result containing the CheckpointResult, or a LakeSyncError on failure
+     */
+    generate(baseFileKeys: string[], snapshotHlc: HLCTimestamp): Promise<Result<CheckpointResult, LakeSyncError>>;
+    /**
+     * Get all storage keys produced by a checkpoint generation.
+     * Useful for adding to activeKeys in maintenance to prevent orphan removal.
+     */
+    getCheckpointKeys(chunkCount: number): string[];
+    private chunkFileName;
+    private flushChunk;
+}
+
+/** Configuration for the compaction process */
+interface CompactionConfig {
+    /** Minimum number of delta files before compaction triggers */
+    minDeltaFiles: number;
+    /** Maximum number of delta files to compact in one pass */
+    maxDeltaFiles: number;
+    /** Target base file size in bytes */
+    targetFileSizeBytes: number;
+}
+/** Default compaction configuration values */
+declare const DEFAULT_COMPACTION_CONFIG: CompactionConfig;
+/** Result of a compaction operation */
+interface CompactionResult {
+    /** Number of base data files written */
+    baseFilesWritten: number;
+    /** Number of equality delete files written */
+    deleteFilesWritten: number;
+    /** Number of delta files that were compacted */
+    deltaFilesCompacted: number;
+    /** Total bytes read during compaction */
+    bytesRead: number;
+    /** Total bytes written during compaction */
+    bytesWritten: number;
+}
+
+/**
+ * Compacts delta files into consolidated base data files and equality delete files.
+ *
+ * Reads delta Parquet files from the lake adapter, resolves all deltas per row
+ * using LWW (last-writer-wins based on HLC ordering), then writes the final
+ * materialised state back as base files and delete files.
+ */
+declare class Compactor {
+    private readonly adapter;
+    private readonly config;
+    private readonly schema;
+    /**
+     * Create a new Compactor instance.
+     *
+     * @param adapter - The lake adapter for reading/writing Parquet files
+     * @param config - Compaction configuration (thresholds and limits)
+     * @param schema - The table schema describing user-defined columns
+     */
+    constructor(adapter: LakeAdapter, config: CompactionConfig, schema: TableSchema);
+    /**
+     * Compact delta files into base data files.
+     *
+     * Reads delta files from storage, resolves all deltas per row using LWW,
+     * and writes consolidated base files + equality delete files.
+     *
+     * @param deltaFileKeys - Storage keys of the delta Parquet files to compact
+     * @param outputPrefix - Prefix for the output base/delete file keys
+     * @returns A Result containing the CompactionResult, or a LakeSyncError on failure
+     */
+    compact(deltaFileKeys: string[], outputPrefix: string): Promise<Result<CompactionResult, LakeSyncError>>;
+    /**
+     * Read delta files one at a time and incrementally resolve to final row state.
+     *
+     * Memory usage is O(unique rows x columns) rather than O(total deltas),
+     * since each file's deltas are processed and discarded before reading the next.
+     */
+    private readAndResolveIncrementally;
+    /** Write base Parquet file(s) for live rows and equality delete file(s) for deleted rows. */
+    private writeOutputFiles;
+    /**
+     * Generate a timestamp string for output file naming.
+     * Uses the current wall clock time with a random suffix for uniqueness.
+     */
+    private generateTimestamp;
+}
+
+/**
+ * Write an Iceberg equality delete file.
+ *
+ * Contains only the equality columns (table + rowId) needed to identify
+ * deleted rows. The file is encoded as a Parquet file using synthetic
+ * DELETE RowDeltas with no user columns.
+ *
+ * @param deletedRows - Array of row identifiers (table + rowId) for deleted rows
+ * @param _schema - The table schema (reserved for future use with custom equality columns)
+ * @returns A Result containing the Parquet bytes, or a FlushError on failure
+ */
+declare function writeEqualityDeletes(deletedRows: Array<{
+    table: string;
+    rowId: string;
+}>, _schema: TableSchema): Promise<Result<Uint8Array, FlushError>>;
+/**
+ * Read an equality delete file back into row identifiers.
+ *
+ * Deserialises a Parquet equality delete file and extracts the
+ * table + rowId pairs that identify deleted rows.
+ *
+ * @param data - The Parquet file bytes to read
+ * @returns A Result containing the row identifiers, or a FlushError on failure
+ */
+declare function readEqualityDeletes(data: Uint8Array): Promise<Result<Array<{
+    table: string;
+    rowId: string;
+}>, FlushError>>;
+
+/** Configuration for the maintenance cycle */
+interface MaintenanceConfig {
+    /** Number of recent snapshots to retain */
+    retainSnapshots: number;
+    /** Minimum age (ms) before orphaned files can be deleted */
+    orphanAgeMs: number;
+}
+/** Default maintenance configuration values */
+declare const DEFAULT_MAINTENANCE_CONFIG: MaintenanceConfig;
+/** Report produced by a full maintenance cycle */
+interface MaintenanceReport {
+    /** Result of the compaction step */
+    compaction: CompactionResult;
+    /** Number of expired snapshots removed */
+    snapshotsExpired: number;
+    /** Number of orphaned files removed */
+    orphansRemoved: number;
+    /** Result of checkpoint generation (if a generator was configured) */
+    checkpoint?: CheckpointResult;
+}
+/**
+ * Runs a full maintenance cycle: compact, expire snapshots, and clean orphans.
+ *
+ * The runner orchestrates the three maintenance phases in order:
+ * 1. **Compact** — merge delta files into consolidated base/delete files
+ * 2. **Expire** — (reserved for future snapshot expiry logic)
+ * 3. **Clean** — remove orphaned files that are no longer referenced
+ */
+declare class MaintenanceRunner {
+    private readonly compactor;
+    private readonly adapter;
+    private readonly config;
+    private readonly checkpointGenerator;
+    /**
+     * Create a new MaintenanceRunner instance.
+     *
+     * @param compactor - The compactor instance for merging delta files
+     * @param adapter - The lake adapter for storage operations
+     * @param config - Maintenance configuration (retention and age thresholds)
+     * @param checkpointGenerator - Optional checkpoint generator; when provided,
+     *   checkpoints are generated after successful compaction
+     */
+    constructor(compactor: Compactor, adapter: LakeAdapter, config: MaintenanceConfig, checkpointGenerator?: CheckpointGenerator);
+    /**
+     * Run the full maintenance cycle: compact, expire, and clean.
+     *
+     * Compacts delta files into base/delete files, then removes orphaned
+     * storage objects that are no longer referenced by any active data.
+     * Files younger than `orphanAgeMs` are never deleted to avoid races
+     * with in-progress flush operations.
+     *
+     * @param deltaFileKeys - Storage keys of the delta Parquet files to compact
+     * @param outputPrefix - Prefix for the output base/delete file keys
+     * @param storagePrefix - Prefix under which all related storage files live
+     * @returns A Result containing the MaintenanceReport, or a LakeSyncError on failure
+     */
+    run(deltaFileKeys: string[], outputPrefix: string, storagePrefix: string): Promise<Result<MaintenanceReport, LakeSyncError>>;
+    /**
+     * Delete orphaned files not referenced by any active data.
+     *
+     * Lists all files under the given storage prefix, compares each
+     * against the set of active keys, and deletes files that are both
+     * unreferenced and older than `orphanAgeMs`. This age guard
+     * prevents deletion of files created by in-progress flush operations.
+     *
+     * @param storagePrefix - The storage prefix to scan for orphaned files
+     * @param activeKeys - Set of storage keys that must be retained
+     * @returns A Result containing the count of deleted files, or a LakeSyncError on failure
+     */
+    removeOrphans(storagePrefix: string, activeKeys: Set<string>): Promise<Result<number, LakeSyncError>>;
+    /**
+     * Identify orphaned file keys from a list of storage objects.
+     *
+     * A file is considered an orphan if it is not in the active keys set
+     * and its last modification time is older than the configured orphan age.
+     */
+    private findOrphans;
+}
+
+/** Parameters for a single maintenance run */
+interface MaintenanceTask {
+    /** Storage keys of the delta Parquet files to compact */
+    deltaFileKeys: string[];
+    /** Prefix for the output base/delete file keys */
+    outputPrefix: string;
+    /** Prefix under which all related storage files live */
+    storagePrefix: string;
+}
+/**
+ * Provider function that resolves the maintenance task parameters for each run.
+ * Called before every scheduled tick to determine what files to compact.
+ * Return `null` to skip this tick (e.g. when there is nothing to compact).
+ */
+type MaintenanceTaskProvider = () => Promise<MaintenanceTask | null>;
+/** Configuration for the compaction scheduler */
+interface SchedulerConfig {
+    /** Interval between maintenance runs in milliseconds (default 60000) */
+    intervalMs: number;
+    /** Whether the scheduler is enabled (default true) */
+    enabled: boolean;
+}
+/** Default scheduler configuration values */
+declare const DEFAULT_SCHEDULER_CONFIG: SchedulerConfig;
+/**
+ * Manages interval-based compaction scheduling.
+ *
+ * Wraps a {@link MaintenanceRunner} and executes maintenance cycles on a
+ * configurable interval. The scheduler is safe against concurrent runs:
+ * if a previous tick is still in progress when the next fires, the tick
+ * is silently skipped.
+ */
+declare class CompactionScheduler {
+    private readonly runner;
+    private readonly taskProvider;
+    private readonly config;
+    private timer;
+    private running;
+    private inFlightPromise;
+    /**
+     * Create a new CompactionScheduler instance.
+     *
+     * @param runner - The maintenance runner to execute on each tick
+     * @param taskProvider - Function that provides maintenance task parameters for each run
+     * @param config - Scheduler configuration (interval and enabled flag)
+     */
+    constructor(runner: MaintenanceRunner, taskProvider: MaintenanceTaskProvider, config?: Partial<SchedulerConfig>);
+    /**
+     * Whether the scheduler is currently active (timer is ticking).
+     */
+    get isRunning(): boolean;
+    /**
+     * Start the scheduler interval timer.
+     *
+     * Begins executing maintenance runs at the configured interval.
+     * If the scheduler is already running or disabled, returns an error.
+     *
+     * @returns A Result indicating success or a descriptive error
+     */
+    start(): Result<void, LakeSyncError>;
+    /**
+     * Stop the scheduler and wait for any in-progress run to finish.
+     *
+     * Clears the interval timer and, if a maintenance run is currently
+     * executing, awaits its completion before returning.
+     *
+     * @returns A Result indicating success or a descriptive error
+     */
+    stop(): Promise<Result<void, LakeSyncError>>;
+    /**
+     * Manually trigger a single maintenance run.
+     *
+     * Useful for testing or administrative purposes. If a run is already
+     * in progress, skips and returns an error.
+     *
+     * @returns A Result containing the MaintenanceReport, or a LakeSyncError on failure
+     */
+    runOnce(): Promise<Result<MaintenanceReport, LakeSyncError>>;
+    /**
+     * Internal tick handler called by the interval timer.
+     * Skips if a previous run is still in progress.
+     */
+    private tick;
+    /**
+     * Execute a single maintenance cycle.
+     *
+     * Calls the task provider to get parameters, then runs the maintenance
+     * runner. Tracks the in-flight promise so concurrent runs are prevented.
+     */
+    private executeMaintenance;
+    /**
+     * Resolve the maintenance task from the provider, wrapping any thrown
+     * exceptions into a Result error.
+     */
+    private resolveTask;
+}
+
+export { type CheckpointConfig, CheckpointGenerator, type CheckpointManifest, type CheckpointResult, type CompactionConfig, type CompactionResult, CompactionScheduler, Compactor, DEFAULT_CHECKPOINT_CONFIG, DEFAULT_COMPACTION_CONFIG, DEFAULT_MAINTENANCE_CONFIG, DEFAULT_SCHEDULER_CONFIG, type MaintenanceConfig, type MaintenanceReport, MaintenanceRunner, type MaintenanceTask, type MaintenanceTaskProvider, type SchedulerConfig, readEqualityDeletes, writeEqualityDeletes };