@sixb/ducklake 0.0.1

package/src/internal/ducklake-maintenance.ts

@@ -0,0 +1,116 @@
+import { LakeStorageError } from "@sixb/core/lake-storage"
+import type {
+  DuckLakeMaintenanceOptions,
+  DuckLakeMaintenanceReport,
+  DuckLakeStorageOptions,
+} from "../types"
+import type { DuckDbQueryRuntime } from "./duckdb-runtime"
+import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
+import { duckLakeAlias, quoteSqlString } from "./sql"
+
+const DEFAULT_RETENTION = "7 days"
+
+interface MaintenanceCounts {
+  readonly snapshots: number
+  readonly oldFiles: number
+  readonly orphanedFiles: number
+}
+
+export class DuckLakeMaintenance {
+  constructor(
+    private readonly options: DuckLakeStorageOptions,
+    private readonly connections: DuckLakeConnectionManager
+  ) {}
+
+  async runMaintenance(
+    options: DuckLakeMaintenanceOptions = {}
+  ): Promise<DuckLakeMaintenanceReport> {
+    this.connections.assertOpen()
+
+    const expireOlderThan = options.expireOlderThan ?? DEFAULT_RETENTION
+    const deleteOlderThan = options.deleteOlderThan ?? expireOlderThan
+    const dryRun = options.dryRun ?? false
+
+    if (this.options.readOnly && !dryRun) {
+      throw new LakeStorageError(
+        "[SixbDuckLake] DuckLake maintenance cannot run in read-only mode."
+      )
+    }
+
+    return this.connections.withExclusiveAttached(async (runtime) => {
+      const counts = await runMaintenanceProcedures(
+        runtime,
+        this.options,
+        {
+          expireOlderThan,
+          deleteOlderThan,
+        },
+        dryRun
+      )
+
+      if (!dryRun) {
+        this.connections.markLocalCatalogChanged()
+      }
+
+      return {
+        dryRun,
+        expireOlderThan,
+        deleteOlderThan,
+        ...counts,
+      }
+    })
+  }
+}
+
+async function runMaintenanceProcedures(
+  runtime: DuckDbQueryRuntime,
+  options: DuckLakeStorageOptions,
+  retention: Pick<DuckLakeMaintenanceReport, "expireOlderThan" | "deleteOlderThan">,
+  dryRun: boolean
+): Promise<MaintenanceCounts> {
+  const aliasSql = quoteSqlString(duckLakeAlias(options))
+  const snapshots = await runtime.query(
+    maintenanceProcedureSql(
+      "ducklake_expire_snapshots",
+      aliasSql,
+      retention.expireOlderThan,
+      dryRun
+    )
+  )
+  const oldFiles = await runtime.query(
+    maintenanceProcedureSql(
+      "ducklake_cleanup_old_files",
+      aliasSql,
+      retention.deleteOlderThan,
+      dryRun
+    )
+  )
+  const orphanedFiles = await runtime.query(
+    maintenanceProcedureSql(
+      "ducklake_delete_orphaned_files",
+      aliasSql,
+      retention.deleteOlderThan,
+      dryRun
+    )
+  )
+
+  return {
+    snapshots: snapshots.length,
+    oldFiles: oldFiles.length,
+    orphanedFiles: orphanedFiles.length,
+  }
+}
+
+function maintenanceProcedureSql(
+  procedure: string,
+  aliasSql: string,
+  olderThan: string,
+  dryRun: boolean
+): string {
+  const olderThanSql = quoteSqlString(olderThan)
+  const dryRunSql = dryRun ? "dry_run => true, " : ""
+  return (
+    `CALL ${procedure}(${aliasSql}, ${dryRunSql}` +
+    `older_than => now() - INTERVAL ${olderThanSql})`
+  )
+}

package/src/internal/ducklake-row-reader.ts

@@ -0,0 +1,143 @@
+import type { DatasetColumnDefinition, DatasetRow } from "@sixb/core"
+import type { DatasetVersion, ReadDatasetRowsInput } from "@sixb/core/lake-storage"
+import { LakeStorageError } from "@sixb/core/lake-storage"
+import type { DuckLakeStorageOptions } from "../types"
+import type { DuckDbQueryRuntime } from "./duckdb-runtime"
+import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
+import { type DatasetTableRef, resolveDatasetTableRef } from "./ducklake-dataset-table-ref"
+import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader"
+import { normalizeReadValue } from "./schema"
+import { qualifiedTableName, quoteIdentifier } from "./sql"
+import { parseVersionId } from "./versions"
+
+/**
+ * Reads dataset rows from DuckLake snapshots.
+ *
+ * This module owns the last mile of reads: resolve the requested Sixb version
+ * to a DuckLake snapshot id, render a DuckLake time-travel query, and normalize
+ * DuckDB values back into Sixb row shape.
+ */
+export class DuckLakeRowReader {
+  constructor(
+    private readonly options: DuckLakeStorageOptions,
+    private readonly connections: DuckLakeConnectionManager,
+    private readonly snapshots: DuckLakeSnapshotReader
+  ) {}
+
+  async *readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow> {
+    this.connections.assertOpen()
+
+    const lease = await this.connections.acquireAttachedRuntime()
+    try {
+      const runtime = lease.runtime
+
+      // Step 1: resolve the physical table from DuckLake metadata directly. Row
+      // previews must not go through broad catalog introspection.
+      const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId)
+      if (!tableRef) {
+        throw new LakeStorageError(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`)
+      }
+
+      const version = await this.resolveVersion(runtime, tableRef, input.versionId)
+      const snapshotId = parseVersionId(version.versionId)
+      const selectedColumns = this.resolveReadColumns(
+        tableRef.datasetId,
+        version.schema,
+        input.columns
+      )
+
+      // Step 2: query DuckLake at the exact snapshot id. Version ids are just
+      // `ducklake:<snapshot_id>`, so reads can use native DuckLake time travel
+      // directly after validation.
+      const columnsSql = selectedColumns.map((column) => quoteIdentifier(column.name)).join(", ")
+      const tableSql = qualifiedTableName(this.options, tableRef.tableName)
+      // A user column named `rowid` shadows DuckDB's virtual row id. In that edge case the
+      // connection-level invariant still preserves insertion order; otherwise make it explicit.
+      const orderSql = version.schema.columns.some(
+        (column) => column.name.toLowerCase() === "rowid"
+      )
+        ? ""
+        : " ORDER BY rowid"
+      const limitSql =
+        input.limit === undefined ? "" : ` LIMIT ${Math.max(0, Math.trunc(input.limit))}`
+      const offsetSql =
+        input.offset === undefined ? "" : ` OFFSET ${Math.max(0, Math.trunc(input.offset))}`
+      const sql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})${orderSql}${limitSql}${offsetSql}`
+
+      // HTTP previews are bounded, so materialize them eagerly and release the
+      // runtime queue as soon as DuckDB has returned the small page.
+      const rows = input.limit === undefined ? runtime.streamRows(sql) : await runtime.query(sql)
+
+      // Step 3: DuckDB returns driver-native values. Normalize each projected
+      // column through the schema that was active at the resolved version.
+      yield* this.normalizeRows(rows, selectedColumns)
+    } finally {
+      await lease.release()
+    }
+  }
+
+  private async *normalizeRows(
+    rows: Iterable<Record<string, unknown>> | AsyncIterable<Record<string, unknown>>,
+    selectedColumns: readonly DatasetColumnDefinition[]
+  ): AsyncIterable<DatasetRow> {
+    for await (const row of rows) {
+      const output: Record<string, unknown> = {}
+      for (const column of selectedColumns) {
+        output[column.name] = normalizeReadValue(row[column.name], column)
+      }
+      yield output
+    }
+  }
+
+  private async resolveVersion(
+    runtime: DuckDbQueryRuntime,
+    tableRef: DatasetTableRef,
+    versionId?: string
+  ): Promise<DatasetVersion> {
+    if (versionId !== undefined) {
+      const snapshotId = parseVersionId(versionId)
+      const version = await this.snapshots.getVersionForTableRef(runtime, tableRef, snapshotId)
+      if (!version) {
+        this.throwNoCommittedVersion(tableRef.datasetId)
+      }
+
+      return version
+    }
+
+    const latestVersion = await this.snapshots.getLatestVersionForTableRef(runtime, tableRef)
+    if (!latestVersion) {
+      this.throwNoCommittedVersion(tableRef.datasetId)
+    }
+
+    return latestVersion
+  }
+
+  private resolveReadColumns(
+    datasetId: string,
+    schema: DatasetVersion["schema"],
+    columns?: readonly string[]
+  ): readonly DatasetColumnDefinition[] {
+    if (columns === undefined || columns.length === 0) {
+      return schema.columns
+    }
+
+    const columnsByName = new Map(schema.columns.map((column) => [column.name, column] as const))
+
+    return columns.map((columnName) => {
+      const column = columnsByName.get(columnName)
+      if (!column) {
+        throw new LakeStorageError(
+          `[SixbDuckLake] Dataset '${datasetId}' does not have column '${columnName}' at the requested version.`
+        )
+      }
+
+      return column
+    })
+  }
+
+  private throwNoCommittedVersion(datasetId: string): never {
+    throw new LakeStorageError(
+      `[SixbDuckLake] No committed version found for dataset '${datasetId}'.`
+    )
+  }
+}