@sixb/ducklake 0.1.5 → 0.1.7

package/src/internal/ducklake-change-reader.ts

@@ -0,0 +1,157 @@
+import { randomUUID } from "node:crypto"
+import type { DatasetColumnDefinition, DatasetRow } from "@sixb/core"
+import {
+  type DatasetChanges,
+  type DatasetRowChange,
+  LakeStorageError,
+  type ReadDatasetChangesInput,
+  resolveDatasetChangeColumns,
+} from "@sixb/core/lake-storage"
+import type { DuckLakeStorageOptions } from "../types"
+import { getBigIntLike } from "./duckdb-row"
+import type { DuckDbQueryRuntime } from "./duckdb-runtime"
+import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
+import { resolveDatasetTableRef } from "./ducklake-dataset-table-ref"
+import type { DuckLakeSnapshotReader } from "./ducklake-snapshot-reader"
+import { normalizeReadValue } from "./schema"
+import { qualifiedTableName, quoteIdentifier } from "./sql"
+import { parseVersionId } from "./versions"
+
+const PAGE_ROWS = 5_000
+
+/** Compare once in DuckDB, then page only the delta without holding the connection between yields. */
+export class DuckLakeChangeReader {
+  constructor(
+    private readonly options: DuckLakeStorageOptions,
+    private readonly connections: DuckLakeConnectionManager,
+    private readonly snapshots: DuckLakeSnapshotReader
+  ) {}
+
+  async readChanges(input: ReadDatasetChangesInput): Promise<DatasetChanges | null> {
+    input.signal?.throwIfAborted()
+    const before = await this.snapshots.getVersion(input.datasetId, input.fromVersionId)
+    const after = await this.snapshots.getVersion(input.datasetId, input.toVersionId)
+    const columns = resolveDatasetChangeColumns(input, before, after)
+    if (!columns) return null
+    const lease = await this.connections.acquireAttachedRuntime()
+    const runtime = lease.runtime
+    const temporary = quoteIdentifier(`sixb_delta_${randomUUID().replaceAll("-", "")}`)
+    let retained = false
+    let created = false
+    try {
+      const ref = await resolveDatasetTableRef(this.options, runtime, input.datasetId)
+      if (!ref) return null
+      const table = qualifiedTableName(this.options, ref.tableName)
+      const oldRelation = `${table} AT (VERSION => ${parseVersionId(input.fromVersionId)})`
+      const newRelation = `${table} AT (VERSION => ${parseVersionId(input.toVersionId)})`
+      const previousCount = await uniqueRowCount(runtime, oldRelation, input.keyColumns)
+      const nextCount = await uniqueRowCount(runtime, newRelation, input.keyColumns)
+      if (previousCount === null || nextCount === null) return null
+      input.signal?.throwIfAborted()
+      const key = quoteIdentifier(input.keyColumns[0]!)
+      const values = columns.flatMap((column, index) => [
+        `b.${quoteIdentifier(column.name)} AS b${index}`,
+        `a.${quoteIdentifier(column.name)} AS a${index}`,
+      ])
+      const join = input.keyColumns
+        .map((name) => `b.${quoteIdentifier(name)} = a.${quoteIdentifier(name)}`)
+        .join(" AND ")
+      const changed = columns
+        .map(
+          (column) =>
+            `b.${quoteIdentifier(column.name)} IS DISTINCT FROM a.${quoteIdentifier(column.name)}`
+        )
+        .join(" OR ")
+      // The native change feed reflects DELETE/INSERT writes: snapshot ingestion can produce 2N
+      // physical changes for one logical edit. Compare the pinned states instead of replaying it.
+      await runtime.run(`CREATE TEMP TABLE ${temporary} AS
+        SELECT b.${key} IS NOT NULL AS before_present, a.${key} IS NOT NULL AS after_present,
+          ${values.join(", ")}
+        FROM (SELECT * FROM ${oldRelation}) b
+        FULL OUTER JOIN (SELECT * FROM ${newRelation}) a ON ${join}
+        WHERE b.${key} IS NULL OR a.${key} IS NULL OR ${changed}`)
+      created = true
+      const [count] = await runtime.query(`SELECT count(*) AS count FROM ${temporary}`)
+      const changeCount = safeCount(count!, "count")
+      input.signal?.throwIfAborted()
+      let closePromise: Promise<void> | undefined
+      const close = () => {
+        closePromise ??= runtime.run(`DROP TABLE IF EXISTS ${temporary}`)
+        return closePromise
+      }
+      const changes = (async function* (): AsyncIterable<DatasetRowChange> {
+        try {
+          let cursor = -1n
+          let delivered = 0
+          while (delivered < changeCount) {
+            input.signal?.throwIfAborted()
+            if (closePromise) throw new LakeStorageError("[SixbDuckLake] Change reader is closed.")
+            const page = await runtime.query(`SELECT rowid AS position, * FROM ${temporary}
+              WHERE rowid > ${cursor} ORDER BY rowid LIMIT ${PAGE_ROWS}`)
+            if (page.length === 0)
+              throw new LakeStorageError(
+                "[SixbDuckLake] Change read ended before its declared count."
+              )
+            for (const row of page) {
+              input.signal?.throwIfAborted()
+              delivered++
+              yield {
+                before: row.before_present === true ? rowImage(row, "b", columns) : null,
+                after: row.after_present === true ? rowImage(row, "a", columns) : null,
+              }
+            }
+            cursor = getBigIntLike(page[page.length - 1]!, "position")
+          }
+        } finally {
+          await close()
+        }
+      })()
+      retained = true
+      return { fromRowCount: previousCount, toRowCount: nextCount, changeCount, changes, close }
+    } finally {
+      // The delta is a DuckDB temp table, independent of the attached DuckLake catalog. Release
+      // the local attachment lock before returning it, otherwise consumer lake operations deadlock.
+      try {
+        if (!retained && created) await runtime.run(`DROP TABLE IF EXISTS ${temporary}`)
+      } finally {
+        await lease.release()
+      }
+    }
+  }
+}
+
+async function uniqueRowCount(
+  runtime: DuckDbQueryRuntime,
+  relation: string,
+  keys: readonly string[]
+): Promise<number | null> {
+  const names = keys.map(quoteIdentifier)
+  const invalid = names.map((name) => `${name} IS NULL OR trim(${name}) = ''`).join(" OR ")
+  const [row] = await runtime.query(`SELECT count(*) AS total,
+    count(DISTINCT (${names.join(", ")})) AS unique_keys,
+    count(*) FILTER (WHERE ${invalid}) AS invalid_keys FROM ${relation}`)
+  const count = safeCount(row!, "total")
+  return safeCount(row!, "unique_keys") === count && safeCount(row!, "invalid_keys") === 0
+    ? count
+    : null
+}
+
+function safeCount(row: Readonly<Record<string, unknown>>, name: string): number {
+  const count = Number(getBigIntLike(row, name))
+  if (!Number.isSafeInteger(count) || count < 0)
+    throw new LakeStorageError("[SixbDuckLake] Change count exceeds the supported integer range.")
+  return count
+}
+
+function rowImage(
+  row: Readonly<Record<string, unknown>>,
+  prefix: "a" | "b",
+  columns: readonly DatasetColumnDefinition[]
+): DatasetRow {
+  return Object.fromEntries(
+    columns.map((column, index) => [
+      column.name,
+      normalizeReadValue(row[`${prefix}${index}`], column),
+    ])
+  )
+}

package/src/internal/ducklake-connection-manager.ts

@@ -1,6 +1,7 @@
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
 import { localCatalogCoordinationKey } from "./catalog-key"
+import type { DuckDbReader } from "./duckdb-reader"
 import {
   createDuckDbRuntime,
   type DuckDbExclusiveRuntime,
@@ -9,6 +10,7 @@ import {
   installDuckLakeExtensions,
   setupDuckLake,
 } from "./duckdb-runtime"
+import { DuckLakeReadGate } from "./ducklake-read-gate"
 import {
   buildAttachSql,
   buildConfigurePostgresMetadataPoolSql,
@@ -37,14 +39,89 @@ export class DuckLakeConnectionManager {
   // Local catalogs need DETACH/ATTACH to refresh metadata visibility. Keep
   // those attachment changes outside whole read operations so a read cannot
   // observe the DuckLake alias disappearing between its metadata queries.
-  private localAttachmentLock: Promise<void> = Promise.resolve()
+  private attachmentLock: Promise<void> = Promise.resolve()
   private observedLocalCatalogGeneration: number
   private attached = false
   private closed = false
+  private closePromise: Promise<void> | undefined
   private runtimePoisoned = false
+  private pinnedReaders = 0
+  private refreshRequested = false
+  private readonly shutdown = new AbortController()
+  private readonly reads = new DuckLakeReadGate()
+  private readonly maxStreamingReads: number
 
   constructor(private readonly options: DuckLakeStorageOptions) {
     this.observedLocalCatalogGeneration = currentLocalCatalogGeneration(options)
+    this.maxStreamingReads = options.maxStreamingReads ?? 4
+    if (!Number.isSafeInteger(this.maxStreamingReads) || this.maxStreamingReads < 1) {
+      throw new LakeStorageError("[SixbDuckLake] maxStreamingReads must be a positive integer.")
+    }
+  }
+
+  async acquireRead(signal?: AbortSignal): Promise<{
+    readonly signal: AbortSignal
+    readonly release: () => void
+  }> {
+    this.assertOpen()
+    const combined = signal ? AbortSignal.any([signal, this.shutdown.signal]) : this.shutdown.signal
+    return { signal: combined, release: await this.reads.acquire(combined) }
+  }
+
+  private async nativeReaderLimit(runtime: DuckDbRuntime): Promise<number> {
+    if (this.options.catalog.type === "duckdb") return this.maxStreamingReads
+    // SQLite metadata transactions keep a read lock for the native stream's lifetime.
+    // Unknown/custom catalogs use bounded queries too, without assuming their locking behavior.
+    if (this.options.catalog.type !== "postgres") return 0
+    const [setting] = await runtime.query(
+      "SELECT current_setting('pg_pool_max_connections') AS value"
+    )
+    const budget = Number(setting?.value)
+    // Reserve metadata/write headroom, including transient leases from bounded scans. A
+    // four-connection pool must stay paged: one paused native cursor can still starve an append.
+    // The extension can exceed its cache limit; this is not a server-wide connection guarantee.
+    return Number.isFinite(budget) ? Math.min(this.maxStreamingReads, Math.max(0, budget - 4)) : 0
+  }
+
+  /** Caller holds an attachment lease until this query has started. */
+  async openReader(
+    runtime: DuckDbRuntime,
+    sql: string,
+    signal: AbortSignal,
+    releaseRead: () => void
+  ): Promise<DuckDbReader | null> {
+    const limit = await this.nativeReaderLimit(runtime)
+    signal.throwIfAborted()
+    if (this.pinnedReaders >= limit) return null
+    this.pinnedReaders++
+    let starting = true
+    let released = false
+    const onClose = async () => {
+      if (released) return
+      released = true
+      this.pinnedReaders--
+      if (signal.aborted || this.closed || this.runtimePoisoned) releaseRead()
+      // On startup failure the caller still holds its lease and will detach on release.
+      if (starting || this.closed || this.runtimePoisoned) return
+      await this.withAttachmentLock(async () => {
+        if (this.pinnedReaders === 0 && this.refreshRequested) await this.resetRuntimeUnlocked()
+        await this.detachDuckDbCatalogAfterLease()
+      })
+    }
+    try {
+      return await runtime.openReader(sql, { signal, onClose })
+    } catch (error) {
+      await onClose()
+      throw error
+    } finally {
+      starting = false
+    }
+  }
+
+  async withMaintenanceRuntime<T>(
+    run: (runtime: DuckDbExclusiveRuntime) => Promise<T>
+  ): Promise<T> {
+    return this.reads.withMaintenance(this.shutdown.signal, () => this.withExclusiveAttached(run))
   }
 
   assertOpen(): void {
@@ -87,10 +164,9 @@ export class DuckLakeConnectionManager {
   async acquireAttachedRuntime(): Promise<DuckLakeAttachedRuntimeLease> {
     this.assertOpen()
 
-    // The lease is intentionally wider than one DuckDB queue operation. It
-    // covers the full Sixb read/commit boundary for local catalogs while
-    // remaining a no-op for PostgreSQL catalogs.
-    const releaseLock = await this.acquireLocalAttachmentLock()
+    // Keep attachment changes and native-query preparation outside each other's critical section.
+    // The lease ends before rows are yielded; an active reader only retains the attachment.
+    const releaseLock = await this.acquireAttachmentLock()
     let released = false
     const release = async () => {
       if (released) {
@@ -100,8 +176,8 @@ export class DuckLakeConnectionManager {
       released = true
       try {
         if (this.runtimePoisoned) {
-          this.runtimePoisoned = false
           await this.discardRuntimeUnlocked()
+          this.runtimePoisoned = false
         } else {
           await this.detachDuckDbCatalogAfterLease()
         }
@@ -168,7 +244,7 @@ export class DuckLakeConnectionManager {
   }
 
   async resetRuntime(): Promise<void> {
-    const releaseLock = await this.acquireLocalAttachmentLock()
+    const releaseLock = await this.acquireAttachmentLock()
     try {
       await this.resetRuntimeUnlocked()
     } finally {
@@ -177,6 +253,11 @@ export class DuckLakeConnectionManager {
   }
 
   private async resetRuntimeUnlocked(): Promise<void> {
+    if (this.pinnedReaders > 0) {
+      this.refreshRequested = true
+      return
+    }
+    this.refreshRequested = false
     const runtimePromise = this.runtimePromise
     if (runtimePromise === undefined) {
       return
@@ -226,7 +307,7 @@ export class DuckLakeConnectionManager {
 
     if (this.refreshPromise === undefined) {
       let refreshPromise: Promise<void>
-      refreshPromise = this.withLocalAttachmentLock(() =>
+      refreshPromise = this.withAttachmentLock(() =>
         this.refreshForExternalChangesUnlocked()
       ).finally(() => {
         if (this.refreshPromise === refreshPromise) {
@@ -253,8 +334,10 @@ export class DuckLakeConnectionManager {
     }
 
     try {
-      await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`)
-      this.attached = false
+      if (this.pinnedReaders === 0) {
+        await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`)
+        this.attached = false
+      }
     } catch {
       // The commit already succeeded. Keep the original write result and let a
       // later refresh/close retry detaching this local catalog if needed.
@@ -278,12 +361,14 @@ export class DuckLakeConnectionManager {
     return this.observedLocalCatalogGeneration !== currentLocalCatalogGeneration(this.options)
   }
 
-  async close(): Promise<void> {
-    if (this.closed) {
-      return
-    }
+  close(): Promise<void> {
+    this.closePromise ??= this.finishClose()
+    return this.closePromise
+  }
 
+  private async finishClose(): Promise<void> {
     this.closed = true
+    this.shutdown.abort(new LakeStorageError("[SixbDuckLake] DuckLakeStorage is closed."))
 
     const runtimePromise = this.runtimePromise
     this.runtimePromise = undefined
@@ -364,7 +449,7 @@ export class DuckLakeConnectionManager {
   }
 
   private async detachRuntime(): Promise<void> {
-    await this.withLocalAttachmentLock(() => this.detachRuntimeUnlocked())
+    await this.withAttachmentLock(() => this.detachRuntimeUnlocked())
   }
 
   /**
@@ -377,7 +462,7 @@ export class DuckLakeConnectionManager {
   }
 
   private async detachDuckDbCatalogAfterLease(): Promise<void> {
-    if (this.options.catalog.type !== "duckdb" || !this.attached) {
+    if (this.options.catalog.type !== "duckdb" || !this.attached || this.pinnedReaders > 0) {
       return
     }
 
@@ -403,6 +488,7 @@ export class DuckLakeConnectionManager {
   }
 
   private async detachRuntimeUnlocked(): Promise<void> {
+    if (this.pinnedReaders > 0) return
     if (this.attachPromise !== undefined) {
       await this.attachPromise
     }
@@ -416,8 +502,8 @@ export class DuckLakeConnectionManager {
     this.attached = false
   }
 
-  private async withLocalAttachmentLock<T>(run: () => Promise<T>): Promise<T> {
-    const releaseLock = await this.acquireLocalAttachmentLock()
+  private async withAttachmentLock<T>(run: () => Promise<T>): Promise<T> {
+    const releaseLock = await this.acquireAttachmentLock()
     try {
       return await run()
     } finally {
@@ -425,17 +511,13 @@ export class DuckLakeConnectionManager {
     }
   }
 
-  private async acquireLocalAttachmentLock(): Promise<() => void> {
-    if (this.options.catalog.type === "postgres") {
-      return () => {}
-    }
-
-    const previous = this.localAttachmentLock.catch(() => {})
+  private async acquireAttachmentLock(): Promise<() => void> {
+    const previous = this.attachmentLock.catch(() => {})
     let release!: () => void
     const current = new Promise<void>((resolve) => {
       release = resolve
     })
-    this.localAttachmentLock = previous.then(() => current)
+    this.attachmentLock = previous.then(() => current)
 
     await previous
 
@@ -475,16 +557,18 @@ export class DuckLakeConnectionManager {
     runtime: DuckDbRuntime,
     options: { readonly detach: boolean }
   ): Promise<void> {
-    if (options.detach) {
-      try {
-        await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`)
-      } catch {
-        // The runtime may already be detached, partially initialized, or in an
-        // error state. Closing the native connection is still required.
+    try {
+      await runtime.stopReaders()
+      if (options.detach) {
+        try {
+          await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`)
+        } catch {
+          // Detaching a partially initialized or failed catalog can fail; still close the engine.
+        }
       }
+    } finally {
+      await runtime.close()
     }
-
-    await runtime.close()
   }
 }
 

package/src/internal/ducklake-maintenance.ts

@@ -37,7 +37,7 @@ export class DuckLakeMaintenance {
       )
     }
 
-    return this.connections.withExclusiveAttached(async (runtime) => {
+    return this.connections.withMaintenanceRuntime(async (runtime) => {
       const counts = await runMaintenanceProcedures(
         runtime,
         this.options,

package/src/internal/ducklake-read-gate.ts

@@ -0,0 +1,58 @@
+/** Keeps destructive maintenance outside reads, including paged overflow readers. */
+export class DuckLakeReadGate {
+  private active = 0
+  private maintenance = false
+  private readonly waiters = new Set<() => void>()
+
+  async acquire(signal: AbortSignal): Promise<() => void> {
+    while (true) {
+      signal.throwIfAborted()
+      if (!this.maintenance) {
+        this.active++
+        let released = false
+        return () => {
+          if (released) return
+          released = true
+          this.active--
+          this.wake()
+        }
+      }
+      await this.wait(signal)
+    }
+  }
+
+  async withMaintenance<T>(signal: AbortSignal, run: () => Promise<T>): Promise<T> {
+    // Waiting maintenance must let an active pipeline open the rest of its inputs. Only block
+    // new reads once maintenance can actually start; continuous reads may defer it indefinitely.
+    while (this.active > 0 || this.maintenance) await this.wait(signal)
+    signal.throwIfAborted()
+    this.maintenance = true
+    try {
+      return await run()
+    } finally {
+      this.maintenance = false
+      this.wake()
+    }
+  }
+
+  private wait(signal: AbortSignal): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const wake = () => {
+        signal.removeEventListener("abort", abort)
+        this.waiters.delete(wake)
+        resolve()
+      }
+      const abort = () => {
+        this.waiters.delete(wake)
+        reject(signal.reason)
+      }
+      signal.throwIfAborted()
+      this.waiters.add(wake)
+      signal.addEventListener("abort", abort, { once: true })
+    })
+  }
+
+  private wake(): void {
+    for (const wake of this.waiters) wake()
+  }
+}