@sixb/ducklake 0.1.6 → 0.1.7

package/src/internal/ducklake-connection-manager.ts

@@ -1,6 +1,7 @@
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
 import { localCatalogCoordinationKey } from "./catalog-key"
+import type { DuckDbReader } from "./duckdb-reader"
 import {
   createDuckDbRuntime,
   type DuckDbExclusiveRuntime,
@@ -9,6 +10,7 @@ import {
   installDuckLakeExtensions,
   setupDuckLake,
 } from "./duckdb-runtime"
+import { DuckLakeReadGate } from "./ducklake-read-gate"
 import {
   buildAttachSql,
   buildConfigurePostgresMetadataPoolSql,
@@ -37,14 +39,89 @@ export class DuckLakeConnectionManager {
   // Local catalogs need DETACH/ATTACH to refresh metadata visibility. Keep
   // those attachment changes outside whole read operations so a read cannot
   // observe the DuckLake alias disappearing between its metadata queries.
-  private localAttachmentLock: Promise<void> = Promise.resolve()
+  private attachmentLock: Promise<void> = Promise.resolve()
   private observedLocalCatalogGeneration: number
   private attached = false
   private closed = false
+  private closePromise: Promise<void> | undefined
   private runtimePoisoned = false
+  private pinnedReaders = 0
+  private refreshRequested = false
+  private readonly shutdown = new AbortController()
+  private readonly reads = new DuckLakeReadGate()
+  private readonly maxStreamingReads: number
 
   constructor(private readonly options: DuckLakeStorageOptions) {
     this.observedLocalCatalogGeneration = currentLocalCatalogGeneration(options)
+    this.maxStreamingReads = options.maxStreamingReads ?? 4
+    if (!Number.isSafeInteger(this.maxStreamingReads) || this.maxStreamingReads < 1) {
+      throw new LakeStorageError("[SixbDuckLake] maxStreamingReads must be a positive integer.")
+    }
+  }
+
+  async acquireRead(signal?: AbortSignal): Promise<{
+    readonly signal: AbortSignal
+    readonly release: () => void
+  }> {
+    this.assertOpen()
+    const combined = signal ? AbortSignal.any([signal, this.shutdown.signal]) : this.shutdown.signal
+    return { signal: combined, release: await this.reads.acquire(combined) }
+  }
+
+  private async nativeReaderLimit(runtime: DuckDbRuntime): Promise<number> {
+    if (this.options.catalog.type === "duckdb") return this.maxStreamingReads
+    // SQLite metadata transactions keep a read lock for the native stream's lifetime.
+    // Unknown/custom catalogs use bounded queries too, without assuming their locking behavior.
+    if (this.options.catalog.type !== "postgres") return 0
+    const [setting] = await runtime.query(
+      "SELECT current_setting('pg_pool_max_connections') AS value"
+    )
+    const budget = Number(setting?.value)
+    // Reserve metadata/write headroom, including transient leases from bounded scans. A
+    // four-connection pool must stay paged: one paused native cursor can still starve an append.
+    // The extension can exceed its cache limit; this is not a server-wide connection guarantee.
+    return Number.isFinite(budget) ? Math.min(this.maxStreamingReads, Math.max(0, budget - 4)) : 0
+  }
+
+  /** Caller holds an attachment lease until this query has started. */
+  async openReader(
+    runtime: DuckDbRuntime,
+    sql: string,
+    signal: AbortSignal,
+    releaseRead: () => void
+  ): Promise<DuckDbReader | null> {
+    const limit = await this.nativeReaderLimit(runtime)
+    signal.throwIfAborted()
+    if (this.pinnedReaders >= limit) return null
+    this.pinnedReaders++
+    let starting = true
+    let released = false
+    const onClose = async () => {
+      if (released) return
+      released = true
+      this.pinnedReaders--
+      if (signal.aborted || this.closed || this.runtimePoisoned) releaseRead()
+      // On startup failure the caller still holds its lease and will detach on release.
+      if (starting || this.closed || this.runtimePoisoned) return
+      await this.withAttachmentLock(async () => {
+        if (this.pinnedReaders === 0 && this.refreshRequested) await this.resetRuntimeUnlocked()
+        await this.detachDuckDbCatalogAfterLease()
+      })
+    }
+    try {
+      return await runtime.openReader(sql, { signal, onClose })
+    } catch (error) {
+      await onClose()
+      throw error
+    } finally {
+      starting = false
+    }
+  }
+
+  async withMaintenanceRuntime<T>(
+    run: (runtime: DuckDbExclusiveRuntime) => Promise<T>
+  ): Promise<T> {
+    return this.reads.withMaintenance(this.shutdown.signal, () => this.withExclusiveAttached(run))
   }
 
   assertOpen(): void {
@@ -87,10 +164,9 @@ export class DuckLakeConnectionManager {
   async acquireAttachedRuntime(): Promise<DuckLakeAttachedRuntimeLease> {
     this.assertOpen()
 
-    // The lease is intentionally wider than one DuckDB queue operation. It
-    // covers the full Sixb read/commit boundary for local catalogs while
-    // remaining a no-op for PostgreSQL catalogs.
-    const releaseLock = await this.acquireLocalAttachmentLock()
+    // Keep attachment changes and native-query preparation outside each other's critical section.
+    // The lease ends before rows are yielded; an active reader only retains the attachment.
+    const releaseLock = await this.acquireAttachmentLock()
     let released = false
     const release = async () => {
       if (released) {
@@ -100,8 +176,8 @@ export class DuckLakeConnectionManager {
       released = true
       try {
         if (this.runtimePoisoned) {
-          this.runtimePoisoned = false
           await this.discardRuntimeUnlocked()
+          this.runtimePoisoned = false
         } else {
           await this.detachDuckDbCatalogAfterLease()
         }
@@ -168,7 +244,7 @@ export class DuckLakeConnectionManager {
   }
 
   async resetRuntime(): Promise<void> {
-    const releaseLock = await this.acquireLocalAttachmentLock()
+    const releaseLock = await this.acquireAttachmentLock()
     try {
       await this.resetRuntimeUnlocked()
     } finally {
@@ -177,6 +253,11 @@ export class DuckLakeConnectionManager {
   }
 
   private async resetRuntimeUnlocked(): Promise<void> {
+    if (this.pinnedReaders > 0) {
+      this.refreshRequested = true
+      return
+    }
+    this.refreshRequested = false
     const runtimePromise = this.runtimePromise
     if (runtimePromise === undefined) {
       return
@@ -226,7 +307,7 @@ export class DuckLakeConnectionManager {
 
     if (this.refreshPromise === undefined) {
       let refreshPromise: Promise<void>
-      refreshPromise = this.withLocalAttachmentLock(() =>
+      refreshPromise = this.withAttachmentLock(() =>
         this.refreshForExternalChangesUnlocked()
       ).finally(() => {
         if (this.refreshPromise === refreshPromise) {
@@ -253,8 +334,10 @@ export class DuckLakeConnectionManager {
     }
 
     try {
-      await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`)
-      this.attached = false
+      if (this.pinnedReaders === 0) {
+        await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`)
+        this.attached = false
+      }
     } catch {
       // The commit already succeeded. Keep the original write result and let a
       // later refresh/close retry detaching this local catalog if needed.
@@ -278,12 +361,14 @@ export class DuckLakeConnectionManager {
     return this.observedLocalCatalogGeneration !== currentLocalCatalogGeneration(this.options)
   }
 
-  async close(): Promise<void> {
-    if (this.closed) {
-      return
-    }
+  close(): Promise<void> {
+    this.closePromise ??= this.finishClose()
+    return this.closePromise
+  }
 
+  private async finishClose(): Promise<void> {
     this.closed = true
+    this.shutdown.abort(new LakeStorageError("[SixbDuckLake] DuckLakeStorage is closed."))
 
     const runtimePromise = this.runtimePromise
     this.runtimePromise = undefined
@@ -364,7 +449,7 @@ export class DuckLakeConnectionManager {
   }
 
   private async detachRuntime(): Promise<void> {
-    await this.withLocalAttachmentLock(() => this.detachRuntimeUnlocked())
+    await this.withAttachmentLock(() => this.detachRuntimeUnlocked())
   }
 
   /**
@@ -377,7 +462,7 @@ export class DuckLakeConnectionManager {
   }
 
   private async detachDuckDbCatalogAfterLease(): Promise<void> {
-    if (this.options.catalog.type !== "duckdb" || !this.attached) {
+    if (this.options.catalog.type !== "duckdb" || !this.attached || this.pinnedReaders > 0) {
       return
     }
 
@@ -403,6 +488,7 @@ export class DuckLakeConnectionManager {
   }
 
   private async detachRuntimeUnlocked(): Promise<void> {
+    if (this.pinnedReaders > 0) return
     if (this.attachPromise !== undefined) {
       await this.attachPromise
     }
@@ -416,8 +502,8 @@ export class DuckLakeConnectionManager {
     this.attached = false
   }
 
-  private async withLocalAttachmentLock<T>(run: () => Promise<T>): Promise<T> {
-    const releaseLock = await this.acquireLocalAttachmentLock()
+  private async withAttachmentLock<T>(run: () => Promise<T>): Promise<T> {
+    const releaseLock = await this.acquireAttachmentLock()
     try {
       return await run()
     } finally {
@@ -425,17 +511,13 @@ export class DuckLakeConnectionManager {
     }
   }
 
-  private async acquireLocalAttachmentLock(): Promise<() => void> {
-    if (this.options.catalog.type === "postgres") {
-      return () => {}
-    }
-
-    const previous = this.localAttachmentLock.catch(() => {})
+  private async acquireAttachmentLock(): Promise<() => void> {
+    const previous = this.attachmentLock.catch(() => {})
     let release!: () => void
     const current = new Promise<void>((resolve) => {
       release = resolve
     })
-    this.localAttachmentLock = previous.then(() => current)
+    this.attachmentLock = previous.then(() => current)
 
     await previous
 
@@ -475,16 +557,18 @@ export class DuckLakeConnectionManager {
     runtime: DuckDbRuntime,
     options: { readonly detach: boolean }
   ): Promise<void> {
-    if (options.detach) {
-      try {
-        await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`)
-      } catch {
-        // The runtime may already be detached, partially initialized, or in an
-        // error state. Closing the native connection is still required.
+    try {
+      await runtime.stopReaders()
+      if (options.detach) {
+        try {
+          await runtime.run(`DETACH ${quoteIdentifier(duckLakeAlias(this.options))}`)
+        } catch {
+          // Detaching a partially initialized or failed catalog can fail; still close the engine.
+        }
       }
+    } finally {
+      await runtime.close()
     }
-
-    await runtime.close()
   }
 }
 

package/src/internal/ducklake-maintenance.ts

@@ -37,7 +37,7 @@ export class DuckLakeMaintenance {
       )
     }
 
-    return this.connections.withExclusiveAttached(async (runtime) => {
+    return this.connections.withMaintenanceRuntime(async (runtime) => {
       const counts = await runMaintenanceProcedures(
         runtime,
         this.options,

package/src/internal/ducklake-read-gate.ts

@@ -0,0 +1,58 @@
+/** Keeps destructive maintenance outside reads, including paged overflow readers. */
+export class DuckLakeReadGate {
+  private active = 0
+  private maintenance = false
+  private readonly waiters = new Set<() => void>()
+
+  async acquire(signal: AbortSignal): Promise<() => void> {
+    while (true) {
+      signal.throwIfAborted()
+      if (!this.maintenance) {
+        this.active++
+        let released = false
+        return () => {
+          if (released) return
+          released = true
+          this.active--
+          this.wake()
+        }
+      }
+      await this.wait(signal)
+    }
+  }
+
+  async withMaintenance<T>(signal: AbortSignal, run: () => Promise<T>): Promise<T> {
+    // Waiting maintenance must let an active pipeline open the rest of its inputs. Only block
+    // new reads once maintenance can actually start; continuous reads may defer it indefinitely.
+    while (this.active > 0 || this.maintenance) await this.wait(signal)
+    signal.throwIfAborted()
+    this.maintenance = true
+    try {
+      return await run()
+    } finally {
+      this.maintenance = false
+      this.wake()
+    }
+  }
+
+  private wait(signal: AbortSignal): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const wake = () => {
+        signal.removeEventListener("abort", abort)
+        this.waiters.delete(wake)
+        resolve()
+      }
+      const abort = () => {
+        this.waiters.delete(wake)
+        reject(signal.reason)
+      }
+      signal.throwIfAborted()
+      this.waiters.add(wake)
+      signal.addEventListener("abort", abort, { once: true })
+    })
+  }
+
+  private wake(): void {
+    for (const wake of this.waiters) wake()
+  }
+}

package/src/internal/ducklake-row-reader.ts

@@ -2,6 +2,7 @@ import type { DatasetColumnDefinition, DatasetRow } from "@sixb/core"
 import type { DatasetVersion, ReadDatasetRowsInput } from "@sixb/core/lake-storage"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckLakeStorageOptions } from "../types"
+import type { DuckDbReader } from "./duckdb-reader"
 import { getBigIntLike } from "./duckdb-row"
 import type { DuckDbQueryRuntime } from "./duckdb-runtime"
 import type { DuckLakeConnectionManager } from "./ducklake-connection-manager"
@@ -11,10 +12,7 @@ import { normalizeReadValue } from "./schema"
 import { qualifiedTableName, quoteIdentifier } from "./sql"
 import { parseVersionId } from "./versions"
 
-// Large reads release the provider's single DuckDB queue slot between bounded pages. Besides
-// bounding converted-row memory, this lets a JavaScript pipeline feed one DuckLake dataset into
-// another: the destination appender can use the runtime after each page instead of waiting forever
-// behind its own still-open source stream.
+// Bounded fallback for catalogs whose metadata transactions cannot stay open during a read.
 const READ_PAGE_ROWS = 5_000
 const PHYSICAL_ROW_ID_ALIAS = "__sixb_physical_row_id"
 
@@ -33,78 +31,88 @@ export class DuckLakeRowReader {
   ) {}
 
   async *readRows(input: ReadDatasetRowsInput): AsyncIterable<DatasetRow> {
-    this.connections.assertOpen()
-
-    // Keep the attachment lease across pages so local catalogs do not pay DETACH/ATTACH for every
-    // page. Individual query() calls still release the DuckDB queue before rows are yielded.
-    const lease = await this.connections.acquireAttachedRuntime()
+    const read = await this.connections.acquireRead(input.signal)
+    let reader: DuckDbReader | null = null
     try {
-      const runtime = lease.runtime
-      // Resolve latest exactly once so a paged read stays pinned even if another writer commits
-      // between pages. DuckLake snapshots are immutable; every page names this snapshot.
-      const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId)
-      if (!tableRef) {
-        throw new LakeStorageError(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`)
+      const prepared = await this.connections.withAttachedRuntime(async (runtime) => {
+        read.signal.throwIfAborted()
+        const tableRef = await resolveDatasetTableRef(this.options, runtime, input.datasetId)
+        if (!tableRef) {
+          throw new LakeStorageError(`[SixbDuckLake] Unknown dataset '${input.datasetId}'.`)
+        }
+        // Resolve latest once, before yielding any rows. Every query names the same snapshot.
+        const version = await this.resolveVersion(runtime, tableRef, input.versionId)
+        const columns = this.resolveReadColumns(tableRef.datasetId, version.schema, input.columns)
+        const names = new Set(version.schema.columns.map((column) => column.name.toLowerCase()))
+        const physicalOrder = !names.has("rowid") && !names.has(PHYSICAL_ROW_ID_ALIAS)
+        const relation = `${qualifiedTableName(this.options, tableRef.tableName)} AT (VERSION => ${parseVersionId(version.versionId)})`
+        const columnsSql = columns.map((column) => quoteIdentifier(column.name)).join(", ")
+        const offset = Math.max(0, Math.trunc(input.offset ?? 0))
+        const limit = input.limit === undefined ? undefined : Math.max(0, Math.trunc(input.limit))
+        const sql = `SELECT ${columnsSql} FROM ${relation}${physicalOrder ? " ORDER BY rowid" : ""}${limit === undefined ? "" : ` LIMIT ${limit}`}${offset > 0 ? ` OFFSET ${offset}` : ""}`
+        const cursor =
+          limit === undefined || limit > READ_PAGE_ROWS
+            ? await this.connections.openReader(runtime, sql, read.signal, read.release)
+            : null
+        const query = { columnsSql, relation, physicalOrder }
+        const firstPage =
+          cursor || limit === 0
+            ? []
+            : await runtime.query(
+                pageSql(query, Math.min(limit ?? READ_PAGE_ROWS, READ_PAGE_ROWS), offset, null)
+              )
+        return { columns, ...query, offset, limit, cursor, firstPage }
+      })
+      reader = prepared.cursor
+      // A paused paged reader owns no native query. Cancellation can release its read lease without
+      // waiting for the consumer to call next()/return(); native readers release after cleanup.
+      if (!reader) read.signal.addEventListener("abort", read.release, { once: true })
+      read.signal.throwIfAborted()
+      if (reader) {
+        yield* this.normalizeRows(reader.rows(), prepared.columns, read.signal)
+        return
       }
-      const version = await this.resolveVersion(runtime, tableRef, input.versionId)
-      const selectedColumns = this.resolveReadColumns(
-        tableRef.datasetId,
-        version.schema,
-        input.columns
-      )
-      const snapshotId = parseVersionId(version.versionId)
-      const columnsSql = selectedColumns.map((column) => quoteIdentifier(column.name)).join(", ")
-      const tableSql = qualifiedTableName(this.options, tableRef.tableName)
-      // A user column named `rowid` shadows DuckDB's virtual row id. In that edge case the
-      // connection-level invariant still preserves insertion order; otherwise make it explicit.
-      const schemaNames = new Set(version.schema.columns.map((column) => column.name.toLowerCase()))
-      const canUsePhysicalRowId =
-        !schemaNames.has("rowid") && !schemaNames.has(PHYSICAL_ROW_ID_ALIAS.toLowerCase())
-      const orderSql = canUsePhysicalRowId ? " ORDER BY rowid" : ""
-      const baseSql = `SELECT ${columnsSql} FROM ${tableSql} AT (VERSION => ${snapshotId})`
-      const requestedOffset = Math.max(0, Math.trunc(input.offset ?? 0))
-      let remaining = input.limit === undefined ? undefined : Math.max(0, Math.trunc(input.limit))
-      if (remaining === 0) return
 
-      // Materialize one bounded page and release its queue operation before yielding it. A
-      // consumer may enqueue another DuckLake operation while processing the yielded rows.
-      // Holding a native stream open here would make that operation wait behind itself.
-      let offset = requestedOffset
-      let physicalCursor: bigint | null = null
-      while (true) {
-        const pageRows =
-          remaining === undefined ? READ_PAGE_ROWS : Math.min(READ_PAGE_ROWS, remaining)
-        const pageSql = canUsePhysicalRowId
-          ? `SELECT rowid AS ${quoteIdentifier(PHYSICAL_ROW_ID_ALIAS)}, ${columnsSql}
-              FROM ${tableSql} AT (VERSION => ${snapshotId})
-              ${physicalCursor === null ? "" : `WHERE rowid > ${physicalCursor}`}
-              ORDER BY rowid LIMIT ${pageRows}
-              ${physicalCursor === null && offset > 0 ? `OFFSET ${offset}` : ""}`
-          : `${baseSql}${orderSql} LIMIT ${pageRows} OFFSET ${offset}`
-        const rows = await runtime.query(pageSql)
-        if (rows.length === 0) return
-
-        yield* this.normalizeRows(rows, selectedColumns)
+      // Each bounded query releases both the runtime queue and attachment lease before yielding.
+      // SQLite metadata locks and PostgreSQL pool slots are released before consumer operations.
+      let offset = prepared.offset
+      let remaining = prepared.limit
+      let cursor: bigint | null = null
+      let page: readonly Record<string, unknown>[] | undefined = prepared.firstPage
+      while (remaining !== 0) {
+        read.signal.throwIfAborted()
+        const pageSize = Math.min(remaining ?? READ_PAGE_ROWS, READ_PAGE_ROWS)
+        const rows: readonly Record<string, unknown>[] =
+          page ??
+          (await this.connections.withAttachedRuntime((runtime) =>
+            runtime.query(pageSql(prepared, pageSize, offset, cursor))
+          ))
+        page = undefined
+        read.signal.throwIfAborted()
+        yield* this.normalizeRows(rows, prepared.columns, read.signal)
+        if (remaining !== undefined) remaining -= rows.length
+        if (rows.length < pageSize) return
+        if (prepared.physicalOrder)
+          cursor = getBigIntLike(rows[rows.length - 1]!, PHYSICAL_ROW_ID_ALIAS)
         offset += rows.length
-        if (remaining !== undefined) {
-          remaining -= rows.length
-          if (remaining === 0) return
-        }
-        if (canUsePhysicalRowId) {
-          physicalCursor = getBigIntLike(rows[rows.length - 1]!, PHYSICAL_ROW_ID_ALIAS)
-        }
-        if (rows.length < pageRows) return
       }
     } finally {
-      await lease.release()
+      read.signal.removeEventListener("abort", read.release)
+      try {
+        await reader?.close()
+      } finally {
+        read.release()
+      }
     }
   }
 
   private async *normalizeRows(
     rows: Iterable<Record<string, unknown>> | AsyncIterable<Record<string, unknown>>,
-    selectedColumns: readonly DatasetColumnDefinition[]
+    selectedColumns: readonly DatasetColumnDefinition[],
+    signal: AbortSignal
   ): AsyncIterable<DatasetRow> {
     for await (const row of rows) {
+      signal.throwIfAborted()
       const output: Record<string, unknown> = {}
       for (const column of selectedColumns) {
         output[column.name] = normalizeReadValue(row[column.name], column)
@@ -165,3 +173,16 @@ export class DuckLakeRowReader {
     )
   }
 }
+
+function pageSql(
+  read: { readonly columnsSql: string; readonly relation: string; readonly physicalOrder: boolean },
+  limit: number,
+  offset: number,
+  cursor: bigint | null
+): string {
+  return read.physicalOrder
+    ? `SELECT rowid AS ${quoteIdentifier(PHYSICAL_ROW_ID_ALIAS)}, ${read.columnsSql}
+        FROM ${read.relation} ${cursor === null ? "" : `WHERE rowid > ${cursor}`}
+        ORDER BY rowid LIMIT ${limit} ${cursor === null && offset > 0 ? `OFFSET ${offset}` : ""}`
+    : `SELECT ${read.columnsSql} FROM ${read.relation} LIMIT ${limit} OFFSET ${offset}`
+}

package/src/internal/ducklake-snapshot-reader.ts

@@ -642,7 +642,12 @@ export class DuckLakeSnapshotReader {
       return []
     }
 
-    const where: string[] = []
+    // A dataset cannot have a version before its table exists. Use the earliest table record:
+    // a rename may create a newer record with the same table id and must not hide older versions.
+    const ducklakeTable = duckLakeMetadataTableName(this.options, "ducklake_table")
+    const where = [
+      `snapshot.snapshot_id >= (SELECT min(begin_snapshot) FROM ${ducklakeTable} WHERE table_id = ${input.tableId})`,
+    ]
     if (input.exactSnapshotId !== undefined) {
       assertDuckLakeSnapshotId(input.exactSnapshotId)
       where.push(`snapshot.snapshot_id = ${input.exactSnapshotId}`)
@@ -652,7 +657,7 @@ export class DuckLakeSnapshotReader {
       where.push(`snapshot.snapshot_id < ${input.beforeSnapshotId}`)
     }
 
-    const whereSql = where.length === 0 ? "" : `WHERE ${where.join(" AND ")}`
+    const whereSql = `WHERE ${where.join(" AND ")}`
     const limitSql =
       input.limit === undefined ? "" : `LIMIT ${Math.max(0, Math.trunc(input.limit))}`
     const ducklakeSnapshot = duckLakeMetadataTableName(this.options, "ducklake_snapshot")