@sixb/ducklake 0.0.1

package/src/internal/dataset-row-commit.ts

@@ -0,0 +1,128 @@
+import type { DatasetDefinition } from "@sixb/core"
+import { type DatasetWriteMode, LakeStorageError } from "@sixb/core/lake-storage"
+import type { DuckLakeStorageOptions } from "../types"
+import { getBigIntLike } from "./duckdb-row"
+import type { DuckDbQueryRuntime } from "./duckdb-runtime"
+import { encodeDatasetTableName } from "./names"
+import { datasetSchemaColumnNamesSql } from "./schema"
+import { qualifiedTableName } from "./sql"
+
+export type CommitRowCount =
+  | { readonly kind: "exact"; readonly value: number }
+  | { readonly kind: "unknown" }
+
+export interface ApplyDatasetRowsResult {
+  readonly dataChangeExpected: boolean
+  readonly sourceRowCount: number
+  readonly resultingRowCount: CommitRowCount
+}
+
+export function assertDatasetWriteMode(
+  mode: unknown,
+  operation: string
+): asserts mode is DatasetWriteMode {
+  if (mode !== "snapshot" && mode !== "append") {
+    throw new LakeStorageError(
+      `[SixbDuckLake] Invalid ${operation} mode '${String(mode)}'. Expected 'snapshot' or 'append'.`
+    )
+  }
+}
+
+export async function applyDatasetRowsFromRelation(input: {
+  readonly options: DuckLakeStorageOptions
+  readonly runtime: DuckDbQueryRuntime
+  readonly dataset: DatasetDefinition
+  readonly mode: DatasetWriteMode
+  /** Provider-rendered relation SQL, such as a quoted staging or temp table. */
+  readonly sourceRelationSql: string
+  readonly previousRowCount?: number
+}): Promise<ApplyDatasetRowsResult> {
+  const tableName = encodeDatasetTableName(input.dataset.id)
+  const table = qualifiedTableName(input.options, tableName)
+  const columnsSql = datasetSchemaColumnNamesSql(input.dataset.schema)
+
+  if (input.mode === "snapshot") {
+    const source = await fingerprintRelation(input.runtime, input.sourceRelationSql, columnsSql)
+    const target = await fingerprintRelation(input.runtime, table, columnsSql)
+
+    if (sameFingerprint(source, target)) {
+      return {
+        dataChangeExpected: false,
+        sourceRowCount: source.rowCount,
+        resultingRowCount: { kind: "exact", value: target.rowCount },
+      }
+    }
+
+    await input.runtime.run(`DELETE FROM ${table}`)
+    await input.runtime.run(
+      `INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`
+    )
+
+    return {
+      dataChangeExpected: source.rowCount > 0 || target.rowCount > 0,
+      sourceRowCount: source.rowCount,
+      resultingRowCount: { kind: "exact", value: source.rowCount },
+    }
+  }
+
+  const sourceRowCount = await countRows(input.runtime, input.sourceRelationSql)
+  await input.runtime.run(
+    `INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`
+  )
+
+  return {
+    dataChangeExpected: sourceRowCount > 0,
+    sourceRowCount,
+    resultingRowCount:
+      input.previousRowCount === undefined
+        ? { kind: "unknown" }
+        : { kind: "exact", value: input.previousRowCount + sourceRowCount },
+  }
+}
+
+interface RelationFingerprint {
+  readonly rowCount: number
+  readonly hashSum: string
+  readonly hashXor: string
+}
+
+async function fingerprintRelation(
+  runtime: DuckDbQueryRuntime,
+  relationSql: string,
+  columnsSql: string
+): Promise<RelationFingerprint> {
+  const [row] = await runtime.query(`
+    SELECT
+      count(*) AS row_count,
+      coalesce(sum(hash(${columnsSql})::HUGEINT), 0::HUGEINT) AS hash_sum,
+      coalesce(bit_xor(hash('sixb:snapshot:v1', ${columnsSql})), 0::UBIGINT) AS hash_xor
+    FROM ${relationSql}
+  `)
+
+  if (row === undefined) {
+    return { rowCount: 0, hashSum: "0", hashXor: "0" }
+  }
+
+  return {
+    rowCount: Number(getBigIntLike(row, "row_count")),
+    hashSum: getBigIntLike(row, "hash_sum").toString(),
+    hashXor: getBigIntLike(row, "hash_xor").toString(),
+  }
+}
+
+function sameFingerprint(left: RelationFingerprint, right: RelationFingerprint): boolean {
+  return (
+    left.rowCount === right.rowCount &&
+    left.hashSum === right.hashSum &&
+    left.hashXor === right.hashXor
+  )
+}
+
+async function countRows(runtime: DuckDbQueryRuntime, relationSql: string): Promise<number> {
+  const [row] = await runtime.query(`SELECT count(*) AS row_count FROM ${relationSql}`)
+  if (row === undefined) {
+    return 0
+  }
+
+  return Number(getBigIntLike(row, "row_count"))
+}

package/src/internal/duckdb-column-types.ts

@@ -0,0 +1,68 @@
+import type { DatasetColumnType } from "@sixb/core"
+
+interface DuckDbColumnTypeMapping {
+  readonly sql: string
+  readonly matches: (typeSql: string) => boolean
+}
+
+interface DuckDbDecimalTypeMapping extends DuckDbColumnTypeMapping {
+  readonly precision: number
+  readonly scale: number
+}
+
+function normalizeDuckDbTypeSql(typeSql: string): string {
+  return typeSql
+    .trim()
+    .toUpperCase()
+    .replace(/\s+/g, " ")
+    .replace(/\s*([(),])\s*/g, "$1")
+}
+
+function defineDuckDbColumnType<const Sql extends string>(
+  sql: Sql,
+  aliases: readonly string[] = []
+): DuckDbColumnTypeMapping & { readonly sql: Sql } {
+  const acceptedTypes = new Set([sql, ...aliases].map(normalizeDuckDbTypeSql))
+  return {
+    sql,
+    matches: (typeSql) => acceptedTypes.has(normalizeDuckDbTypeSql(typeSql)),
+  }
+}
+
+function defineDuckDbDecimalType(precision: number, scale: number): DuckDbDecimalTypeMapping {
+  return {
+    ...defineDuckDbColumnType(`DECIMAL(${precision}, ${scale})`),
+    precision,
+    scale,
+  }
+}
+
+/**
+ * DuckDB representations used for every Sixb dataset column type.
+ *
+ * Each entry owns both its CREATE TABLE representation and the metadata
+ * spellings accepted when reconstructing a Sixb dataset schema.
+ */
+export const DUCKDB_COLUMN_TYPES = {
+  string: defineDuckDbColumnType("VARCHAR"),
+  boolean: defineDuckDbColumnType("BOOLEAN"),
+  int64: defineDuckDbColumnType("BIGINT", ["INT64"]),
+  float64: defineDuckDbColumnType("DOUBLE", ["FLOAT64"]),
+  decimal: defineDuckDbDecimalType(38, 9),
+  date: defineDuckDbColumnType("DATE"),
+  timestamp: defineDuckDbColumnType("TIMESTAMPTZ", ["TIMESTAMP WITH TIME ZONE"]),
+  json: defineDuckDbColumnType("JSON"),
+  fileRef: defineDuckDbColumnType(
+    "STRUCT(blobId VARCHAR, digest VARCHAR, sizeBytes BIGINT, fileName VARCHAR, mediaType VARCHAR, logicalPath VARCHAR)"
+  ),
+} satisfies Record<DatasetColumnType, DuckDbColumnTypeMapping>
+
+export function findDatasetColumnTypeForDuckDbSql(typeSql: string): DatasetColumnType | undefined {
+  for (const type of Object.keys(DUCKDB_COLUMN_TYPES) as DatasetColumnType[]) {
+    if (DUCKDB_COLUMN_TYPES[type].matches(typeSql)) {
+      return type
+    }
+  }
+
+  return undefined
+}

package/src/internal/duckdb-decimal.ts

@@ -0,0 +1,36 @@
+import { type DecimalValue, normalizeDecimalValue } from "@sixb/core"
+import { LakeStorageError } from "@sixb/core/lake-storage"
+import { DUCKDB_COLUMN_TYPES } from "./duckdb-column-types"
+
+const decimalType = DUCKDB_COLUMN_TYPES.decimal
+const maxIntegerDigits = decimalType.precision - decimalType.scale
+
+/**
+ * Normalize a dataset decimal and reject values DuckDB would round or overflow.
+ */
+export function normalizeDuckDbDecimalValue(value: unknown, columnName: string): DecimalValue {
+  if (typeof value !== "string") {
+    throw new LakeStorageError(
+      `[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`
+    )
+  }
+
+  let normalized: DecimalValue
+  try {
+    normalized = normalizeDecimalValue(value)
+  } catch {
+    throw new LakeStorageError(
+      `[SixbDuckLake] Decimal column '${columnName}' must be an exact decimal string.`
+    )
+  }
+
+  const unsigned = normalized.startsWith("-") ? normalized.slice(1) : normalized
+  const [integer, fraction = ""] = unsigned.split(".")
+  if (integer.length > maxIntegerDigits || fraction.length > decimalType.scale) {
+    throw new LakeStorageError(
+      `[SixbDuckLake] Decimal column '${columnName}' cannot be represented exactly as ${decimalType.sql}; expected at most ${maxIntegerDigits} integer digits and ${decimalType.scale} fractional digits.`
+    )
+  }
+
+  return normalized
+}

package/src/internal/duckdb-row.ts

@@ -0,0 +1,83 @@
+import { LakeStorageError } from "@sixb/core/lake-storage"
+
+export type DuckDbRow = Readonly<Record<string, unknown>>
+
+export function getString(row: DuckDbRow, key: string): string {
+  const value = row[key]
+  if (typeof value !== "string") {
+    throw new LakeStorageError(
+      `[SixbDuckLake] Expected DuckDB column '${key}' to be a string, got ${typeof value}.`
+    )
+  }
+
+  return value
+}
+
+export function getOptionalString(row: DuckDbRow, key: string): string | undefined {
+  const value = row[key]
+  if (value === null || value === undefined) {
+    return undefined
+  }
+
+  if (typeof value !== "string") {
+    throw new LakeStorageError(
+      `[SixbDuckLake] Expected DuckDB column '${key}' to be a string, got ${typeof value}.`
+    )
+  }
+
+  return value
+}
+
+export function getBoolean(row: DuckDbRow, key: string): boolean {
+  const value = row[key]
+  if (typeof value !== "boolean") {
+    throw new LakeStorageError(
+      `[SixbDuckLake] Expected DuckDB column '${key}' to be a boolean, got ${typeof value}.`
+    )
+  }
+
+  return value
+}
+
+export function getBigIntLike(row: DuckDbRow, key: string): bigint {
+  const value = row[key]
+  if (typeof value === "bigint") {
+    return value
+  }
+
+  if (typeof value === "number" && Number.isInteger(value)) {
+    return BigInt(value)
+  }
+
+  if (typeof value === "string" && /^-?\d+$/.test(value)) {
+    return BigInt(value)
+  }
+
+  throw new LakeStorageError(
+    `[SixbDuckLake] Expected DuckDB column '${key}' to be an integer, got ${typeof value}.`
+  )
+}
+
+export function getOptionalBigIntLike(row: DuckDbRow, key: string): bigint | undefined {
+  const value = row[key]
+  if (value === null || value === undefined) {
+    return undefined
+  }
+
+  return getBigIntLike(row, key)
+}
+
+export function getDate(row: DuckDbRow, key: string): Date {
+  const value = row[key]
+  if (value instanceof Date) {
+    return value
+  }
+
+  if (typeof value === "string") {
+    return new Date(value)
+  }
+
+  throw new LakeStorageError(
+    `[SixbDuckLake] Expected DuckDB column '${key}' to be a date, got ${typeof value}.`
+  )
+}

package/src/internal/duckdb-runtime.ts

@@ -0,0 +1,347 @@
+import {
+  type DuckDBConnection,
+  DuckDBInstance,
+  type DuckDBValue,
+  type DuckDBAppender as NodeDuckDBAppender,
+} from "@duckdb/node-api"
+import { LakeStorageError } from "@sixb/core/lake-storage"
+import type { DuckDbRuntimeOptions, DuckLakeStorageOptions } from "../types"
+import { sixbDuckDbValueConverter } from "./duckdb-value-converter"
+import {
+  buildAttachSql,
+  buildConfigurePostgresMetadataPoolSql,
+  buildCreateSecretSql,
+  buildSetPostgresPoolSql,
+  quoteIdentifier,
+  requiredExtensions,
+} from "./sql"
+
+/**
+ * Narrow adapter around `@duckdb/node-api`.
+ *
+ * Keeping the public provider behind this interface confines the driver types to
+ * this module instead of leaking them into `DuckLakeStorage`. Calls on a runtime
+ * are serialized through one queue because a runtime owns one DuckDB connection.
+ */
+export interface DuckDbQueryRuntime {
+  run(sql: string, values?: readonly DuckDBValue[]): Promise<void>
+  runStatements(statements: readonly string[]): Promise<void>
+  query(sql: string, values?: readonly DuckDBValue[]): Promise<readonly Record<string, unknown>[]>
+  withAppender<T>(
+    tableName: string,
+    useAppender: (appender: DuckDbAppender) => T | Promise<T>
+  ): Promise<T>
+}
+
+export interface DuckDbRuntime extends DuckDbQueryRuntime {
+  streamRows(sql: string, values?: readonly DuckDBValue[]): AsyncIterable<Record<string, unknown>>
+  /**
+   * Reserve the runtime queue for a multi-statement critical section.
+   *
+   * Use this for sequences such as BEGIN/read metadata/write/COMMIT where an
+   * interleaved read or write would observe the wrong connection state.
+   */
+  withExclusive<T>(useRuntime: (runtime: DuckDbExclusiveRuntime) => Promise<T>): Promise<T>
+  close(): Promise<void>
+}
+
+/**
+ * Runtime surface available while the caller already owns the queue slot.
+ *
+ * This is the query/appender subset only: no streaming, no close, and no nested
+ * exclusive block.
+ */
+export type DuckDbExclusiveRuntime = DuckDbQueryRuntime
+
+export interface DuckDbAppender {
+  appendNull(): void
+  appendBoolean(value: boolean): void
+  appendVarchar(value: string): void
+  appendBigInt(value: bigint): void
+  appendDouble(value: number): void
+  appendStruct(value: Readonly<Record<string, DuckDBValue>>): void
+  endRow(): void
+}
+
+interface SetupDuckLakeOptions {
+  readonly installExtensions?: boolean
+  readonly attach?: boolean
+}
+
+class NodeDuckDbRuntime implements DuckDbRuntime {
+  private closing = false
+  private closed = false
+  // One DuckDBConnection is shared by the runtime. Serialize calls so request
+  // handlers cannot interleave statements or close/reset an active query.
+  private operations: Promise<void> = Promise.resolve()
+
+  constructor(
+    private readonly instance: DuckDBInstance,
+    private readonly connection: DuckDBConnection
+  ) {}
+
+  async run(sql: string, values?: readonly DuckDBValue[]): Promise<void> {
+    await this.enqueue(async () => {
+      await this.connection.run(sql, values === undefined ? undefined : [...values])
+    })
+  }
+
+  async runStatements(statements: readonly string[]): Promise<void> {
+    await this.enqueue(async () => {
+      for (const sql of statements) {
+        await this.connection.run(sql)
+      }
+    })
+  }
+
+  async query(
+    sql: string,
+    values?: readonly DuckDBValue[]
+  ): Promise<readonly Record<string, unknown>[]> {
+    return this.enqueue(async () => {
+      const reader = await this.connection.runAndReadAll(
+        sql,
+        values === undefined ? undefined : [...values]
+      )
+      return reader.convertRowObjects(sixbDuckDbValueConverter)
+    })
+  }
+
+  async *streamRows(
+    sql: string,
+    values?: readonly DuckDBValue[]
+  ): AsyncIterable<Record<string, unknown>> {
+    this.assertAcceptingOperations()
+
+    // A stream holds its queue slot until the consumer finishes or abandons the
+    // iterator. This keeps later writes from running while rows are still being
+    // read from the same DuckDB connection.
+    let releaseOperation: (() => void) | undefined
+    const operationFinished = new Promise<void>((resolve) => {
+      releaseOperation = resolve
+    })
+    const waitForTurn = this.operations.then(() => {
+      this.assertNotClosed()
+    })
+    this.operations = waitForTurn
+      .then(() => operationFinished)
+      .then(
+        () => {},
+        () => {}
+      )
+
+    try {
+      await waitForTurn
+      const result = await this.connection.stream(
+        sql,
+        values === undefined ? undefined : [...values]
+      )
+      for await (const batch of result.yieldConvertedRowObjects(sixbDuckDbValueConverter)) {
+        for (const row of batch) {
+          yield row as Record<string, unknown>
+        }
+      }
+    } finally {
+      releaseOperation?.()
+    }
+  }
+
+  async withAppender<T>(
+    tableName: string,
+    useAppender: (appender: DuckDbAppender) => T | Promise<T>
+  ): Promise<T> {
+    return this.enqueue(async () => {
+      const appender = await this.connection.createAppender(tableName)
+      try {
+        return await useAppender(new NodeDuckDbAppenderAdapter(appender))
+      } finally {
+        appender.closeSync()
+      }
+    })
+  }
+
+  async withExclusive<T>(useRuntime: (runtime: DuckDbExclusiveRuntime) => Promise<T>): Promise<T> {
+    return this.enqueue(async () => {
+      // The exclusive adapter talks directly to the underlying connection while
+      // this queued operation is active. Do not pass `this` into the callback.
+      return useRuntime(new NodeDuckDbExclusiveRuntime(this.connection))
+    })
+  }
+
+  async close(): Promise<void> {
+    if (this.closed) {
+      return
+    }
+
+    this.closing = true
+    // resetRuntime()/close() can run while an API read is still using this
+    // runtime. Wait for accepted work before closing the native connection.
+    await this.operations.catch(() => {})
+    this.closed = true
+    this.connection.closeSync()
+    this.instance.closeSync()
+  }
+
+  private enqueue<T>(operation: () => T | Promise<T>): Promise<T> {
+    this.assertAcceptingOperations()
+
+    // Keep the chain alive after failures so one rejected query does not block
+    // all later work on this runtime.
+    const result = this.operations.then(async () => {
+      this.assertNotClosed()
+      return operation()
+    })
+    this.operations = result.then(
+      () => {},
+      () => {}
+    )
+    return result
+  }
+
+  private assertAcceptingOperations(): void {
+    if (this.closing || this.closed) {
+      throw new LakeStorageError("[SixbDuckLake] DuckDB runtime is closed.")
+    }
+  }
+
+  private assertNotClosed(): void {
+    if (this.closed) {
+      throw new LakeStorageError("[SixbDuckLake] DuckDB runtime is closed.")
+    }
+  }
+}
+
+class NodeDuckDbExclusiveRuntime implements DuckDbExclusiveRuntime {
+  constructor(private readonly connection: DuckDBConnection) {}
+
+  async run(sql: string, values?: readonly DuckDBValue[]): Promise<void> {
+    await this.connection.run(sql, values === undefined ? undefined : [...values])
+  }
+
+  async runStatements(statements: readonly string[]): Promise<void> {
+    for (const sql of statements) {
+      await this.connection.run(sql)
+    }
+  }
+
+  async query(
+    sql: string,
+    values?: readonly DuckDBValue[]
+  ): Promise<readonly Record<string, unknown>[]> {
+    const reader = await this.connection.runAndReadAll(
+      sql,
+      values === undefined ? undefined : [...values]
+    )
+    return reader.convertRowObjects(sixbDuckDbValueConverter)
+  }
+
+  async withAppender<T>(
+    tableName: string,
+    useAppender: (appender: DuckDbAppender) => T | Promise<T>
+  ): Promise<T> {
+    const appender = await this.connection.createAppender(tableName)
+    try {
+      return await useAppender(new NodeDuckDbAppenderAdapter(appender))
+    } finally {
+      appender.closeSync()
+    }
+  }
+}
+
+class NodeDuckDbAppenderAdapter implements DuckDbAppender {
+  constructor(private readonly appender: NodeDuckDBAppender) {}
+
+  appendNull(): void {
+    this.appender.appendNull()
+  }
+
+  appendBoolean(value: boolean): void {
+    this.appender.appendBoolean(value)
+  }
+
+  appendVarchar(value: string): void {
+    this.appender.appendVarchar(value)
+  }
+
+  appendBigInt(value: bigint): void {
+    this.appender.appendBigInt(value)
+  }
+
+  appendDouble(value: number): void {
+    this.appender.appendDouble(value)
+  }
+
+  appendStruct(value: Readonly<Record<string, DuckDBValue>>): void {
+    this.appender.appendStruct(value)
+  }
+
+  endRow(): void {
+    this.appender.endRow()
+  }
+}
+
+/**
+ * Create the DuckDB runtime used by the provider.
+ */
+export async function createDuckDbRuntime(
+  options: DuckDbRuntimeOptions = {}
+): Promise<DuckDbRuntime> {
+  const instance = await DuckDBInstance.create(options.path ?? ":memory:", options.config ?? {})
+  return new NodeDuckDbRuntime(instance, await instance.connect())
+}
+
+/**
+ * Install DuckLake and any catalog/data-path extensions into DuckDB's local
+ * extension directory. Installation is idempotent and can be shared by many
+ * short-lived runtimes created by one provider instance.
+ */
+export async function installDuckLakeExtensions(
+  runtime: DuckDbRuntime,
+  options: DuckLakeStorageOptions
+): Promise<void> {
+  await runtime.run("INSTALL ducklake")
+
+  for (const extension of requiredExtensions(options)) {
+    await runtime.run(`INSTALL ${quoteIdentifier(extension)}`)
+  }
+}
+
+/**
+ * Load DuckLake, load catalog extensions, run caller setup SQL, and optionally
+ * attach the configured DuckLake catalog.
+ */
+export async function setupDuckLake(
+  runtime: DuckDbRuntime,
+  options: DuckLakeStorageOptions,
+  setupOptions: SetupDuckLakeOptions = {}
+): Promise<void> {
+  if (setupOptions.installExtensions ?? true) {
+    await installDuckLakeExtensions(runtime, options)
+  }
+
+  await runtime.run("LOAD ducklake")
+
+  for (const extension of requiredExtensions(options)) {
+    await runtime.run(`LOAD ${quoteIdentifier(extension)}`)
+  }
+
+  for (const sql of options.setupSql ?? []) {
+    await runtime.run(sql)
+  }
+
+  if (options.catalog.type === "postgres") {
+    await runtime.runStatements(buildSetPostgresPoolSql(options))
+  }
+
+  for (const secret of options.secrets ?? []) {
+    await runtime.run(buildCreateSecretSql(secret))
+  }
+
+  if (setupOptions.attach ?? true) {
+    await runtime.run(buildAttachSql(options))
+
+    if (options.catalog.type === "postgres") {
+      await runtime.run(buildConfigurePostgresMetadataPoolSql(options))
+    }
+  }
+}

package/src/internal/duckdb-value-converter.ts

@@ -0,0 +1,27 @@
+import {
+  DuckDBDecimalValue,
+  DuckDBTypeId,
+  type DuckDBValueConverter,
+  type JS,
+  JSDuckDBValueConverter,
+} from "@duckdb/node-api"
+import { LakeStorageError } from "@sixb/core/lake-storage"
+
+/**
+ * Preserve DuckDB decimals as exact strings while retaining the driver's normal
+ * JavaScript conversions for every other scalar and nested value.
+ */
+export const sixbDuckDbValueConverter: DuckDBValueConverter<JS> = (value, type, converter) => {
+  if (value === null) {
+    return null
+  }
+
+  if (type.typeId === DuckDBTypeId.DECIMAL) {
+    if (!(value instanceof DuckDBDecimalValue)) {
+      throw new LakeStorageError("[SixbDuckLake] DuckDB returned an invalid DECIMAL value.")
+    }
+    return value.toString()
+  }
+
+  return JSDuckDBValueConverter(value, type, converter)
+}