@sixb/ducklake 0.2.0 → 0.2.2

package/README.md

@@ -65,6 +65,18 @@ const lakeStorage = new DuckLakeStorage({
 Use `close()` during service shutdown so DuckDB can close its runtime and release
 catalog connections cleanly.
 
+### Extensions
+
+On first use, the provider installs DuckLake and the extensions its catalog,
+data path and secrets need. A missing extension is downloaded over HTTPS from
+`https://extensions.duckdb.org` into DuckDB's extension directory
+(`duckdb.config.extension_directory`, `~/.duckdb/extensions` by default), so a
+host that allows only that HTTPS domain, such as a restricted Vercel sandbox,
+can open a lake. When that download fails, the provider logs a warning and
+DuckDB's own `INSTALL` downloads the extension over plain HTTP, as before.
+With `custom_extension_repository` set, DuckDB installs from that repository
+alone.
+
 ### Maintenance
 
 DuckLake keeps historical snapshots and deleted files until maintenance runs.
@@ -102,6 +114,10 @@ budget and PostgreSQL metadata pool. Metadata and writes remain available betwee
 Each read stays pinned to one immutable snapshot. Maintenance waits until no reads remain; it
 blocks new reads only while running, so continuous traffic can postpone it.
 
+`readChanges` compares two versions in DuckDB. When no version between them changed rows
+(`rowsChanged` in the commit metadata, read from at most 32 snapshots), it returns an empty change
+set without reading the table.
+
 `maxStreamingReads` limits native streaming readers per provider (default: `4`); extra reads use
 bounded pages on the main connection. It does not limit pipeline inputs. Raise it when memory and
 catalog capacity allow. Finish the iterator, use `for await` with `break`, or pass `signal` to cancel
@@ -190,8 +206,10 @@ Datasets declaring `sequenceBy` instead compare source revisions: older changes
 identical ties are unchanged, and conflicting ties abort the merge. Deletes require
 `change.delete(key, { sequence })`. Source revisions and deletion tombstones are stored in a
 companion DuckLake table in the same transaction as visible rows, surviving snapshot expiration.
-New tombstone state creates a version even when the visible row count stays zero. Sequenced
-provider writes use merge sessions; snapshot syncs convert rows to ordered upserts.
+New tombstone state creates a version even when the visible rows stay the same; such a version
+records `rowsChanged: false`. So does a newer upsert that differs from the stored row only in its
+`sequenceBy` column: the row is kept and only its source revision advances. Sequenced provider
+writes use merge sessions; snapshot syncs convert rows to ordered upserts.
 Snapshot syncs use `commit({ createInitialVersion: true })` to make a first empty result
 addressable. Later no-op commits reuse the current version, including after a concurrent write.
 Use `commit({ retryOnConflict: true })` to rebase retained changes, with at most 3 attempts.

package/dist/index.js

@@ -647,6 +647,7 @@ function parseCommitMetadata(value) {
   const producer = parseDatasetProducer(parsed.sixb.producer);
   const rowCount = parsed.sixb.rowCount;
   const validatedPrimaryKeyColumns = parsed.sixb.validatedPrimaryKeyColumns;
+  const { rowsChanged, lineageChanged } = parsed.sixb;
   return {
     kind: "datasetVersion",
     datasetId: parsed.sixb.datasetId,
@@ -657,7 +658,9 @@ function parseCommitMetadata(value) {
     ...typeof commitId === "string" && isRowCount(rowCount) ? { rowCount } : {},
     ...typeof commitId === "string" && isValidatedPrimaryKeyColumns(validatedPrimaryKeyColumns) ? { validatedPrimaryKeyColumns } : {},
     ...isSchemaChangeMetadata(parsed.sixb.schemaChange) ? { schemaChange: parsed.sixb.schemaChange } : {},
-    ...parsed.sixb.lineage === undefined ? {} : { lineage: parsed.sixb.lineage }
+    ...parsed.sixb.lineage === undefined ? {} : { lineage: parsed.sixb.lineage },
+    ...typeof rowsChanged === "boolean" ? { rowsChanged } : {},
+    ...typeof lineageChanged === "boolean" ? { lineageChanged } : {}
   };
 }
 function changesTouchTables(changesMade, tableIds) {
@@ -729,6 +732,7 @@ class DuckLakeChangeReader {
     const columns = resolveDatasetChangeColumns(input, before, after);
     if (!columns)
       return null;
+    const unchanged = (input.toVersionId === input.fromVersionId || !after.rowsChanged) && await this.snapshots.holdsRowsOf(input.datasetId, input.toVersionId, input.fromVersionId);
     const lease = await this.connections.acquireAttachedRuntime();
     const runtime = lease.runtime;
     const temporary = quoteIdentifier(`sixb_delta_${randomUUID().replaceAll("-", "")}`);
@@ -741,6 +745,9 @@ class DuckLakeChangeReader {
       const table = qualifiedTableName(this.options, ref.tableName);
       const oldRelation = `${table} AT (VERSION => ${parseVersionId(input.fromVersionId)})`;
       const newRelation = `${table} AT (VERSION => ${parseVersionId(input.toVersionId)})`;
+      if (unchanged) {
+        return noChanges(after.rowCount ?? before.rowCount ?? await countRows(runtime, newRelation));
+      }
       const previousCount = await uniqueRowCount(runtime, oldRelation, input.keyColumns);
       const nextCount = await uniqueRowCount(runtime, newRelation, input.keyColumns);
       if (previousCount === null || nextCount === null)
@@ -806,6 +813,21 @@ class DuckLakeChangeReader {
     }
   }
 }
+function noChanges(rowCount) {
+  return {
+    fromRowCount: rowCount,
+    toRowCount: rowCount,
+    changeCount: 0,
+    changes: async function* () {
+      yield* [];
+    }(),
+    async close() {}
+  };
+}
+async function countRows(runtime, relation) {
+  const [row] = await runtime.query(`SELECT count(*) AS total FROM ${relation}`);
+  return safeCount(row, "total");
+}
 async function uniqueRowCount(runtime, relation, keys) {
   const names = keys.map(quoteIdentifier);
   const invalid = names.map((name) => `${name} IS NULL OR trim(${name}) = ''`).join(" OR ");
@@ -849,13 +871,78 @@ function localCatalogCoordinationKey(options) {
 }
 
 // src/internal/duckdb-runtime.ts
-import { mkdir, mkdtemp, rm } from "fs/promises";
-import { join, resolve as resolve2 } from "path";
+import { mkdir, mkdtemp as mkdtemp2, rm as rm2 } from "fs/promises";
+import { join as join2, resolve as resolve2 } from "path";
 import {
   DuckDBInstance
 } from "@duckdb/node-api";
 import { LakeStorageError as LakeStorageError11 } from "@sixb/core/lake-storage";
 
+// src/internal/duckdb-extensions.ts
+import { mkdtemp, rm, writeFile } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+import { promisify } from "util";
+import { gunzip } from "zlib";
+var extensionsUrl = "https://extensions.duckdb.org";
+var downloadTimeoutMillis = 60000;
+var gunzipAsync = promisify(gunzip);
+async function prefetchDuckDbExtensions(runtime, extensions) {
+  await prefetch(runtime, extensions).catch((error) => warnFallback(extensions.join("', '"), error));
+}
+async function prefetch(runtime, extensions) {
+  const [target] = await runtime.query(`
+    SELECT
+      version() AS version,
+      (SELECT platform FROM pragma_platform()) AS platform,
+      current_setting('custom_extension_repository') AS repository
+  `);
+  if (target?.repository !== "")
+    return;
+  const missing = await missingExtensions(runtime, extensions);
+  if (missing.length === 0)
+    return;
+  const directory = await mkdtemp(join(tmpdir(), "sixb-duckdb-extensions-"));
+  try {
+    const downloads = await Promise.all(missing.map(async (name) => ({
+      name,
+      file: await download(`${target.version}/${target.platform}/${name}`, directory, name).catch((error) => warnFallback(name, error))
+    })));
+    for (const { name, file } of downloads) {
+      if (file === undefined)
+        continue;
+      await runtime.run(`INSTALL ${quoteSqlString(file)}`).catch((error) => warnFallback(name, error));
+    }
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}
+function warnFallback(name, error) {
+  const reason = error instanceof Error ? error.message : String(error);
+  console.warn(`[SixbDuckLake] Could not install DuckDB extension '${name}' over HTTPS (${reason}); falling back to DuckDB's INSTALL.`);
+  return;
+}
+async function missingExtensions(runtime, extensions) {
+  const requested = `[${quoteSqlStringList(extensions.map((extension) => extension.toLowerCase()))}]`;
+  const rows = await runtime.query(`
+    SELECT extension_name
+    FROM duckdb_extensions()
+    WHERE NOT installed
+      AND (list_contains(${requested}, extension_name) OR list_has_any(${requested}, aliases))
+  `);
+  return rows.map((row) => String(row.extension_name));
+}
+async function download(path, directory, name) {
+  const response = await fetch(`${extensionsUrl}/${path}.duckdb_extension.gz`, {
+    signal: AbortSignal.timeout(downloadTimeoutMillis)
+  });
+  if (!response.ok)
+    throw new Error(`HTTP ${response.status}`);
+  const file = join(directory, `${name}.duckdb_extension`);
+  await writeFile(file, await gunzipAsync(new Uint8Array(await response.arrayBuffer())));
+  return file;
+}
+
 // src/internal/duckdb-reader.ts
 import { LakeStorageError as LakeStorageError10 } from "@sixb/core/lake-storage";
 
@@ -1072,7 +1159,7 @@ class NodeDuckDbRuntime {
     this.connection.closeSync();
     this.instance.closeSync();
     if (this.temporaryDirectory)
-      await rm(this.temporaryDirectory, { recursive: true, force: true });
+      await rm2(this.temporaryDirectory, { recursive: true, force: true });
     const failed = results.find((result) => result.status === "rejected");
     if (failed?.status === "rejected")
       throw failed.reason;
@@ -1159,7 +1246,7 @@ async function createDuckDbRuntime(options = {}) {
   if (base !== "") {
     const directory = resolve2(base);
     await mkdir(directory, { recursive: true });
-    temporaryDirectory = await mkdtemp(join(directory, "sixb-"));
+    temporaryDirectory = await mkdtemp2(join2(directory, "sixb-"));
   }
   let instance;
   try {
@@ -1171,13 +1258,15 @@ async function createDuckDbRuntime(options = {}) {
   } catch (error) {
     instance?.closeSync();
     if (temporaryDirectory)
-      await rm(temporaryDirectory, { recursive: true, force: true });
+      await rm2(temporaryDirectory, { recursive: true, force: true });
     throw error;
   }
 }
 async function installDuckLakeExtensions(runtime, options) {
+  const extensions = requiredExtensions(options);
+  await prefetchDuckDbExtensions(runtime, ["ducklake", ...extensions]);
   await runtime.run("INSTALL ducklake");
-  for (const extension of requiredExtensions(options)) {
+  for (const extension of extensions) {
     await runtime.run(`INSTALL ${quoteIdentifier(extension)}`);
   }
 }
@@ -2619,6 +2708,7 @@ import { LakeStorageError as LakeStorageError19 } from "@sixb/core/lake-storage"
 var SNAPSHOT_ROW_BATCH_SIZE = 128;
 var FEED_PAGE_SIZE = 512;
 var SHORT_READ_BATCH_SIZE = 8;
+var UNCHANGED_ROWS_WALK_LIMIT = 32;
 var CATALOG_SNAPSHOT_SCAN_LIMIT = 512;
 
 class DuckLakeSnapshotReader {
@@ -2661,6 +2751,33 @@ class DuckLakeSnapshotReader {
       return this.getVersionForTableRef(runtime, tableRef, snapshotId);
     });
   }
+  async holdsRowsOf(datasetId, versionId, earlierVersionId) {
+    const snapshotId = parseVersionId(versionId);
+    const earlierSnapshotId = parseVersionId(earlierVersionId);
+    if (snapshotId === earlierSnapshotId) {
+      return true;
+    }
+    if (BigInt(snapshotId) < BigInt(earlierSnapshotId)) {
+      return false;
+    }
+    this.connections.assertOpen();
+    return this.connections.withAttachedRuntime(async (runtime) => {
+      const tableRef = await resolveDatasetTableRef(this.options, runtime, datasetId);
+      if (!tableRef) {
+        return false;
+      }
+      const candidates = await this.querySnapshotCandidates(runtime, {
+        table: tableRef,
+        beforeSnapshotId: String(BigInt(snapshotId) + 1n),
+        fromSnapshotId: earlierSnapshotId,
+        limit: UNCHANGED_ROWS_WALK_LIMIT + 1
+      });
+      if (candidates[0]?.snapshotId !== snapshotId || candidates[candidates.length - 1]?.snapshotId !== earlierSnapshotId) {
+        return false;
+      }
+      return candidates.slice(0, -1).every((candidate) => this.candidateToSnapshotRow(datasetId, tableRef.tableId, candidate)?.rowsChanged === false);
+    });
+  }
   async listDatasetCatalogState(datasetIds) {
     this.connections.assertOpen();
     if (datasetIds.length === 0) {
@@ -3101,6 +3218,9 @@ class DuckLakeSnapshotReader {
     if (input.beforeSnapshotId !== undefined) {
       assertDuckLakeSnapshotId2(input.beforeSnapshotId);
     }
+    if (input.fromSnapshotId !== undefined) {
+      assertDuckLakeSnapshotId2(input.fromSnapshotId);
+    }
     const rows = await runtime.query(buildDuckLakeMetadataQuery(this.options, (table) => {
       const bounds = [
         `changes.snapshot_id >= (
@@ -3115,6 +3235,9 @@ class DuckLakeSnapshotReader {
       if (input.beforeSnapshotId !== undefined) {
         bounds.push(`changes.snapshot_id < ${input.beforeSnapshotId}`);
       }
+      if (input.fromSnapshotId !== undefined) {
+        bounds.push(`changes.snapshot_id >= ${input.fromSnapshotId}`);
+      }
       const namesDatasetTable = datasetTableIds(input.table).map((tableId) => `strpos(',' || changes.changes_made || ',', ${quoteSqlString(`:${tableId},`)}) > 0`).join(" OR ");
       return `
           SELECT
@@ -3175,6 +3298,8 @@ class DuckLakeSnapshotReader {
         tableId,
         createdAt: candidate.createdAt,
         mode: metadata.mode ?? "schema",
+        rowsChanged: metadata.rowsChanged ?? true,
+        lineageChanged: metadata.lineageChanged ?? true,
         metadata
       };
     }
@@ -3184,6 +3309,8 @@ class DuckLakeSnapshotReader {
       tableId,
       createdAt: candidate.createdAt,
       mode: candidate.metadata?.mode ?? (hasDeleteChange ? "snapshot" : "append"),
+      rowsChanged: true,
+      lineageChanged: candidate.metadata?.lineageChanged ?? true,
       ...candidate.metadata !== undefined ? { metadata: candidate.metadata } : {}
     };
   }
@@ -3209,6 +3336,8 @@ function snapshotToVersion(tableRef, row, schema) {
     producer: row.metadata?.producer,
     inputs: row.metadata?.inputs,
     ...storedLineage(tableRef.datasetId, row),
+    rowsChanged: row.rowsChanged,
+    lineageChanged: row.lineageChanged,
     ...row.metadata?.rowCount !== undefined ? { rowCount: row.metadata.rowCount } : {}
   };
 }
@@ -3306,7 +3435,7 @@ async function applyDatasetRowsFromRelation(input) {
       changedTableRows
     };
   }
-  const sourceRowCount = await countRows(input.runtime, input.sourceRelationSql);
+  const sourceRowCount = await countRows2(input.runtime, input.sourceRelationSql);
   const knownRowCount = baselineRowCount ?? input.previousRowCount;
   const targetWasEmpty = knownRowCount === undefined ? await isEmptyRelation(input.runtime, table) : knownRowCount === 0;
   await input.runtime.run(`INSERT INTO ${table} (${columnsSql}) SELECT ${columnsSql} FROM ${input.sourceRelationSql}`);
@@ -3376,7 +3505,7 @@ async function fingerprintRelation(runtime, relationSql, columnsSql) {
 function sameFingerprint(left, right) {
   return left.rowCount === right.rowCount && left.hashSum === right.hashSum && left.hashXor === right.hashXor;
 }
-async function countRows(runtime, relationSql) {
+async function countRows2(runtime, relationSql) {
   const [row] = await runtime.query(`SELECT count(*) AS row_count FROM ${relationSql}`);
   if (row === undefined) {
     return 0;
@@ -4707,6 +4836,7 @@ import { randomUUID as randomUUID8 } from "crypto";
 import {
   assertUnsequencedDatasetWrite,
   getDatasetPrimaryKeyColumns as getDatasetPrimaryKeyColumns4,
+  hasDatasetLineageChanges,
   hasDatasetProvenanceChanges,
   LakeConcurrencyError as LakeConcurrencyError2,
   LakeStorageError as LakeStorageError26,
@@ -4727,7 +4857,7 @@ async function applyDatasetMergeFromRelation(input) {
   }
   const stagingTable = quoteIdentifier(input.stagingTableName);
   const targetTable = qualifiedTableName(input.options, encodeDatasetTableName(input.dataset.id));
-  const sourceRowCount = await countRows2(input.runtime, stagingTable);
+  const sourceRowCount = await countRows3(input.runtime, stagingTable);
   const previousRowCount = await currentBaselineRowCount(input, targetTable, primaryKeyColumns);
   return withMaterializedFinalChanges(input, primaryKeyColumns, async (finalChanges, orderingChanged) => {
     const effects = await countMergeEffects(input, targetTable, finalChanges, primaryKeyColumns);
@@ -4827,7 +4957,7 @@ async function applySourceOrdering(input, acceptedTable) {
 }
 async function currentBaselineRowCount(input, targetTable, primaryKeyColumns) {
   if (sameColumns(input.validatedPrimaryKeyColumns, primaryKeyColumns)) {
-    return input.previousRowCount ?? countRows2(input.runtime, targetTable);
+    return input.previousRowCount ?? countRows3(input.runtime, targetTable);
   }
   return inspectUniqueKeyedRelation({
     runtime: input.runtime,
@@ -4920,9 +5050,10 @@ function primaryKeyMatchSql(primaryKeyColumns, leftAlias, rightAlias) {
   return primaryKeyColumns.map((column) => `${leftAlias}.${quoteIdentifier(column)} = ${rightAlias}.${quoteIdentifier(column)}`).join(" AND ");
 }
 function datasetRowDifferenceSql(dataset, leftAlias, rightAlias) {
-  return dataset.schema.columns.map((column) => `${leftAlias}.${quoteIdentifier(column.name)} IS DISTINCT FROM ${rightAlias}.${quoteIdentifier(column.name)}`).join(" OR ");
+  const differences = dataset.schema.columns.filter((column) => column.name !== dataset.sequenceBy).map((column) => `${leftAlias}.${quoteIdentifier(column.name)} IS DISTINCT FROM ${rightAlias}.${quoteIdentifier(column.name)}`);
+  return differences.length > 0 ? differences.join(" OR ") : "FALSE";
 }
-async function countRows2(runtime, relationSql) {
+async function countRows3(runtime, relationSql) {
   const [row] = await runtime.query(`SELECT count(*) AS row_count FROM ${relationSql}`);
   return row === undefined ? 0 : Number(getBigIntLike(row, "row_count"));
 }
@@ -5399,13 +5530,22 @@ class DuckLakeWriteCoordinator {
         validatedPrimaryKeyColumns: latestVersion?.validatedPrimaryKeyColumns
       });
       input.signal?.throwIfAborted();
-      const noOpBase = changeResult.dataChangeExpected ? undefined : latestVersion === undefined ? await this.latestVersionAtReadSnapshot(input) : latestVersion;
+      const parent = latestVersion === undefined ? await this.latestVersionAtReadSnapshot(input) : latestVersion;
+      const noOpBase = changeResult.dataChangeExpected ? undefined : parent;
       const createsVersion = noOpBase === undefined || this.noOpCreatesVersion(input, noOpBase, lineage);
       if (createsVersion && mustClaimTable(input, changeResult, lineage)) {
         await this.claimTable(input);
       }
       const commitId = randomUUID8();
-      await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion), lineage);
+      await this.setCommitMetadata(input, commitId, commitRowCount(input, changeResult, latestVersion), lineage, {
+        rowsChanged: parent === null || changeResult.changedTableRows,
+        lineageChanged: hasDatasetLineageChanges(parent, {
+          datasetId: input.dataset.id,
+          schema: input.dataset.schema,
+          producer: input.producer,
+          lineage
+        })
+      });
       await commitVersionTransaction(input.runtime, input.dataset.id, createsVersion && mustCommitOnReadSnapshot(input, changeResult) ? { poisonRuntime: () => this.connections.poisonRuntime() } : undefined);
       committed = true;
       const committedWriteSnapshotId = await this.lastCommittedSnapshotId(input.runtime);
@@ -5475,7 +5615,7 @@ class DuckLakeWriteCoordinator {
     }
     return result;
   }
-  async setCommitMetadata(input, commitId, rowCount, lineage) {
+  async setCommitMetadata(input, commitId, rowCount, lineage, changed) {
     const validatedPrimaryKeyColumns = getDatasetPrimaryKeyColumns4(input.dataset);
     const metadata = {
       kind: "datasetVersion",
@@ -5486,7 +5626,9 @@ class DuckLakeWriteCoordinator {
       inputs: input.inputs ? structuredClone(input.inputs) : undefined,
       ...rowCount.kind === "exact" ? { rowCount: rowCount.value } : {},
       ...validatedPrimaryKeyColumns !== null ? { validatedPrimaryKeyColumns: [...validatedPrimaryKeyColumns] } : {},
-      ...lineage === undefined ? {} : { lineage: encodeLineage(lineage) }
+      ...lineage === undefined ? {} : { lineage: encodeLineage(lineage) },
+      rowsChanged: changed.rowsChanged,
+      lineageChanged: changed.lineageChanged
     };
     await input.runtime.run(`CALL ${quoteIdentifier(duckLakeAlias(this.options))}.set_commit_message(${quoteSqlString("Sixb")}, ${quoteSqlString(input.commitMessage)}, extra_info => ${quoteSqlString(JSON.stringify({ sixb: metadata }))})`);
   }
@@ -5737,4 +5879,4 @@ export {
   DuckLakeStorage
 };
 
-//# debugId=9F26B0918A26BCC464756E2164756E21
+//# debugId=11CCDDEF58B2058164756E2164756E21