@sixb/ducklake 0.2.1 → 0.2.2

package/dist/internal/duckdb-extensions.d.ts

@@ -0,0 +1,15 @@
+import type { DuckDbQueryRuntime } from "./duckdb-runtime";
+/**
+ * Install missing core extensions from `https://extensions.duckdb.org`.
+ *
+ * DuckDB's `INSTALL` downloads from `http://extensions.duckdb.org`, and fetching over HTTPS needs
+ * `httpfs`, which DuckDB itself fetches over HTTP. A host that only allows HTTPS by name, such as
+ * a Vercel sandbox whose firewall filters on TLS SNI, can reach neither. So the file is fetched
+ * here and handed to `INSTALL '<file>'`, which puts it where DuckDB looks; the plain `INSTALL`
+ * that follows is then a no-op, and `LOAD` still checks the extension's signature.
+ *
+ * Best effort: a failure is logged and leaves the extension to DuckDB's own `INSTALL`. A project
+ * that set `custom_extension_repository` keeps its own repository.
+ */
+export declare function prefetchDuckDbExtensions(runtime: DuckDbQueryRuntime, extensions: readonly string[]): Promise<void>;
+//# sourceMappingURL=duckdb-extensions.d.ts.map

package/dist/internal/duckdb-extensions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"duckdb-extensions.d.ts","sourceRoot":"","sources":["../../src/internal/duckdb-extensions.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAO1D;;;;;;;;;;;GAWG;AACH,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,kBAAkB,EAC3B,UAAU,EAAE,SAAS,MAAM,EAAE,GAC5B,OAAO,CAAC,IAAI,CAAC,CAIf"}

package/dist/internal/duckdb-runtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"duckdb-runtime.d.ts","sourceRoot":"","sources":["../../src/internal/duckdb-runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EAGL,KAAK,WAAW,EAEjB,MAAM,kBAAkB,CAAA;AAEzB,OAAO,KAAK,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAC5E,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAoB,MAAM,iBAAiB,CAAA;AAW/F;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChE,aAAa,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC3D,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,WAAW,EAAE,GAAG,OAAO,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAChG,YAAY,CAAC,CAAC,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GACxD,OAAO,CAAC,CAAC,CAAC,CAAA;CACd;AAED,MAAM,WAAW,aAAc,SAAQ,kBAAkB;IACvD,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAC7E,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;IACzF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAED;;;;;GAKG;AACH,MAAM,MAAM,sBAAsB,GAAG,kBAAkB,CAAA;AAEvD,MAAM,WAAW,cAAc;IAC7B,UAAU,IAAI,IAAI,CAAA;IAClB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAA;IACnC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IAClC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,GAAG,IAAI,CAAA;IAChE,MAAM,IAAI,IAAI,CAAA;CACf;AAED,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAC1B;AAsND;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,aAAa,CAAC,CAuBxB;AAED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,sBAAsB,EAC/B,YAAY,GAAE,oBAAyB,GACtC,OAAO,CAAC,IAAI,CAAC,CA8Bf"}
+{"version":3,"file":"duckdb-runtime.d.ts","sourceRoot":"","sources":["../../src/internal/duckdb-runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EAGL,KAAK,WAAW,EAEjB,MAAM,kBAAkB,CAAA;AAEzB,OAAO,KAAK,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAA;AAE5E,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAoB,MAAM,iBAAiB,CAAA;AAW/F;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChE,aAAa,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC3D,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,WAAW,EAAE,GAAG,OAAO,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAChG,YAAY,CAAC,CAAC,EACZ,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GACxD,OAAO,CAAC,CAAC,CAAC,CAAA;CACd;AAED,MAAM,WAAW,aAAc,SAAQ,kBAAkB;IACvD,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAC7E,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,OAAO,EAAE,sBAAsB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;IACzF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAED;;;;;GAKG;AACH,MAAM,MAAM,sBAAsB,GAAG,kBAAkB,CAAA;AAEvD,MAAM,WAAW,cAAc;IAC7B,UAAU,IAAI,IAAI,CAAA;IAClB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAA;IACnC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IAClC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,GAAG,IAAI,CAAA;IAChE,MAAM,IAAI,IAAI,CAAA;CACf;AAED,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;IACpC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAC1B;AAsND;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,aAAa,CAAC,CAuBxB;AAED;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,IAAI,CAAC,CASf;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,sBAAsB,EAC/B,YAAY,GAAE,oBAAyB,GACtC,OAAO,CAAC,IAAI,CAAC,CA8Bf"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sixb/ducklake",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "DuckDB/DuckLake lake storage provider for Sixb",
   "keywords": [
     "sixb",
@@ -36,7 +36,7 @@
     "typecheck": "tsc --noEmit",
     "test": "bun test",
     "test:e2e": "bun run test:e2e:local && bun run test:e2e:remote",
-    "test:e2e:local": "bun test --preload ./tests/extensions.ts ./tests/ducklake-reader.e2e.ts ./tests/ducklake-changes.e2e.ts ./tests/ducklake-contract.e2e.ts ./tests/ducklake-catalog-state.e2e.ts ./tests/ducklake-catalogs.e2e.ts ./tests/ducklake-concurrency.e2e.ts ./tests/ducklake-datasets.e2e.ts ./tests/ducklake-driver.e2e.ts ./tests/ducklake-maintenance.e2e.ts ./tests/ducklake-merges.e2e.ts ./tests/ducklake-sql-transforms.e2e.ts ./tests/ducklake-versions.e2e.ts ./tests/ducklake-long-history.e2e.ts ./tests/ducklake-writes.e2e.ts ./tests/ducklake-sqlite-writes.e2e.ts",
+    "test:e2e:local": "bun test --preload ./tests/extensions.ts ./tests/duckdb-extensions.e2e.ts ./tests/ducklake-reader.e2e.ts ./tests/ducklake-changes.e2e.ts ./tests/ducklake-contract.e2e.ts ./tests/ducklake-catalog-state.e2e.ts ./tests/ducklake-catalogs.e2e.ts ./tests/ducklake-concurrency.e2e.ts ./tests/ducklake-datasets.e2e.ts ./tests/ducklake-driver.e2e.ts ./tests/ducklake-maintenance.e2e.ts ./tests/ducklake-merges.e2e.ts ./tests/ducklake-sql-transforms.e2e.ts ./tests/ducklake-versions.e2e.ts ./tests/ducklake-long-history.e2e.ts ./tests/ducklake-writes.e2e.ts ./tests/ducklake-sqlite-writes.e2e.ts",
     "test:e2e:remote": "bun test --preload ./tests/extensions.ts --preload ./tests/setup.ts ./tests/ducklake-remote.e2e.ts",
     "prepack": "bun run build"
   },
@@ -44,10 +44,10 @@
     "@duckdb/node-api": "1.5.2-r.2"
   },
   "peerDependencies": {
-    "@sixb/core": "^0.2.1"
+    "@sixb/core": "^0.2.2"
   },
   "devDependencies": {
-    "@sixb/core": "0.2.1",
+    "@sixb/core": "0.2.2",
     "@types/bun": "latest",
     "typescript": "^5.7.0"
   },

package/src/internal/duckdb-extensions.ts

@@ -0,0 +1,102 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { promisify } from "node:util"
+import { gunzip } from "node:zlib"
+import type { DuckDbQueryRuntime } from "./duckdb-runtime"
+import { quoteSqlString, quoteSqlStringList } from "./sql"
+
+const extensionsUrl = "https://extensions.duckdb.org"
+const downloadTimeoutMillis = 60_000
+const gunzipAsync = promisify(gunzip)
+
+/**
+ * Install missing core extensions from `https://extensions.duckdb.org`.
+ *
+ * DuckDB's `INSTALL` downloads from `http://extensions.duckdb.org`, and fetching over HTTPS needs
+ * `httpfs`, which DuckDB itself fetches over HTTP. A host that only allows HTTPS by name, such as
+ * a Vercel sandbox whose firewall filters on TLS SNI, can reach neither. So the file is fetched
+ * here and handed to `INSTALL '<file>'`, which puts it where DuckDB looks; the plain `INSTALL`
+ * that follows is then a no-op, and `LOAD` still checks the extension's signature.
+ *
+ * Best effort: a failure is logged and leaves the extension to DuckDB's own `INSTALL`. A project
+ * that set `custom_extension_repository` keeps its own repository.
+ */
+export async function prefetchDuckDbExtensions(
+  runtime: DuckDbQueryRuntime,
+  extensions: readonly string[]
+): Promise<void> {
+  await prefetch(runtime, extensions).catch((error: unknown) =>
+    warnFallback(extensions.join("', '"), error)
+  )
+}
+
+async function prefetch(runtime: DuckDbQueryRuntime, extensions: readonly string[]) {
+  const [target] = await runtime.query(`
+    SELECT
+      version() AS version,
+      (SELECT platform FROM pragma_platform()) AS platform,
+      current_setting('custom_extension_repository') AS repository
+  `)
+  if (target?.repository !== "") return
+
+  const missing = await missingExtensions(runtime, extensions)
+  if (missing.length === 0) return
+
+  const directory = await mkdtemp(join(tmpdir(), "sixb-duckdb-extensions-"))
+  try {
+    const downloads = await Promise.all(
+      missing.map(async (name) => ({
+        name,
+        file: await download(`${target.version}/${target.platform}/${name}`, directory, name).catch(
+          (error: unknown) => warnFallback(name, error)
+        ),
+      }))
+    )
+
+    // One at a time: the runtime runs statements in order anyway, and this keeps failures apart.
+    for (const { name, file } of downloads) {
+      if (file === undefined) continue
+      await runtime
+        .run(`INSTALL ${quoteSqlString(file)}`)
+        .catch((error: unknown) => warnFallback(name, error))
+    }
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+}
+
+function warnFallback(name: string, error: unknown): undefined {
+  const reason = error instanceof Error ? error.message : String(error)
+  console.warn(
+    `[SixbDuckLake] Could not install DuckDB extension '${name}' over HTTPS (${reason}); falling back to DuckDB's INSTALL.`
+  )
+  return undefined
+}
+
+/** Canonical names of the requested extensions not installed yet: `sqlite` is `sqlite_scanner`. */
+async function missingExtensions(
+  runtime: DuckDbQueryRuntime,
+  extensions: readonly string[]
+): Promise<string[]> {
+  const requested = `[${quoteSqlStringList(extensions.map((extension) => extension.toLowerCase()))}]`
+  const rows = await runtime.query(`
+    SELECT extension_name
+    FROM duckdb_extensions()
+    WHERE NOT installed
+      AND (list_contains(${requested}, extension_name) OR list_has_any(${requested}, aliases))
+  `)
+  return rows.map((row) => String(row.extension_name))
+}
+
+/** `INSTALL '<file>'` names the extension after the file, so it is written as `<name>.duckdb_extension`. */
+async function download(path: string, directory: string, name: string): Promise<string> {
+  const response = await fetch(`${extensionsUrl}/${path}.duckdb_extension.gz`, {
+    signal: AbortSignal.timeout(downloadTimeoutMillis),
+  })
+  if (!response.ok) throw new Error(`HTTP ${response.status}`)
+
+  const file = join(directory, `${name}.duckdb_extension`)
+  await writeFile(file, await gunzipAsync(new Uint8Array(await response.arrayBuffer())))
+  return file
+}

package/src/internal/duckdb-runtime.ts

@@ -8,6 +8,7 @@ import {
 } from "@duckdb/node-api"
 import { LakeStorageError } from "@sixb/core/lake-storage"
 import type { DuckDbRuntimeOptions, DuckLakeStorageOptions } from "../types"
+import { prefetchDuckDbExtensions } from "./duckdb-extensions"
 import { type DuckDbReader, type DuckDbReaderOptions, NodeDuckDbReader } from "./duckdb-reader"
 import { sixbDuckDbValueConverter } from "./duckdb-value-converter"
 import {
@@ -323,9 +324,12 @@ export async function installDuckLakeExtensions(
   runtime: DuckDbRuntime,
   options: DuckLakeStorageOptions
 ): Promise<void> {
+  const extensions = requiredExtensions(options)
+  await prefetchDuckDbExtensions(runtime, ["ducklake", ...extensions])
+
   await runtime.run("INSTALL ducklake")
 
-  for (const extension of requiredExtensions(options)) {
+  for (const extension of extensions) {
     await runtime.run(`INSTALL ${quoteIdentifier(extension)}`)
   }
 }