@leonardovida-md/drizzle-neo-duckdb 1.3.2 → 1.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -136,10 +136,7 @@ DuckDB executes one query per connection. The async `drizzle()` entrypoints crea
136
136
 
137
137
  ```typescript
138
138
  import { DuckDBInstance } from '@duckdb/node-api';
139
- import {
140
- createDuckDBConnectionPool,
141
- drizzle,
142
- } from '@duckdbfan/drizzle-duckdb';
139
+ import { createDuckDBConnectionPool, drizzle } from '@duckdbfan/drizzle-duckdb';
143
140
 
144
141
  const instance = await DuckDBInstance.create('md:', {
145
142
  motherduck_token: process.env.MOTHERDUCK_TOKEN,
package/dist/driver.d.ts CHANGED
@@ -13,6 +13,7 @@ import { DuckDBSelectBuilder } from './select-builder.ts';
13
13
  import type { ExecuteBatchesRawChunk, ExecuteInBatchesOptions, RowData } from './client.ts';
14
14
  import { type DuckDBPoolConfig, type PoolPreset } from './pool.ts';
15
15
  import { type PreparedStatementCacheConfig, type PrepareCacheOption } from './options.ts';
16
+ import { type DuckLakeConfig } from './ducklake.ts';
16
17
  export interface PgDriverOptions {
17
18
  logger?: Logger;
18
19
  rejectStringArrayLiterals?: boolean;
@@ -38,6 +39,8 @@ export interface DuckDBDrizzleConfig<TSchema extends Record<string, unknown> = R
38
39
  prepareCache?: PrepareCacheOption;
39
40
  /** Pool configuration. Use preset name, size config, or false to disable. */
40
41
  pool?: DuckDBPoolConfig | PoolPreset | false;
42
+ /** Optional DuckLake configuration */
43
+ ducklake?: DuckLakeConfig;
41
44
  }
42
45
  export interface DuckDBDrizzleConfigWithConnection<TSchema extends Record<string, unknown> = Record<string, never>> extends DuckDBDrizzleConfig<TSchema> {
43
46
  /** Connection string or config object */
@@ -599,6 +599,16 @@ function isSavepointSyntaxError(error) {
599
599
  }
600
600
  return error.message.toLowerCase().includes("savepoint") && error.message.toLowerCase().includes("syntax error");
601
601
  }
602
+ var VALID_TRANSACTION_ISOLATION_LEVELS = new Set([
603
+ "read uncommitted",
604
+ "read committed",
605
+ "repeatable read",
606
+ "serializable"
607
+ ]);
608
+ var VALID_TRANSACTION_ACCESS_MODES = new Set([
609
+ "read only",
610
+ "read write"
611
+ ]);
602
612
 
603
613
  class DuckDBPreparedQuery extends PgPreparedQuery {
604
614
  client;
@@ -780,6 +790,15 @@ class DuckDBTransaction extends PgTransaction {
780
790
  throw new TransactionRollbackError;
781
791
  }
782
792
  getTransactionConfigSQL(config) {
793
+ if (config.isolationLevel && !VALID_TRANSACTION_ISOLATION_LEVELS.has(config.isolationLevel)) {
794
+ throw new Error(`Invalid transaction isolation level "${config.isolationLevel}". Expected one of: ${Array.from(VALID_TRANSACTION_ISOLATION_LEVELS).join(", ")}.`);
795
+ }
796
+ if (config.accessMode && !VALID_TRANSACTION_ACCESS_MODES.has(config.accessMode)) {
797
+ throw new Error(`Invalid transaction access mode "${config.accessMode}". Expected one of: ${Array.from(VALID_TRANSACTION_ACCESS_MODES).join(", ")}.`);
798
+ }
799
+ if (config.deferrable !== undefined && typeof config.deferrable !== "boolean") {
800
+ throw new Error(`Invalid transaction deferrable flag "${String(config.deferrable)}". Expected a boolean.`);
801
+ }
783
802
  const chunks = [];
784
803
  if (config.isolationLevel) {
785
804
  chunks.push(`isolation level ${config.isolationLevel}`);
@@ -1891,6 +1910,7 @@ function createDuckDBConnectionPool(instance, options = {}) {
1891
1910
  const maxWaitingRequests = options.maxWaitingRequests ?? 100;
1892
1911
  const maxLifetimeMs = options.maxLifetimeMs;
1893
1912
  const idleTimeoutMs = options.idleTimeoutMs;
1913
+ const setup = options.setup;
1894
1914
  const metadata = new WeakMap;
1895
1915
  const idle = [];
1896
1916
  const waiting = [];
@@ -1931,6 +1951,14 @@ function createDuckDBConnectionPool(instance, options = {}) {
1931
1951
  total += 1;
1932
1952
  try {
1933
1953
  const connection = await DuckDBConnection.create(instance);
1954
+ if (setup) {
1955
+ try {
1956
+ await setup(connection);
1957
+ } catch (error) {
1958
+ await closeClientConnection(connection);
1959
+ throw error;
1960
+ }
1961
+ }
1934
1962
  if (closed) {
1935
1963
  await closeClientConnection(connection);
1936
1964
  total -= 1;
@@ -2054,6 +2082,151 @@ function resolvePrepareCacheOption(option) {
2054
2082
  return { size: Math.max(1, Math.floor(size)) };
2055
2083
  }
2056
2084
 
2085
+ // src/ducklake.ts
2086
+ var DEFAULT_ALIAS = "ducklake";
2087
+ function escapeSqlString(value) {
2088
+ return value.replace(/'/g, "''");
2089
+ }
2090
+ function quoteString(value) {
2091
+ return `'${escapeSqlString(value)}'`;
2092
+ }
2093
+ function quoteIdentifier(value) {
2094
+ return `"${value.replace(/"/g, '""')}"`;
2095
+ }
2096
+ function isDuckLakeUri(catalog) {
2097
+ return catalog.startsWith("ducklake:");
2098
+ }
2099
+ function normalizeDuckLakeConfig(config) {
2100
+ if (!config.catalog) {
2101
+ throw new Error("DuckLake config requires a catalog");
2102
+ }
2103
+ const catalog = isDuckLakeUri(config.catalog) ? config.catalog : `ducklake:${config.catalog}`;
2104
+ return {
2105
+ catalog,
2106
+ alias: config.alias?.trim() || DEFAULT_ALIAS,
2107
+ use: config.use ?? true,
2108
+ install: config.install ?? false,
2109
+ load: config.load ?? false,
2110
+ attachOptions: config.attachOptions
2111
+ };
2112
+ }
2113
+ function optionValueToSql(value) {
2114
+ if (typeof value === "boolean") {
2115
+ return value ? "true" : "false";
2116
+ }
2117
+ if (typeof value === "number") {
2118
+ return Number.isFinite(value) ? String(value) : "0";
2119
+ }
2120
+ return quoteString(value);
2121
+ }
2122
+ function buildDuckLakeAttachSql(config) {
2123
+ const normalized = normalizeDuckLakeConfig(config);
2124
+ const options = [];
2125
+ const attachOptions = normalized.attachOptions;
2126
+ if (attachOptions) {
2127
+ if (attachOptions.createIfNotExists !== undefined) {
2128
+ options.push(`CREATE_IF_NOT_EXISTS=${optionValueToSql(attachOptions.createIfNotExists)}`);
2129
+ }
2130
+ if (attachOptions.dataInliningRowLimit !== undefined) {
2131
+ options.push(`DATA_INLINING_ROW_LIMIT=${optionValueToSql(attachOptions.dataInliningRowLimit)}`);
2132
+ }
2133
+ if (attachOptions.dataPath) {
2134
+ options.push(`DATA_PATH=${optionValueToSql(attachOptions.dataPath)}`);
2135
+ }
2136
+ if (attachOptions.encrypted !== undefined) {
2137
+ options.push(`ENCRYPTED=${optionValueToSql(attachOptions.encrypted)}`);
2138
+ }
2139
+ if (attachOptions.metaParameterName) {
2140
+ options.push(`META_PARAMETER_NAME=${optionValueToSql(attachOptions.metaParameterName)}`);
2141
+ }
2142
+ if (attachOptions.metadataCatalog) {
2143
+ options.push(`METADATA_CATALOG=${optionValueToSql(attachOptions.metadataCatalog)}`);
2144
+ }
2145
+ if (attachOptions.overrideDataPath !== undefined) {
2146
+ options.push(`OVERRIDE_DATA_PATH=${optionValueToSql(attachOptions.overrideDataPath)}`);
2147
+ }
2148
+ if (attachOptions.readOnly !== undefined) {
2149
+ options.push(`READ_ONLY=${optionValueToSql(attachOptions.readOnly)}`);
2150
+ }
2151
+ }
2152
+ const attachSql = [
2153
+ `ATTACH ${quoteString(normalized.catalog)} AS ${quoteIdentifier(normalized.alias)}`
2154
+ ];
2155
+ if (options.length > 0) {
2156
+ attachSql.push(`(${options.join(", ")})`);
2157
+ }
2158
+ return attachSql.join(" ");
2159
+ }
2160
+ async function configureDuckLake(connection, config) {
2161
+ const normalized = normalizeDuckLakeConfig(config);
2162
+ if (normalized.install) {
2163
+ await connection.run("INSTALL ducklake");
2164
+ }
2165
+ if (normalized.load) {
2166
+ await connection.run("LOAD ducklake");
2167
+ }
2168
+ const attachSql = buildDuckLakeAttachSql(normalized);
2169
+ await connection.run(attachSql);
2170
+ if (normalized.use) {
2171
+ await connection.run(`USE ${quoteIdentifier(normalized.alias)}`);
2172
+ }
2173
+ }
2174
+ function wrapDuckLakePool(pool, config) {
2175
+ const configuredConnections = new WeakSet;
2176
+ const poolWithSize = pool;
2177
+ const size = typeof poolWithSize.size === "number" ? poolWithSize.size : undefined;
2178
+ const wrapped = {
2179
+ async acquire() {
2180
+ const connection = await pool.acquire();
2181
+ if (!configuredConnections.has(connection)) {
2182
+ await configureDuckLake(connection, config);
2183
+ configuredConnections.add(connection);
2184
+ }
2185
+ return connection;
2186
+ },
2187
+ release(connection) {
2188
+ return pool.release(connection);
2189
+ },
2190
+ close: pool.close?.bind(pool)
2191
+ };
2192
+ if (size !== undefined) {
2193
+ wrapped.size = size;
2194
+ }
2195
+ return wrapped;
2196
+ }
2197
+ function isDuckDbFileCatalog(catalog) {
2198
+ const trimmed = catalog.trim();
2199
+ if (!trimmed)
2200
+ return false;
2201
+ if (trimmed === ":memory:")
2202
+ return true;
2203
+ if (isDuckLakeUri(trimmed)) {
2204
+ return isDuckDbFileCatalog(trimmed.slice("ducklake:".length));
2205
+ }
2206
+ if (/^[a-zA-Z]:[\\/]/.test(trimmed))
2207
+ return true;
2208
+ if (trimmed.startsWith("md:"))
2209
+ return false;
2210
+ if (/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(trimmed))
2211
+ return false;
2212
+ if (/^(postgres|postgresql|mysql|mariadb|sqlite|snowflake|bigquery):/i.test(trimmed)) {
2213
+ return false;
2214
+ }
2215
+ return trimmed.endsWith(".duckdb") || trimmed.endsWith(".ddb") || trimmed.includes("/") || trimmed.includes("\\") || trimmed.startsWith("./") || trimmed.startsWith("../") || trimmed.startsWith("~");
2216
+ }
2217
+ function resolveDuckLakePoolSize(poolSetting, ducklakeConfig) {
2218
+ const hasPoolSetting = poolSetting !== undefined;
2219
+ const resolvedPoolSize = resolvePoolSize(poolSetting);
2220
+ const isLocalCatalog = ducklakeConfig !== undefined ? isDuckDbFileCatalog(normalizeDuckLakeConfig(ducklakeConfig).catalog) : false;
2221
+ const poolSize = !hasPoolSetting && isLocalCatalog ? 1 : resolvedPoolSize;
2222
+ return {
2223
+ poolSize,
2224
+ resolvedPoolSize,
2225
+ isLocalCatalog,
2226
+ hasPoolSetting
2227
+ };
2228
+ }
2229
+
2057
2230
  // src/driver.ts
2058
2231
  class DuckDBDriver {
2059
2232
  client;
@@ -2081,6 +2254,14 @@ function isConfigObject(data) {
2081
2254
  return "connection" in data || "client" in data || "pool" in data || "schema" in data || "logger" in data;
2082
2255
  }
2083
2256
  function createFromClient(client, config = {}, instance) {
2257
+ let finalClient = client;
2258
+ if (config.ducklake) {
2259
+ if (isPool(client)) {
2260
+ finalClient = wrapDuckLakePool(client, config.ducklake);
2261
+ } else {
2262
+ throw new Error('DuckLake configuration requires a connection string or pool. Use drizzle("path", { ducklake: ... }) or call configureDuckLake(connection, config) manually.');
2263
+ }
2264
+ }
2084
2265
  const dialect = new DuckDBDialect;
2085
2266
  const prepareCache = resolvePrepareCacheOption(config.prepareCache);
2086
2267
  const logger = config.logger === true ? new DefaultLogger : config.logger || undefined;
@@ -2093,24 +2274,38 @@ function createFromClient(client, config = {}, instance) {
2093
2274
  tableNamesMap: tablesConfig.tableNamesMap
2094
2275
  };
2095
2276
  }
2096
- const driver = new DuckDBDriver(client, dialect, {
2277
+ const driver = new DuckDBDriver(finalClient, dialect, {
2097
2278
  logger,
2098
2279
  rejectStringArrayLiterals: config.rejectStringArrayLiterals,
2099
2280
  prepareCache
2100
2281
  });
2101
2282
  const session = driver.createSession(schema);
2102
- const db = new DuckDBDatabase(dialect, session, schema, client, instance);
2283
+ const db = new DuckDBDatabase(dialect, session, schema, finalClient, instance);
2103
2284
  return db;
2104
2285
  }
2105
2286
  async function createFromConnectionString(path, instanceOptions, config = {}) {
2106
2287
  const instance = await DuckDBInstance2.create(path, instanceOptions);
2107
- const poolSize = resolvePoolSize(config.pool);
2288
+ const ducklakeConfig = config.ducklake;
2289
+ const { poolSize, resolvedPoolSize, isLocalCatalog } = resolveDuckLakePoolSize(config.pool, ducklakeConfig);
2290
+ if (ducklakeConfig && resolvedPoolSize !== false && typeof resolvedPoolSize === "number" && resolvedPoolSize > 1 && isLocalCatalog) {
2291
+ console.warn("[ducklake] DuckDB catalog files support a single client connection. Pool sizes greater than 1 can cause write conflicts.");
2292
+ }
2108
2293
  if (poolSize === false) {
2109
2294
  const connection = await instance.connect();
2110
- return createFromClient(connection, config, instance);
2295
+ if (ducklakeConfig) {
2296
+ await configureDuckLake(connection, ducklakeConfig);
2297
+ }
2298
+ const { ducklake: ducklake2, ...restConfig2 } = config;
2299
+ return createFromClient(connection, restConfig2, instance);
2111
2300
  }
2112
- const pool = createDuckDBConnectionPool(instance, { size: poolSize });
2113
- return createFromClient(pool, config, instance);
2301
+ const pool = createDuckDBConnectionPool(instance, {
2302
+ size: poolSize,
2303
+ setup: ducklakeConfig ? async (connection) => {
2304
+ await configureDuckLake(connection, ducklakeConfig);
2305
+ } : undefined
2306
+ });
2307
+ const { ducklake, ...restConfig } = config;
2308
+ return createFromClient(pool, restConfig, instance);
2114
2309
  }
2115
2310
  function drizzle(clientOrConfigOrPath, config) {
2116
2311
  if (typeof clientOrConfigOrPath === "string") {
@@ -2764,6 +2959,19 @@ function parseArgs(argv) {
2764
2959
  includeViews: false,
2765
2960
  useCustomTimeTypes: true
2766
2961
  };
2962
+ const ensureDuckLakeConfig = () => {
2963
+ if (!options.ducklake) {
2964
+ options.ducklake = { catalog: "" };
2965
+ }
2966
+ return options.ducklake;
2967
+ };
2968
+ const ensureDuckLakeAttachOptions = () => {
2969
+ const config = ensureDuckLakeConfig();
2970
+ if (!config.attachOptions) {
2971
+ config.attachOptions = {};
2972
+ }
2973
+ return config.attachOptions;
2974
+ };
2767
2975
  for (let i = 0;i < argv.length; i += 1) {
2768
2976
  const arg = argv[i];
2769
2977
  switch (arg) {
@@ -2800,6 +3008,50 @@ function parseArgs(argv) {
2800
3008
  case "--import-base":
2801
3009
  options.importBasePath = argv[++i];
2802
3010
  break;
3011
+ case "--ducklake-catalog":
3012
+ ensureDuckLakeConfig().catalog = argv[++i] ?? "";
3013
+ break;
3014
+ case "--ducklake-alias":
3015
+ ensureDuckLakeConfig().alias = argv[++i];
3016
+ break;
3017
+ case "--ducklake-no-use":
3018
+ ensureDuckLakeConfig().use = false;
3019
+ break;
3020
+ case "--ducklake-install":
3021
+ ensureDuckLakeConfig().install = true;
3022
+ break;
3023
+ case "--ducklake-load":
3024
+ ensureDuckLakeConfig().load = true;
3025
+ break;
3026
+ case "--ducklake-data-path":
3027
+ ensureDuckLakeAttachOptions().dataPath = argv[++i];
3028
+ break;
3029
+ case "--ducklake-read-only":
3030
+ ensureDuckLakeAttachOptions().readOnly = true;
3031
+ break;
3032
+ case "--ducklake-create-if-not-exists":
3033
+ ensureDuckLakeAttachOptions().createIfNotExists = true;
3034
+ break;
3035
+ case "--ducklake-override-data-path":
3036
+ ensureDuckLakeAttachOptions().overrideDataPath = true;
3037
+ break;
3038
+ case "--ducklake-data-inlining-row-limit": {
3039
+ const value = argv[++i];
3040
+ const parsed = value ? Number(value) : NaN;
3041
+ if (Number.isFinite(parsed)) {
3042
+ ensureDuckLakeAttachOptions().dataInliningRowLimit = parsed;
3043
+ }
3044
+ break;
3045
+ }
3046
+ case "--ducklake-encrypted":
3047
+ ensureDuckLakeAttachOptions().encrypted = true;
3048
+ break;
3049
+ case "--ducklake-metadata-catalog":
3050
+ ensureDuckLakeAttachOptions().metadataCatalog = argv[++i];
3051
+ break;
3052
+ case "--ducklake-meta-parameter-name":
3053
+ ensureDuckLakeAttachOptions().metaParameterName = argv[++i];
3054
+ break;
2803
3055
  case "--help":
2804
3056
  case "-h":
2805
3057
  printHelp();
@@ -2828,6 +3080,19 @@ Options:
2828
3080
  --include-views Include views in the generated schema
2829
3081
  --use-pg-time Use pg-core timestamp/date/time instead of DuckDB custom helpers
2830
3082
  --import-base Override import path for duckdb helpers (default: package name)
3083
+ --ducklake-catalog DuckLake catalog value after the ducklake: prefix
3084
+ --ducklake-alias Alias for attached DuckLake database
3085
+ --ducklake-no-use Do not run USE after attach
3086
+ --ducklake-install Run INSTALL ducklake before attach
3087
+ --ducklake-load Run LOAD ducklake before attach
3088
+ --ducklake-data-path Data path for DuckLake table storage
3089
+ --ducklake-read-only Attach DuckLake in read-only mode
3090
+ --ducklake-create-if-not-exists Create catalog if it does not exist
3091
+ --ducklake-override-data-path Override data path for existing catalog
3092
+ --ducklake-data-inlining-row-limit Inline row limit for data storage
3093
+ --ducklake-encrypted Enable encryption for the metadata catalog
3094
+ --ducklake-metadata-catalog Override metadata catalog name
3095
+ --ducklake-meta-parameter-name Set meta parameter name for metadata storage
2831
3096
 
2832
3097
  Database Filtering:
2833
3098
  By default, only tables from the current database are introspected. This prevents
@@ -2842,6 +3107,14 @@ Examples:
2842
3107
 
2843
3108
  # MotherDuck (requires MOTHERDUCK_TOKEN env var)
2844
3109
  MOTHERDUCK_TOKEN=xxx bun x duckdb-introspect --url md: --database my_cloud_db --out ./schema.ts
3110
+
3111
+ # DuckLake local catalog with data path
3112
+ bun x duckdb-introspect --url :memory: --ducklake-catalog ./ducklake.duckdb \\
3113
+ --ducklake-data-path ./ducklake-data --out ./schema.ts
3114
+
3115
+ # DuckLake on MotherDuck
3116
+ MOTHERDUCK_TOKEN=xxx bun x duckdb-introspect --url md: \\
3117
+ --ducklake-catalog md:__ducklake_metadata_my_db --out ./schema.ts
2845
3118
  `);
2846
3119
  }
2847
3120
  async function main() {
@@ -2855,6 +3128,12 @@ async function main() {
2855
3128
  const connection = await instance.connect();
2856
3129
  const db = drizzle(connection);
2857
3130
  try {
3131
+ if (options.ducklake && !options.ducklake.catalog) {
3132
+ throw new Error("DuckLake requires --ducklake-catalog");
3133
+ }
3134
+ if (options.ducklake?.catalog) {
3135
+ await configureDuckLake(connection, options.ducklake);
3136
+ }
2858
3137
  const result = await introspect(db, {
2859
3138
  database: options.database,
2860
3139
  allDatabases: options.allDatabases,
@@ -0,0 +1,41 @@
1
+ import type { DuckDBConnection } from '@duckdb/node-api';
2
+ import type { DuckDBConnectionPool } from './client.ts';
3
+ import { type DuckDBPoolConfig, type PoolPreset } from './pool.ts';
4
+ export interface DuckLakeAttachOptions {
5
+ createIfNotExists?: boolean;
6
+ dataInliningRowLimit?: number;
7
+ dataPath?: string;
8
+ encrypted?: boolean;
9
+ metaParameterName?: string;
10
+ metadataCatalog?: string;
11
+ overrideDataPath?: boolean;
12
+ readOnly?: boolean;
13
+ }
14
+ export interface DuckLakeConfig {
15
+ catalog: string;
16
+ alias?: string;
17
+ use?: boolean;
18
+ install?: boolean;
19
+ load?: boolean;
20
+ attachOptions?: DuckLakeAttachOptions;
21
+ }
22
+ export interface NormalizedDuckLakeConfig {
23
+ catalog: string;
24
+ alias: string;
25
+ use: boolean;
26
+ install: boolean;
27
+ load: boolean;
28
+ attachOptions?: DuckLakeAttachOptions;
29
+ }
30
+ export interface DuckLakePoolResolution {
31
+ poolSize: number | false;
32
+ resolvedPoolSize: number | false;
33
+ isLocalCatalog: boolean;
34
+ hasPoolSetting: boolean;
35
+ }
36
+ export declare function normalizeDuckLakeConfig(config: DuckLakeConfig): NormalizedDuckLakeConfig;
37
+ export declare function buildDuckLakeAttachSql(config: DuckLakeConfig): string;
38
+ export declare function configureDuckLake(connection: DuckDBConnection, config: DuckLakeConfig): Promise<void>;
39
+ export declare function wrapDuckLakePool(pool: DuckDBConnectionPool, config: DuckLakeConfig): DuckDBConnectionPool;
40
+ export declare function isDuckDbFileCatalog(catalog: string): boolean;
41
+ export declare function resolveDuckLakePoolSize(poolSetting: DuckDBPoolConfig | PoolPreset | false | undefined, ducklakeConfig: DuckLakeConfig | undefined): DuckLakePoolResolution;
package/dist/index.d.ts CHANGED
@@ -9,3 +9,4 @@ export * from './olap.ts';
9
9
  export * from './value-wrappers.ts';
10
10
  export * from './options.ts';
11
11
  export * from './operators.ts';
12
+ export { configureDuckLake, wrapDuckLakePool, type DuckLakeAttachOptions, type DuckLakeConfig, } from './ducklake.ts';