@leonardovida-md/drizzle-neo-duckdb 1.3.2 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -4
- package/dist/driver.d.ts +3 -0
- package/dist/duckdb-introspect.mjs +285 -6
- package/dist/ducklake.d.ts +41 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.mjs +203 -6
- package/dist/pool.d.ts +3 -1
- package/package.json +4 -1
- package/src/bin/duckdb-introspect.ts +91 -0
- package/src/driver.ts +66 -7
- package/src/ducklake.ts +256 -0
- package/src/index.ts +6 -0
- package/src/pool.ts +11 -0
- package/src/session.ts +45 -0
package/dist/index.mjs
CHANGED
|
@@ -650,6 +650,16 @@ function isSavepointSyntaxError(error) {
|
|
|
650
650
|
}
|
|
651
651
|
return error.message.toLowerCase().includes("savepoint") && error.message.toLowerCase().includes("syntax error");
|
|
652
652
|
}
|
|
653
|
+
var VALID_TRANSACTION_ISOLATION_LEVELS = new Set([
|
|
654
|
+
"read uncommitted",
|
|
655
|
+
"read committed",
|
|
656
|
+
"repeatable read",
|
|
657
|
+
"serializable"
|
|
658
|
+
]);
|
|
659
|
+
var VALID_TRANSACTION_ACCESS_MODES = new Set([
|
|
660
|
+
"read only",
|
|
661
|
+
"read write"
|
|
662
|
+
]);
|
|
653
663
|
|
|
654
664
|
class DuckDBPreparedQuery extends PgPreparedQuery {
|
|
655
665
|
client;
|
|
@@ -831,6 +841,15 @@ class DuckDBTransaction extends PgTransaction {
|
|
|
831
841
|
throw new TransactionRollbackError;
|
|
832
842
|
}
|
|
833
843
|
getTransactionConfigSQL(config) {
|
|
844
|
+
if (config.isolationLevel && !VALID_TRANSACTION_ISOLATION_LEVELS.has(config.isolationLevel)) {
|
|
845
|
+
throw new Error(`Invalid transaction isolation level "${config.isolationLevel}". Expected one of: ${Array.from(VALID_TRANSACTION_ISOLATION_LEVELS).join(", ")}.`);
|
|
846
|
+
}
|
|
847
|
+
if (config.accessMode && !VALID_TRANSACTION_ACCESS_MODES.has(config.accessMode)) {
|
|
848
|
+
throw new Error(`Invalid transaction access mode "${config.accessMode}". Expected one of: ${Array.from(VALID_TRANSACTION_ACCESS_MODES).join(", ")}.`);
|
|
849
|
+
}
|
|
850
|
+
if (config.deferrable !== undefined && typeof config.deferrable !== "boolean") {
|
|
851
|
+
throw new Error(`Invalid transaction deferrable flag "${String(config.deferrable)}". Expected a boolean.`);
|
|
852
|
+
}
|
|
834
853
|
const chunks = [];
|
|
835
854
|
if (config.isolationLevel) {
|
|
836
855
|
chunks.push(`isolation level ${config.isolationLevel}`);
|
|
@@ -1942,6 +1961,7 @@ function createDuckDBConnectionPool(instance, options = {}) {
|
|
|
1942
1961
|
const maxWaitingRequests = options.maxWaitingRequests ?? 100;
|
|
1943
1962
|
const maxLifetimeMs = options.maxLifetimeMs;
|
|
1944
1963
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
1964
|
+
const setup = options.setup;
|
|
1945
1965
|
const metadata = new WeakMap;
|
|
1946
1966
|
const idle = [];
|
|
1947
1967
|
const waiting = [];
|
|
@@ -1982,6 +2002,14 @@ function createDuckDBConnectionPool(instance, options = {}) {
|
|
|
1982
2002
|
total += 1;
|
|
1983
2003
|
try {
|
|
1984
2004
|
const connection = await DuckDBConnection.create(instance);
|
|
2005
|
+
if (setup) {
|
|
2006
|
+
try {
|
|
2007
|
+
await setup(connection);
|
|
2008
|
+
} catch (error) {
|
|
2009
|
+
await closeClientConnection(connection);
|
|
2010
|
+
throw error;
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
1985
2013
|
if (closed) {
|
|
1986
2014
|
await closeClientConnection(connection);
|
|
1987
2015
|
total -= 1;
|
|
@@ -2105,6 +2133,151 @@ function resolvePrepareCacheOption(option) {
|
|
|
2105
2133
|
return { size: Math.max(1, Math.floor(size)) };
|
|
2106
2134
|
}
|
|
2107
2135
|
|
|
2136
|
+
// src/ducklake.ts
|
|
2137
|
+
var DEFAULT_ALIAS = "ducklake";
|
|
2138
|
+
function escapeSqlString(value) {
|
|
2139
|
+
return value.replace(/'/g, "''");
|
|
2140
|
+
}
|
|
2141
|
+
function quoteString(value) {
|
|
2142
|
+
return `'${escapeSqlString(value)}'`;
|
|
2143
|
+
}
|
|
2144
|
+
function quoteIdentifier(value) {
|
|
2145
|
+
return `"${value.replace(/"/g, '""')}"`;
|
|
2146
|
+
}
|
|
2147
|
+
function isDuckLakeUri(catalog) {
|
|
2148
|
+
return catalog.startsWith("ducklake:");
|
|
2149
|
+
}
|
|
2150
|
+
function normalizeDuckLakeConfig(config) {
|
|
2151
|
+
if (!config.catalog) {
|
|
2152
|
+
throw new Error("DuckLake config requires a catalog");
|
|
2153
|
+
}
|
|
2154
|
+
const catalog = isDuckLakeUri(config.catalog) ? config.catalog : `ducklake:${config.catalog}`;
|
|
2155
|
+
return {
|
|
2156
|
+
catalog,
|
|
2157
|
+
alias: config.alias?.trim() || DEFAULT_ALIAS,
|
|
2158
|
+
use: config.use ?? true,
|
|
2159
|
+
install: config.install ?? false,
|
|
2160
|
+
load: config.load ?? false,
|
|
2161
|
+
attachOptions: config.attachOptions
|
|
2162
|
+
};
|
|
2163
|
+
}
|
|
2164
|
+
function optionValueToSql(value) {
|
|
2165
|
+
if (typeof value === "boolean") {
|
|
2166
|
+
return value ? "true" : "false";
|
|
2167
|
+
}
|
|
2168
|
+
if (typeof value === "number") {
|
|
2169
|
+
return Number.isFinite(value) ? String(value) : "0";
|
|
2170
|
+
}
|
|
2171
|
+
return quoteString(value);
|
|
2172
|
+
}
|
|
2173
|
+
function buildDuckLakeAttachSql(config) {
|
|
2174
|
+
const normalized = normalizeDuckLakeConfig(config);
|
|
2175
|
+
const options = [];
|
|
2176
|
+
const attachOptions = normalized.attachOptions;
|
|
2177
|
+
if (attachOptions) {
|
|
2178
|
+
if (attachOptions.createIfNotExists !== undefined) {
|
|
2179
|
+
options.push(`CREATE_IF_NOT_EXISTS=${optionValueToSql(attachOptions.createIfNotExists)}`);
|
|
2180
|
+
}
|
|
2181
|
+
if (attachOptions.dataInliningRowLimit !== undefined) {
|
|
2182
|
+
options.push(`DATA_INLINING_ROW_LIMIT=${optionValueToSql(attachOptions.dataInliningRowLimit)}`);
|
|
2183
|
+
}
|
|
2184
|
+
if (attachOptions.dataPath) {
|
|
2185
|
+
options.push(`DATA_PATH=${optionValueToSql(attachOptions.dataPath)}`);
|
|
2186
|
+
}
|
|
2187
|
+
if (attachOptions.encrypted !== undefined) {
|
|
2188
|
+
options.push(`ENCRYPTED=${optionValueToSql(attachOptions.encrypted)}`);
|
|
2189
|
+
}
|
|
2190
|
+
if (attachOptions.metaParameterName) {
|
|
2191
|
+
options.push(`META_PARAMETER_NAME=${optionValueToSql(attachOptions.metaParameterName)}`);
|
|
2192
|
+
}
|
|
2193
|
+
if (attachOptions.metadataCatalog) {
|
|
2194
|
+
options.push(`METADATA_CATALOG=${optionValueToSql(attachOptions.metadataCatalog)}`);
|
|
2195
|
+
}
|
|
2196
|
+
if (attachOptions.overrideDataPath !== undefined) {
|
|
2197
|
+
options.push(`OVERRIDE_DATA_PATH=${optionValueToSql(attachOptions.overrideDataPath)}`);
|
|
2198
|
+
}
|
|
2199
|
+
if (attachOptions.readOnly !== undefined) {
|
|
2200
|
+
options.push(`READ_ONLY=${optionValueToSql(attachOptions.readOnly)}`);
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
const attachSql = [
|
|
2204
|
+
`ATTACH ${quoteString(normalized.catalog)} AS ${quoteIdentifier(normalized.alias)}`
|
|
2205
|
+
];
|
|
2206
|
+
if (options.length > 0) {
|
|
2207
|
+
attachSql.push(`(${options.join(", ")})`);
|
|
2208
|
+
}
|
|
2209
|
+
return attachSql.join(" ");
|
|
2210
|
+
}
|
|
2211
|
+
async function configureDuckLake(connection, config) {
|
|
2212
|
+
const normalized = normalizeDuckLakeConfig(config);
|
|
2213
|
+
if (normalized.install) {
|
|
2214
|
+
await connection.run("INSTALL ducklake");
|
|
2215
|
+
}
|
|
2216
|
+
if (normalized.load) {
|
|
2217
|
+
await connection.run("LOAD ducklake");
|
|
2218
|
+
}
|
|
2219
|
+
const attachSql = buildDuckLakeAttachSql(normalized);
|
|
2220
|
+
await connection.run(attachSql);
|
|
2221
|
+
if (normalized.use) {
|
|
2222
|
+
await connection.run(`USE ${quoteIdentifier(normalized.alias)}`);
|
|
2223
|
+
}
|
|
2224
|
+
}
|
|
2225
|
+
function wrapDuckLakePool(pool, config) {
|
|
2226
|
+
const configuredConnections = new WeakSet;
|
|
2227
|
+
const poolWithSize = pool;
|
|
2228
|
+
const size = typeof poolWithSize.size === "number" ? poolWithSize.size : undefined;
|
|
2229
|
+
const wrapped = {
|
|
2230
|
+
async acquire() {
|
|
2231
|
+
const connection = await pool.acquire();
|
|
2232
|
+
if (!configuredConnections.has(connection)) {
|
|
2233
|
+
await configureDuckLake(connection, config);
|
|
2234
|
+
configuredConnections.add(connection);
|
|
2235
|
+
}
|
|
2236
|
+
return connection;
|
|
2237
|
+
},
|
|
2238
|
+
release(connection) {
|
|
2239
|
+
return pool.release(connection);
|
|
2240
|
+
},
|
|
2241
|
+
close: pool.close?.bind(pool)
|
|
2242
|
+
};
|
|
2243
|
+
if (size !== undefined) {
|
|
2244
|
+
wrapped.size = size;
|
|
2245
|
+
}
|
|
2246
|
+
return wrapped;
|
|
2247
|
+
}
|
|
2248
|
+
function isDuckDbFileCatalog(catalog) {
|
|
2249
|
+
const trimmed = catalog.trim();
|
|
2250
|
+
if (!trimmed)
|
|
2251
|
+
return false;
|
|
2252
|
+
if (trimmed === ":memory:")
|
|
2253
|
+
return true;
|
|
2254
|
+
if (isDuckLakeUri(trimmed)) {
|
|
2255
|
+
return isDuckDbFileCatalog(trimmed.slice("ducklake:".length));
|
|
2256
|
+
}
|
|
2257
|
+
if (/^[a-zA-Z]:[\\/]/.test(trimmed))
|
|
2258
|
+
return true;
|
|
2259
|
+
if (trimmed.startsWith("md:"))
|
|
2260
|
+
return false;
|
|
2261
|
+
if (/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(trimmed))
|
|
2262
|
+
return false;
|
|
2263
|
+
if (/^(postgres|postgresql|mysql|mariadb|sqlite|snowflake|bigquery):/i.test(trimmed)) {
|
|
2264
|
+
return false;
|
|
2265
|
+
}
|
|
2266
|
+
return trimmed.endsWith(".duckdb") || trimmed.endsWith(".ddb") || trimmed.includes("/") || trimmed.includes("\\") || trimmed.startsWith("./") || trimmed.startsWith("../") || trimmed.startsWith("~");
|
|
2267
|
+
}
|
|
2268
|
+
function resolveDuckLakePoolSize(poolSetting, ducklakeConfig) {
|
|
2269
|
+
const hasPoolSetting = poolSetting !== undefined;
|
|
2270
|
+
const resolvedPoolSize = resolvePoolSize(poolSetting);
|
|
2271
|
+
const isLocalCatalog = ducklakeConfig !== undefined ? isDuckDbFileCatalog(normalizeDuckLakeConfig(ducklakeConfig).catalog) : false;
|
|
2272
|
+
const poolSize = !hasPoolSetting && isLocalCatalog ? 1 : resolvedPoolSize;
|
|
2273
|
+
return {
|
|
2274
|
+
poolSize,
|
|
2275
|
+
resolvedPoolSize,
|
|
2276
|
+
isLocalCatalog,
|
|
2277
|
+
hasPoolSetting
|
|
2278
|
+
};
|
|
2279
|
+
}
|
|
2280
|
+
|
|
2108
2281
|
// src/driver.ts
|
|
2109
2282
|
class DuckDBDriver {
|
|
2110
2283
|
client;
|
|
@@ -2132,6 +2305,14 @@ function isConfigObject(data) {
|
|
|
2132
2305
|
return "connection" in data || "client" in data || "pool" in data || "schema" in data || "logger" in data;
|
|
2133
2306
|
}
|
|
2134
2307
|
function createFromClient(client, config = {}, instance) {
|
|
2308
|
+
let finalClient = client;
|
|
2309
|
+
if (config.ducklake) {
|
|
2310
|
+
if (isPool(client)) {
|
|
2311
|
+
finalClient = wrapDuckLakePool(client, config.ducklake);
|
|
2312
|
+
} else {
|
|
2313
|
+
throw new Error('DuckLake configuration requires a connection string or pool. Use drizzle("path", { ducklake: ... }) or call configureDuckLake(connection, config) manually.');
|
|
2314
|
+
}
|
|
2315
|
+
}
|
|
2135
2316
|
const dialect = new DuckDBDialect;
|
|
2136
2317
|
const prepareCache = resolvePrepareCacheOption(config.prepareCache);
|
|
2137
2318
|
const logger = config.logger === true ? new DefaultLogger : config.logger || undefined;
|
|
@@ -2144,24 +2325,38 @@ function createFromClient(client, config = {}, instance) {
|
|
|
2144
2325
|
tableNamesMap: tablesConfig.tableNamesMap
|
|
2145
2326
|
};
|
|
2146
2327
|
}
|
|
2147
|
-
const driver = new DuckDBDriver(
|
|
2328
|
+
const driver = new DuckDBDriver(finalClient, dialect, {
|
|
2148
2329
|
logger,
|
|
2149
2330
|
rejectStringArrayLiterals: config.rejectStringArrayLiterals,
|
|
2150
2331
|
prepareCache
|
|
2151
2332
|
});
|
|
2152
2333
|
const session = driver.createSession(schema);
|
|
2153
|
-
const db = new DuckDBDatabase(dialect, session, schema,
|
|
2334
|
+
const db = new DuckDBDatabase(dialect, session, schema, finalClient, instance);
|
|
2154
2335
|
return db;
|
|
2155
2336
|
}
|
|
2156
2337
|
async function createFromConnectionString(path, instanceOptions, config = {}) {
|
|
2157
2338
|
const instance = await DuckDBInstance2.create(path, instanceOptions);
|
|
2158
|
-
const
|
|
2339
|
+
const ducklakeConfig = config.ducklake;
|
|
2340
|
+
const { poolSize, resolvedPoolSize, isLocalCatalog } = resolveDuckLakePoolSize(config.pool, ducklakeConfig);
|
|
2341
|
+
if (ducklakeConfig && resolvedPoolSize !== false && typeof resolvedPoolSize === "number" && resolvedPoolSize > 1 && isLocalCatalog) {
|
|
2342
|
+
console.warn("[ducklake] DuckDB catalog files support a single client connection. Pool sizes greater than 1 can cause write conflicts.");
|
|
2343
|
+
}
|
|
2159
2344
|
if (poolSize === false) {
|
|
2160
2345
|
const connection = await instance.connect();
|
|
2161
|
-
|
|
2346
|
+
if (ducklakeConfig) {
|
|
2347
|
+
await configureDuckLake(connection, ducklakeConfig);
|
|
2348
|
+
}
|
|
2349
|
+
const { ducklake: ducklake2, ...restConfig2 } = config;
|
|
2350
|
+
return createFromClient(connection, restConfig2, instance);
|
|
2162
2351
|
}
|
|
2163
|
-
const pool = createDuckDBConnectionPool(instance, {
|
|
2164
|
-
|
|
2352
|
+
const pool = createDuckDBConnectionPool(instance, {
|
|
2353
|
+
size: poolSize,
|
|
2354
|
+
setup: ducklakeConfig ? async (connection) => {
|
|
2355
|
+
await configureDuckLake(connection, ducklakeConfig);
|
|
2356
|
+
} : undefined
|
|
2357
|
+
});
|
|
2358
|
+
const { ducklake, ...restConfig } = config;
|
|
2359
|
+
return createFromClient(pool, restConfig, instance);
|
|
2165
2360
|
}
|
|
2166
2361
|
function drizzle(clientOrConfigOrPath, config) {
|
|
2167
2362
|
if (typeof clientOrConfigOrPath === "string") {
|
|
@@ -3229,6 +3424,7 @@ export {
|
|
|
3229
3424
|
wrapMap,
|
|
3230
3425
|
wrapList,
|
|
3231
3426
|
wrapJson,
|
|
3427
|
+
wrapDuckLakePool,
|
|
3232
3428
|
wrapBlob,
|
|
3233
3429
|
wrapArray,
|
|
3234
3430
|
toIdentifier,
|
|
@@ -3275,6 +3471,7 @@ export {
|
|
|
3275
3471
|
denseRank,
|
|
3276
3472
|
createDuckDBConnectionPool,
|
|
3277
3473
|
countN,
|
|
3474
|
+
configureDuckLake,
|
|
3278
3475
|
coerceArrayString,
|
|
3279
3476
|
closeClientConnection,
|
|
3280
3477
|
buildStructLiteral,
|
package/dist/pool.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DuckDBInstance } from '@duckdb/node-api';
|
|
1
|
+
import { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
|
|
2
2
|
import { type DuckDBConnectionPool } from './client.ts';
|
|
3
3
|
/** Pool size presets for different MotherDuck instance types */
|
|
4
4
|
export type PoolPreset = 'pulse' | 'standard' | 'jumbo' | 'mega' | 'giga' | 'local' | 'memory';
|
|
@@ -24,6 +24,8 @@ export interface DuckDBConnectionPoolOptions {
|
|
|
24
24
|
maxLifetimeMs?: number;
|
|
25
25
|
/** Max idle time (ms) before an idle connection is discarded. */
|
|
26
26
|
idleTimeoutMs?: number;
|
|
27
|
+
/** Optional setup hook for newly created connections. */
|
|
28
|
+
setup?: (connection: DuckDBConnection) => Promise<void>;
|
|
27
29
|
}
|
|
28
30
|
export declare function createDuckDBConnectionPool(instance: DuckDBInstance, options?: DuckDBConnectionPoolOptions): DuckDBConnectionPool & {
|
|
29
31
|
size: number;
|
package/package.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"module": "./dist/index.mjs",
|
|
4
4
|
"main": "./dist/index.mjs",
|
|
5
5
|
"types": "./dist/index.d.ts",
|
|
6
|
-
"version": "1.3.
|
|
6
|
+
"version": "1.3.3",
|
|
7
7
|
"description": "A drizzle ORM client for use with DuckDB. Based on drizzle's Postgres client.",
|
|
8
8
|
"type": "module",
|
|
9
9
|
"scripts": {
|
|
@@ -80,6 +80,9 @@
|
|
|
80
80
|
"dist/*.mjs",
|
|
81
81
|
"dist/**/*.d.ts"
|
|
82
82
|
],
|
|
83
|
+
"overrides": {
|
|
84
|
+
"esbuild": "0.25.12"
|
|
85
|
+
},
|
|
83
86
|
"dependencies": {
|
|
84
87
|
"node-sql-parser": "^5.3.13"
|
|
85
88
|
}
|
|
@@ -4,6 +4,7 @@ import { mkdir, writeFile } from 'node:fs/promises';
|
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import process from 'node:process';
|
|
6
6
|
import { drizzle } from '../index.ts';
|
|
7
|
+
import { configureDuckLake, type DuckLakeConfig } from '../ducklake.ts';
|
|
7
8
|
import { introspect } from '../introspect.ts';
|
|
8
9
|
|
|
9
10
|
interface CliOptions {
|
|
@@ -16,6 +17,7 @@ interface CliOptions {
|
|
|
16
17
|
includeViews: boolean;
|
|
17
18
|
useCustomTimeTypes: boolean;
|
|
18
19
|
importBasePath?: string;
|
|
20
|
+
ducklake?: DuckLakeConfig;
|
|
19
21
|
}
|
|
20
22
|
|
|
21
23
|
function parseArgs(argv: string[]): CliOptions {
|
|
@@ -27,6 +29,23 @@ function parseArgs(argv: string[]): CliOptions {
|
|
|
27
29
|
useCustomTimeTypes: true,
|
|
28
30
|
};
|
|
29
31
|
|
|
32
|
+
const ensureDuckLakeConfig = (): DuckLakeConfig => {
|
|
33
|
+
if (!options.ducklake) {
|
|
34
|
+
options.ducklake = { catalog: '' };
|
|
35
|
+
}
|
|
36
|
+
return options.ducklake;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const ensureDuckLakeAttachOptions = (): NonNullable<
|
|
40
|
+
DuckLakeConfig['attachOptions']
|
|
41
|
+
> => {
|
|
42
|
+
const config = ensureDuckLakeConfig();
|
|
43
|
+
if (!config.attachOptions) {
|
|
44
|
+
config.attachOptions = {};
|
|
45
|
+
}
|
|
46
|
+
return config.attachOptions;
|
|
47
|
+
};
|
|
48
|
+
|
|
30
49
|
for (let i = 0; i < argv.length; i += 1) {
|
|
31
50
|
const arg = argv[i]!;
|
|
32
51
|
switch (arg) {
|
|
@@ -72,6 +91,50 @@ function parseArgs(argv: string[]): CliOptions {
|
|
|
72
91
|
case '--import-base':
|
|
73
92
|
options.importBasePath = argv[++i];
|
|
74
93
|
break;
|
|
94
|
+
case '--ducklake-catalog':
|
|
95
|
+
ensureDuckLakeConfig().catalog = argv[++i] ?? '';
|
|
96
|
+
break;
|
|
97
|
+
case '--ducklake-alias':
|
|
98
|
+
ensureDuckLakeConfig().alias = argv[++i];
|
|
99
|
+
break;
|
|
100
|
+
case '--ducklake-no-use':
|
|
101
|
+
ensureDuckLakeConfig().use = false;
|
|
102
|
+
break;
|
|
103
|
+
case '--ducklake-install':
|
|
104
|
+
ensureDuckLakeConfig().install = true;
|
|
105
|
+
break;
|
|
106
|
+
case '--ducklake-load':
|
|
107
|
+
ensureDuckLakeConfig().load = true;
|
|
108
|
+
break;
|
|
109
|
+
case '--ducklake-data-path':
|
|
110
|
+
ensureDuckLakeAttachOptions().dataPath = argv[++i];
|
|
111
|
+
break;
|
|
112
|
+
case '--ducklake-read-only':
|
|
113
|
+
ensureDuckLakeAttachOptions().readOnly = true;
|
|
114
|
+
break;
|
|
115
|
+
case '--ducklake-create-if-not-exists':
|
|
116
|
+
ensureDuckLakeAttachOptions().createIfNotExists = true;
|
|
117
|
+
break;
|
|
118
|
+
case '--ducklake-override-data-path':
|
|
119
|
+
ensureDuckLakeAttachOptions().overrideDataPath = true;
|
|
120
|
+
break;
|
|
121
|
+
case '--ducklake-data-inlining-row-limit': {
|
|
122
|
+
const value = argv[++i];
|
|
123
|
+
const parsed = value ? Number(value) : NaN;
|
|
124
|
+
if (Number.isFinite(parsed)) {
|
|
125
|
+
ensureDuckLakeAttachOptions().dataInliningRowLimit = parsed;
|
|
126
|
+
}
|
|
127
|
+
break;
|
|
128
|
+
}
|
|
129
|
+
case '--ducklake-encrypted':
|
|
130
|
+
ensureDuckLakeAttachOptions().encrypted = true;
|
|
131
|
+
break;
|
|
132
|
+
case '--ducklake-metadata-catalog':
|
|
133
|
+
ensureDuckLakeAttachOptions().metadataCatalog = argv[++i];
|
|
134
|
+
break;
|
|
135
|
+
case '--ducklake-meta-parameter-name':
|
|
136
|
+
ensureDuckLakeAttachOptions().metaParameterName = argv[++i];
|
|
137
|
+
break;
|
|
75
138
|
case '--help':
|
|
76
139
|
case '-h':
|
|
77
140
|
printHelp();
|
|
@@ -102,6 +165,19 @@ Options:
|
|
|
102
165
|
--include-views Include views in the generated schema
|
|
103
166
|
--use-pg-time Use pg-core timestamp/date/time instead of DuckDB custom helpers
|
|
104
167
|
--import-base Override import path for duckdb helpers (default: package name)
|
|
168
|
+
--ducklake-catalog DuckLake catalog value after the ducklake: prefix
|
|
169
|
+
--ducklake-alias Alias for attached DuckLake database
|
|
170
|
+
--ducklake-no-use Do not run USE after attach
|
|
171
|
+
--ducklake-install Run INSTALL ducklake before attach
|
|
172
|
+
--ducklake-load Run LOAD ducklake before attach
|
|
173
|
+
--ducklake-data-path Data path for DuckLake table storage
|
|
174
|
+
--ducklake-read-only Attach DuckLake in read-only mode
|
|
175
|
+
--ducklake-create-if-not-exists Create catalog if it does not exist
|
|
176
|
+
--ducklake-override-data-path Override data path for existing catalog
|
|
177
|
+
--ducklake-data-inlining-row-limit Inline row limit for data storage
|
|
178
|
+
--ducklake-encrypted Enable encryption for the metadata catalog
|
|
179
|
+
--ducklake-metadata-catalog Override metadata catalog name
|
|
180
|
+
--ducklake-meta-parameter-name Set meta parameter name for metadata storage
|
|
105
181
|
|
|
106
182
|
Database Filtering:
|
|
107
183
|
By default, only tables from the current database are introspected. This prevents
|
|
@@ -116,6 +192,14 @@ Examples:
|
|
|
116
192
|
|
|
117
193
|
# MotherDuck (requires MOTHERDUCK_TOKEN env var)
|
|
118
194
|
MOTHERDUCK_TOKEN=xxx bun x duckdb-introspect --url md: --database my_cloud_db --out ./schema.ts
|
|
195
|
+
|
|
196
|
+
# DuckLake local catalog with data path
|
|
197
|
+
bun x duckdb-introspect --url :memory: --ducklake-catalog ./ducklake.duckdb \\
|
|
198
|
+
--ducklake-data-path ./ducklake-data --out ./schema.ts
|
|
199
|
+
|
|
200
|
+
# DuckLake on MotherDuck
|
|
201
|
+
MOTHERDUCK_TOKEN=xxx bun x duckdb-introspect --url md: \\
|
|
202
|
+
--ducklake-catalog md:__ducklake_metadata_my_db --out ./schema.ts
|
|
119
203
|
`);
|
|
120
204
|
}
|
|
121
205
|
|
|
@@ -136,6 +220,13 @@ async function main() {
|
|
|
136
220
|
const db = drizzle(connection);
|
|
137
221
|
|
|
138
222
|
try {
|
|
223
|
+
if (options.ducklake && !options.ducklake.catalog) {
|
|
224
|
+
throw new Error('DuckLake requires --ducklake-catalog');
|
|
225
|
+
}
|
|
226
|
+
if (options.ducklake?.catalog) {
|
|
227
|
+
await configureDuckLake(connection, options.ducklake);
|
|
228
|
+
}
|
|
229
|
+
|
|
139
230
|
const result = await introspect(db, {
|
|
140
231
|
database: options.database,
|
|
141
232
|
allDatabases: options.allDatabases,
|
package/src/driver.ts
CHANGED
|
@@ -31,7 +31,6 @@ import type {
|
|
|
31
31
|
import { closeClientConnection, isPool } from './client.ts';
|
|
32
32
|
import {
|
|
33
33
|
createDuckDBConnectionPool,
|
|
34
|
-
resolvePoolSize,
|
|
35
34
|
type DuckDBPoolConfig,
|
|
36
35
|
type PoolPreset,
|
|
37
36
|
} from './pool.ts';
|
|
@@ -40,6 +39,12 @@ import {
|
|
|
40
39
|
type PreparedStatementCacheConfig,
|
|
41
40
|
type PrepareCacheOption,
|
|
42
41
|
} from './options.ts';
|
|
42
|
+
import {
|
|
43
|
+
configureDuckLake,
|
|
44
|
+
resolveDuckLakePoolSize,
|
|
45
|
+
wrapDuckLakePool,
|
|
46
|
+
type DuckLakeConfig,
|
|
47
|
+
} from './ducklake.ts';
|
|
43
48
|
|
|
44
49
|
export interface PgDriverOptions {
|
|
45
50
|
logger?: Logger;
|
|
@@ -82,6 +87,8 @@ export interface DuckDBDrizzleConfig<
|
|
|
82
87
|
prepareCache?: PrepareCacheOption;
|
|
83
88
|
/** Pool configuration. Use preset name, size config, or false to disable. */
|
|
84
89
|
pool?: DuckDBPoolConfig | PoolPreset | false;
|
|
90
|
+
/** Optional DuckLake configuration */
|
|
91
|
+
ducklake?: DuckLakeConfig;
|
|
85
92
|
}
|
|
86
93
|
|
|
87
94
|
export interface DuckDBDrizzleConfigWithConnection<
|
|
@@ -119,6 +126,18 @@ function createFromClient<
|
|
|
119
126
|
config: DuckDBDrizzleConfig<TSchema> = {},
|
|
120
127
|
instance?: DuckDBInstance
|
|
121
128
|
): DuckDBDatabase<TSchema, ExtractTablesWithRelations<TSchema>> {
|
|
129
|
+
let finalClient = client;
|
|
130
|
+
|
|
131
|
+
if (config.ducklake) {
|
|
132
|
+
if (isPool(client)) {
|
|
133
|
+
finalClient = wrapDuckLakePool(client, config.ducklake);
|
|
134
|
+
} else {
|
|
135
|
+
throw new Error(
|
|
136
|
+
'DuckLake configuration requires a connection string or pool. Use drizzle("path", { ducklake: ... }) or call configureDuckLake(connection, config) manually.'
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
122
141
|
const dialect = new DuckDBDialect();
|
|
123
142
|
const prepareCache = resolvePrepareCacheOption(config.prepareCache);
|
|
124
143
|
|
|
@@ -139,14 +158,20 @@ function createFromClient<
|
|
|
139
158
|
};
|
|
140
159
|
}
|
|
141
160
|
|
|
142
|
-
const driver = new DuckDBDriver(
|
|
161
|
+
const driver = new DuckDBDriver(finalClient, dialect, {
|
|
143
162
|
logger,
|
|
144
163
|
rejectStringArrayLiterals: config.rejectStringArrayLiterals,
|
|
145
164
|
prepareCache,
|
|
146
165
|
});
|
|
147
166
|
const session = driver.createSession(schema);
|
|
148
167
|
|
|
149
|
-
const db = new DuckDBDatabase(
|
|
168
|
+
const db = new DuckDBDatabase(
|
|
169
|
+
dialect,
|
|
170
|
+
session,
|
|
171
|
+
schema,
|
|
172
|
+
finalClient,
|
|
173
|
+
instance
|
|
174
|
+
);
|
|
150
175
|
return db as DuckDBDatabase<TSchema, ExtractTablesWithRelations<TSchema>>;
|
|
151
176
|
}
|
|
152
177
|
|
|
@@ -159,15 +184,49 @@ async function createFromConnectionString<
|
|
|
159
184
|
config: DuckDBDrizzleConfig<TSchema> = {}
|
|
160
185
|
): Promise<DuckDBDatabase<TSchema, ExtractTablesWithRelations<TSchema>>> {
|
|
161
186
|
const instance = await DuckDBInstance.create(path, instanceOptions);
|
|
162
|
-
const
|
|
187
|
+
const ducklakeConfig = config.ducklake;
|
|
188
|
+
const { poolSize, resolvedPoolSize, isLocalCatalog } =
|
|
189
|
+
resolveDuckLakePoolSize(config.pool, ducklakeConfig);
|
|
190
|
+
|
|
191
|
+
if (
|
|
192
|
+
ducklakeConfig &&
|
|
193
|
+
resolvedPoolSize !== false &&
|
|
194
|
+
typeof resolvedPoolSize === 'number' &&
|
|
195
|
+
resolvedPoolSize > 1 &&
|
|
196
|
+
isLocalCatalog
|
|
197
|
+
) {
|
|
198
|
+
console.warn(
|
|
199
|
+
'[ducklake] DuckDB catalog files support a single client connection. Pool sizes greater than 1 can cause write conflicts.'
|
|
200
|
+
);
|
|
201
|
+
}
|
|
163
202
|
|
|
164
203
|
if (poolSize === false) {
|
|
165
204
|
const connection = await instance.connect();
|
|
166
|
-
|
|
205
|
+
if (ducklakeConfig) {
|
|
206
|
+
await configureDuckLake(connection, ducklakeConfig);
|
|
207
|
+
}
|
|
208
|
+
const { ducklake, ...restConfig } = config;
|
|
209
|
+
return createFromClient(
|
|
210
|
+
connection,
|
|
211
|
+
restConfig as DuckDBDrizzleConfig<TSchema>,
|
|
212
|
+
instance
|
|
213
|
+
);
|
|
167
214
|
}
|
|
168
215
|
|
|
169
|
-
const pool = createDuckDBConnectionPool(instance, {
|
|
170
|
-
|
|
216
|
+
const pool = createDuckDBConnectionPool(instance, {
|
|
217
|
+
size: poolSize,
|
|
218
|
+
setup: ducklakeConfig
|
|
219
|
+
? async (connection) => {
|
|
220
|
+
await configureDuckLake(connection, ducklakeConfig);
|
|
221
|
+
}
|
|
222
|
+
: undefined,
|
|
223
|
+
});
|
|
224
|
+
const { ducklake, ...restConfig } = config;
|
|
225
|
+
return createFromClient(
|
|
226
|
+
pool,
|
|
227
|
+
restConfig as DuckDBDrizzleConfig<TSchema>,
|
|
228
|
+
instance
|
|
229
|
+
);
|
|
171
230
|
}
|
|
172
231
|
|
|
173
232
|
// Overload 1: Connection string (async, auto-pools)
|