@leonardovida-md/drizzle-neo-duckdb 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -18
- package/dist/client.d.ts +12 -0
- package/dist/columns.d.ts +18 -10
- package/dist/driver.d.ts +4 -0
- package/dist/duckdb-introspect.mjs +245 -51
- package/dist/index.d.ts +3 -0
- package/dist/index.mjs +435 -65
- package/dist/introspect.d.ts +12 -0
- package/dist/olap.d.ts +46 -0
- package/dist/session.d.ts +5 -0
- package/dist/value-wrappers.d.ts +104 -0
- package/package.json +7 -3
- package/src/bin/duckdb-introspect.ts +40 -3
- package/src/client.ts +135 -18
- package/src/columns.ts +65 -36
- package/src/dialect.ts +2 -2
- package/src/driver.ts +20 -6
- package/src/index.ts +3 -0
- package/src/introspect.ts +97 -33
- package/src/migrator.ts +1 -3
- package/src/olap.ts +189 -0
- package/src/select-builder.ts +3 -7
- package/src/session.ts +87 -18
- package/src/sql/query-rewriters.ts +5 -8
- package/src/sql/result-mapper.ts +6 -6
- package/src/sql/selection.ts +2 -9
- package/src/value-wrappers.ts +324 -0
package/dist/index.mjs
CHANGED
|
@@ -303,8 +303,137 @@ import { TransactionRollbackError } from "drizzle-orm/errors";
|
|
|
303
303
|
|
|
304
304
|
// src/client.ts
|
|
305
305
|
import {
|
|
306
|
-
listValue
|
|
306
|
+
listValue as listValue2,
|
|
307
|
+
timestampValue as timestampValue2
|
|
307
308
|
} from "@duckdb/node-api";
|
|
309
|
+
|
|
310
|
+
// src/value-wrappers.ts
|
|
311
|
+
import {
|
|
312
|
+
listValue,
|
|
313
|
+
arrayValue,
|
|
314
|
+
structValue,
|
|
315
|
+
mapValue,
|
|
316
|
+
blobValue,
|
|
317
|
+
timestampValue,
|
|
318
|
+
timestampTZValue
|
|
319
|
+
} from "@duckdb/node-api";
|
|
320
|
+
var DUCKDB_VALUE_MARKER = Symbol.for("drizzle-duckdb:value");
|
|
321
|
+
function isDuckDBWrapper(value) {
|
|
322
|
+
return value !== null && typeof value === "object" && DUCKDB_VALUE_MARKER in value && value[DUCKDB_VALUE_MARKER] === true;
|
|
323
|
+
}
|
|
324
|
+
function wrapList(data, elementType) {
|
|
325
|
+
return {
|
|
326
|
+
[DUCKDB_VALUE_MARKER]: true,
|
|
327
|
+
kind: "list",
|
|
328
|
+
data,
|
|
329
|
+
elementType
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
function wrapArray(data, elementType, fixedLength) {
|
|
333
|
+
return {
|
|
334
|
+
[DUCKDB_VALUE_MARKER]: true,
|
|
335
|
+
kind: "array",
|
|
336
|
+
data,
|
|
337
|
+
elementType,
|
|
338
|
+
fixedLength
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
function wrapStruct(data, schema) {
|
|
342
|
+
return {
|
|
343
|
+
[DUCKDB_VALUE_MARKER]: true,
|
|
344
|
+
kind: "struct",
|
|
345
|
+
data,
|
|
346
|
+
schema
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
function wrapMap(data, valueType) {
|
|
350
|
+
return {
|
|
351
|
+
[DUCKDB_VALUE_MARKER]: true,
|
|
352
|
+
kind: "map",
|
|
353
|
+
data,
|
|
354
|
+
valueType
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
function wrapTimestamp(data, withTimezone, precision) {
|
|
358
|
+
return {
|
|
359
|
+
[DUCKDB_VALUE_MARKER]: true,
|
|
360
|
+
kind: "timestamp",
|
|
361
|
+
data,
|
|
362
|
+
withTimezone,
|
|
363
|
+
precision
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
function wrapBlob(data) {
|
|
367
|
+
return {
|
|
368
|
+
[DUCKDB_VALUE_MARKER]: true,
|
|
369
|
+
kind: "blob",
|
|
370
|
+
data
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
function wrapJson(data) {
|
|
374
|
+
return {
|
|
375
|
+
[DUCKDB_VALUE_MARKER]: true,
|
|
376
|
+
kind: "json",
|
|
377
|
+
data
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
function dateToMicros(value) {
|
|
381
|
+
if (value instanceof Date) {
|
|
382
|
+
return BigInt(value.getTime()) * 1000n;
|
|
383
|
+
}
|
|
384
|
+
let normalized = value;
|
|
385
|
+
if (!value.includes("T") && value.includes(" ")) {
|
|
386
|
+
normalized = value.replace(" ", "T");
|
|
387
|
+
}
|
|
388
|
+
if (!normalized.endsWith("Z") && !/[+-]\d{2}:?\d{2}$/.test(normalized)) {
|
|
389
|
+
normalized += "Z";
|
|
390
|
+
}
|
|
391
|
+
const date = new Date(normalized);
|
|
392
|
+
if (isNaN(date.getTime())) {
|
|
393
|
+
throw new Error(`Invalid timestamp string: ${value}`);
|
|
394
|
+
}
|
|
395
|
+
return BigInt(date.getTime()) * 1000n;
|
|
396
|
+
}
|
|
397
|
+
function toUint8Array(data) {
|
|
398
|
+
return data instanceof Uint8Array && !(data instanceof Buffer) ? data : new Uint8Array(data);
|
|
399
|
+
}
|
|
400
|
+
function convertStructEntries(data, toValue) {
|
|
401
|
+
const entries = {};
|
|
402
|
+
for (const [key, val] of Object.entries(data)) {
|
|
403
|
+
entries[key] = toValue(val);
|
|
404
|
+
}
|
|
405
|
+
return entries;
|
|
406
|
+
}
|
|
407
|
+
function convertMapEntries(data, toValue) {
|
|
408
|
+
return Object.entries(data).map(([key, val]) => ({
|
|
409
|
+
key,
|
|
410
|
+
value: toValue(val)
|
|
411
|
+
}));
|
|
412
|
+
}
|
|
413
|
+
function wrapperToNodeApiValue(wrapper, toValue) {
|
|
414
|
+
switch (wrapper.kind) {
|
|
415
|
+
case "list":
|
|
416
|
+
return listValue(wrapper.data.map(toValue));
|
|
417
|
+
case "array":
|
|
418
|
+
return arrayValue(wrapper.data.map(toValue));
|
|
419
|
+
case "struct":
|
|
420
|
+
return structValue(convertStructEntries(wrapper.data, toValue));
|
|
421
|
+
case "map":
|
|
422
|
+
return mapValue(convertMapEntries(wrapper.data, toValue));
|
|
423
|
+
case "timestamp":
|
|
424
|
+
return wrapper.withTimezone ? timestampTZValue(dateToMicros(wrapper.data)) : timestampValue(dateToMicros(wrapper.data));
|
|
425
|
+
case "blob":
|
|
426
|
+
return blobValue(toUint8Array(wrapper.data));
|
|
427
|
+
case "json":
|
|
428
|
+
return JSON.stringify(wrapper.data);
|
|
429
|
+
default: {
|
|
430
|
+
const _exhaustive = wrapper;
|
|
431
|
+
throw new Error(`Unknown wrapper kind: ${_exhaustive.kind}`);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
// src/client.ts
|
|
308
437
|
function isPgArrayLiteral(value) {
|
|
309
438
|
return value.startsWith("{") && value.endsWith("}");
|
|
310
439
|
}
|
|
@@ -333,30 +462,91 @@ function prepareParams(params, options = {}) {
|
|
|
333
462
|
});
|
|
334
463
|
}
|
|
335
464
|
function toNodeApiValue(value) {
|
|
465
|
+
if (value == null)
|
|
466
|
+
return null;
|
|
467
|
+
const t = typeof value;
|
|
468
|
+
if (t === "string" || t === "number" || t === "bigint" || t === "boolean") {
|
|
469
|
+
return value;
|
|
470
|
+
}
|
|
471
|
+
if (t === "object" && DUCKDB_VALUE_MARKER in value) {
|
|
472
|
+
return wrapperToNodeApiValue(value, toNodeApiValue);
|
|
473
|
+
}
|
|
336
474
|
if (Array.isArray(value)) {
|
|
337
|
-
return
|
|
475
|
+
return listValue2(value.map((inner) => toNodeApiValue(inner)));
|
|
476
|
+
}
|
|
477
|
+
if (value instanceof Date) {
|
|
478
|
+
return timestampValue2(BigInt(value.getTime()) * 1000n);
|
|
338
479
|
}
|
|
339
480
|
return value;
|
|
340
481
|
}
|
|
341
|
-
|
|
342
|
-
const values = params.length > 0 ? params.map((param) => toNodeApiValue(param)) : undefined;
|
|
343
|
-
const result = await client.run(query, values);
|
|
344
|
-
const rows = await result.getRowsJS();
|
|
345
|
-
const columns = result.columnNames();
|
|
482
|
+
function deduplicateColumns(columns) {
|
|
346
483
|
const seen = {};
|
|
347
|
-
|
|
484
|
+
return columns.map((col) => {
|
|
348
485
|
const count = seen[col] ?? 0;
|
|
349
486
|
seen[col] = count + 1;
|
|
350
487
|
return count === 0 ? col : `${col}_${count}`;
|
|
351
488
|
});
|
|
352
|
-
|
|
489
|
+
}
|
|
490
|
+
function mapRowsToObjects(columns, rows) {
|
|
491
|
+
return rows.map((vals) => {
|
|
353
492
|
const obj = {};
|
|
354
|
-
|
|
493
|
+
columns.forEach((col, idx) => {
|
|
355
494
|
obj[col] = vals[idx];
|
|
356
495
|
});
|
|
357
496
|
return obj;
|
|
358
497
|
});
|
|
359
498
|
}
|
|
499
|
+
async function closeClientConnection(connection) {
|
|
500
|
+
if ("close" in connection && typeof connection.close === "function") {
|
|
501
|
+
await connection.close();
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
if ("closeSync" in connection && typeof connection.closeSync === "function") {
|
|
505
|
+
connection.closeSync();
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
if ("disconnectSync" in connection && typeof connection.disconnectSync === "function") {
|
|
509
|
+
connection.disconnectSync();
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
async function executeOnClient(client, query, params) {
|
|
513
|
+
const values = params.length > 0 ? params.map((param) => toNodeApiValue(param)) : undefined;
|
|
514
|
+
const result = await client.run(query, values);
|
|
515
|
+
const rows = await result.getRowsJS();
|
|
516
|
+
const columns = result.deduplicatedColumnNames?.() ?? result.columnNames();
|
|
517
|
+
const uniqueColumns = deduplicateColumns(columns);
|
|
518
|
+
return rows ? mapRowsToObjects(uniqueColumns, rows) : [];
|
|
519
|
+
}
|
|
520
|
+
async function* executeInBatches(client, query, params, options = {}) {
|
|
521
|
+
const rowsPerChunk = options.rowsPerChunk && options.rowsPerChunk > 0 ? options.rowsPerChunk : 1e5;
|
|
522
|
+
const values = params.length > 0 ? params.map((param) => toNodeApiValue(param)) : undefined;
|
|
523
|
+
const result = await client.stream(query, values);
|
|
524
|
+
const columns = result.deduplicatedColumnNames?.() ?? result.columnNames();
|
|
525
|
+
const uniqueColumns = deduplicateColumns(columns);
|
|
526
|
+
let buffer = [];
|
|
527
|
+
for await (const chunk of result.yieldRowsJs()) {
|
|
528
|
+
const objects = mapRowsToObjects(uniqueColumns, chunk);
|
|
529
|
+
for (const row of objects) {
|
|
530
|
+
buffer.push(row);
|
|
531
|
+
if (buffer.length >= rowsPerChunk) {
|
|
532
|
+
yield buffer;
|
|
533
|
+
buffer = [];
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
if (buffer.length > 0) {
|
|
538
|
+
yield buffer;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
async function executeArrowOnClient(client, query, params) {
|
|
542
|
+
const values = params.length > 0 ? params.map((param) => toNodeApiValue(param)) : undefined;
|
|
543
|
+
const result = await client.run(query, values);
|
|
544
|
+
const maybeArrow = result.toArrow ?? result.getArrowTable;
|
|
545
|
+
if (typeof maybeArrow === "function") {
|
|
546
|
+
return await maybeArrow.call(result);
|
|
547
|
+
}
|
|
548
|
+
return result.getColumnsObjectJS();
|
|
549
|
+
}
|
|
360
550
|
|
|
361
551
|
// src/session.ts
|
|
362
552
|
class DuckDBPreparedQuery extends PgPreparedQuery {
|
|
@@ -397,11 +587,7 @@ class DuckDBPreparedQuery extends PgPreparedQuery {
|
|
|
397
587
|
this.logger.logQuery(`[duckdb] original query before array rewrite: ${this.queryString}`, params);
|
|
398
588
|
}
|
|
399
589
|
this.logger.logQuery(rewrittenQuery, params);
|
|
400
|
-
const {
|
|
401
|
-
fields,
|
|
402
|
-
joinsNotNullableMap,
|
|
403
|
-
customResultMapper
|
|
404
|
-
} = this;
|
|
590
|
+
const { fields, joinsNotNullableMap, customResultMapper } = this;
|
|
405
591
|
const rows = await executeOnClient(this.client, rewrittenQuery, params);
|
|
406
592
|
if (rows.length === 0 || !fields) {
|
|
407
593
|
return rows;
|
|
@@ -461,6 +647,32 @@ class DuckDBSession extends PgSession {
|
|
|
461
647
|
this.logger.logQuery(`[duckdb] ${arrayLiteralWarning}
|
|
462
648
|
query: ${query}`, []);
|
|
463
649
|
};
|
|
650
|
+
executeBatches(query, options = {}) {
|
|
651
|
+
const builtQuery = this.dialect.sqlToQuery(query);
|
|
652
|
+
const params = prepareParams(builtQuery.params, {
|
|
653
|
+
rejectStringArrayLiterals: this.rejectStringArrayLiterals,
|
|
654
|
+
warnOnStringArrayLiteral: this.rejectStringArrayLiterals ? undefined : () => this.warnOnStringArrayLiteral(builtQuery.sql)
|
|
655
|
+
});
|
|
656
|
+
const rewrittenQuery = this.rewriteArrays ? adaptArrayOperators(builtQuery.sql) : builtQuery.sql;
|
|
657
|
+
if (this.rewriteArrays && rewrittenQuery !== builtQuery.sql) {
|
|
658
|
+
this.logger.logQuery(`[duckdb] original query before array rewrite: ${builtQuery.sql}`, params);
|
|
659
|
+
}
|
|
660
|
+
this.logger.logQuery(rewrittenQuery, params);
|
|
661
|
+
return executeInBatches(this.client, rewrittenQuery, params, options);
|
|
662
|
+
}
|
|
663
|
+
async executeArrow(query) {
|
|
664
|
+
const builtQuery = this.dialect.sqlToQuery(query);
|
|
665
|
+
const params = prepareParams(builtQuery.params, {
|
|
666
|
+
rejectStringArrayLiterals: this.rejectStringArrayLiterals,
|
|
667
|
+
warnOnStringArrayLiteral: this.rejectStringArrayLiterals ? undefined : () => this.warnOnStringArrayLiteral(builtQuery.sql)
|
|
668
|
+
});
|
|
669
|
+
const rewrittenQuery = this.rewriteArrays ? adaptArrayOperators(builtQuery.sql) : builtQuery.sql;
|
|
670
|
+
if (this.rewriteArrays && rewrittenQuery !== builtQuery.sql) {
|
|
671
|
+
this.logger.logQuery(`[duckdb] original query before array rewrite: ${builtQuery.sql}`, params);
|
|
672
|
+
}
|
|
673
|
+
this.logger.logQuery(rewrittenQuery, params);
|
|
674
|
+
return executeArrowOnClient(this.client, rewrittenQuery, params);
|
|
675
|
+
}
|
|
464
676
|
}
|
|
465
677
|
|
|
466
678
|
class DuckDBTransaction extends PgTransaction {
|
|
@@ -484,6 +696,12 @@ class DuckDBTransaction extends PgTransaction {
|
|
|
484
696
|
setTransaction(config) {
|
|
485
697
|
return this.session.execute(sql`set transaction ${this.getTransactionConfigSQL(config)}`);
|
|
486
698
|
}
|
|
699
|
+
executeBatches(query, options = {}) {
|
|
700
|
+
return this.session.executeBatches(query, options);
|
|
701
|
+
}
|
|
702
|
+
executeArrow(query) {
|
|
703
|
+
return this.session.executeArrow(query);
|
|
704
|
+
}
|
|
487
705
|
async transaction(transaction) {
|
|
488
706
|
const nestedTx = new DuckDBTransaction(this.dialect, this.session, this.schema, this.nestedIndex + 1);
|
|
489
707
|
return transaction(nestedTx);
|
|
@@ -582,13 +800,7 @@ import { PgViewBase } from "drizzle-orm/pg-core/view-base";
|
|
|
582
800
|
import { SQL as SQL4 } from "drizzle-orm/sql/sql";
|
|
583
801
|
|
|
584
802
|
// src/sql/selection.ts
|
|
585
|
-
import {
|
|
586
|
-
Column as Column2,
|
|
587
|
-
SQL as SQL3,
|
|
588
|
-
getTableName as getTableName2,
|
|
589
|
-
is as is3,
|
|
590
|
-
sql as sql3
|
|
591
|
-
} from "drizzle-orm";
|
|
803
|
+
import { Column as Column2, SQL as SQL3, getTableName as getTableName2, is as is3, sql as sql3 } from "drizzle-orm";
|
|
592
804
|
function mapEntries(obj, prefix, fullJoin = false) {
|
|
593
805
|
return Object.fromEntries(Object.entries(obj).filter(([key]) => key !== "enableRLS").map(([key, value]) => {
|
|
594
806
|
const qualified = prefix ? `${prefix}.${key}` : key;
|
|
@@ -729,6 +941,12 @@ class DuckDBDatabase extends PgDatabase {
|
|
|
729
941
|
dialect: this.dialect
|
|
730
942
|
});
|
|
731
943
|
}
|
|
944
|
+
executeBatches(query, options = {}) {
|
|
945
|
+
return this.session.executeBatches(query, options);
|
|
946
|
+
}
|
|
947
|
+
executeArrow(query) {
|
|
948
|
+
return this.session.executeArrow(query);
|
|
949
|
+
}
|
|
732
950
|
async transaction(transaction) {
|
|
733
951
|
return await this.session.transaction(transaction);
|
|
734
952
|
}
|
|
@@ -797,19 +1015,12 @@ function buildStructLiteral(value, schema) {
|
|
|
797
1015
|
});
|
|
798
1016
|
return sql4`struct_pack(${sql4.join(parts, sql4.raw(", "))})`;
|
|
799
1017
|
}
|
|
800
|
-
function buildMapLiteral(value, valueType) {
|
|
801
|
-
const keys = Object.keys(value);
|
|
802
|
-
const vals = Object.values(value);
|
|
803
|
-
const keyList = buildListLiteral(keys, "TEXT");
|
|
804
|
-
const valList = buildListLiteral(vals, valueType?.endsWith("[]") ? valueType.slice(0, -2) : valueType);
|
|
805
|
-
return sql4`map(${keyList}, ${valList})`;
|
|
806
|
-
}
|
|
807
1018
|
var duckDbList = (name, elementType) => customType({
|
|
808
1019
|
dataType() {
|
|
809
1020
|
return `${elementType}[]`;
|
|
810
1021
|
},
|
|
811
1022
|
toDriver(value) {
|
|
812
|
-
return
|
|
1023
|
+
return wrapList(value, elementType);
|
|
813
1024
|
},
|
|
814
1025
|
fromDriver(value) {
|
|
815
1026
|
if (Array.isArray(value)) {
|
|
@@ -829,7 +1040,7 @@ var duckDbArray = (name, elementType, fixedLength) => customType({
|
|
|
829
1040
|
return fixedLength ? `${elementType}[${fixedLength}]` : `${elementType}[]`;
|
|
830
1041
|
},
|
|
831
1042
|
toDriver(value) {
|
|
832
|
-
return
|
|
1043
|
+
return wrapArray(value, elementType, fixedLength);
|
|
833
1044
|
},
|
|
834
1045
|
fromDriver(value) {
|
|
835
1046
|
if (Array.isArray(value)) {
|
|
@@ -849,7 +1060,7 @@ var duckDbMap = (name, valueType) => customType({
|
|
|
849
1060
|
return `MAP (STRING, ${valueType})`;
|
|
850
1061
|
},
|
|
851
1062
|
toDriver(value) {
|
|
852
|
-
return
|
|
1063
|
+
return wrapMap(value, valueType);
|
|
853
1064
|
},
|
|
854
1065
|
fromDriver(value) {
|
|
855
1066
|
return value;
|
|
@@ -885,7 +1096,7 @@ var duckDbJson = (name) => customType({
|
|
|
885
1096
|
if (value !== null && typeof value === "object" && "queryChunks" in value) {
|
|
886
1097
|
return value;
|
|
887
1098
|
}
|
|
888
|
-
return
|
|
1099
|
+
return wrapJson(value);
|
|
889
1100
|
},
|
|
890
1101
|
fromDriver(value) {
|
|
891
1102
|
if (typeof value !== "string") {
|
|
@@ -907,8 +1118,7 @@ var duckDbBlob = customType({
|
|
|
907
1118
|
return "BLOB";
|
|
908
1119
|
},
|
|
909
1120
|
toDriver(value) {
|
|
910
|
-
|
|
911
|
-
return sql4`from_hex(${hexString})`;
|
|
1121
|
+
return wrapBlob(value);
|
|
912
1122
|
}
|
|
913
1123
|
});
|
|
914
1124
|
var duckDbInet = (name) => customType({
|
|
@@ -1012,12 +1222,13 @@ import { sql as sql5 } from "drizzle-orm";
|
|
|
1012
1222
|
var SYSTEM_SCHEMAS = new Set(["information_schema", "pg_catalog"]);
|
|
1013
1223
|
var DEFAULT_IMPORT_BASE = "@leonardovida-md/drizzle-neo-duckdb";
|
|
1014
1224
|
async function introspect(db, opts = {}) {
|
|
1015
|
-
const
|
|
1225
|
+
const database = await resolveDatabase(db, opts.database, opts.allDatabases);
|
|
1226
|
+
const schemas = await resolveSchemas(db, database, opts.schemas);
|
|
1016
1227
|
const includeViews = opts.includeViews ?? false;
|
|
1017
|
-
const tables = await loadTables(db, schemas, includeViews);
|
|
1018
|
-
const columns = await loadColumns(db, schemas);
|
|
1019
|
-
const constraints = await loadConstraints(db, schemas);
|
|
1020
|
-
const indexes = await loadIndexes(db, schemas);
|
|
1228
|
+
const tables = await loadTables(db, database, schemas, includeViews);
|
|
1229
|
+
const columns = await loadColumns(db, database, schemas);
|
|
1230
|
+
const constraints = await loadConstraints(db, database, schemas);
|
|
1231
|
+
const indexes = await loadIndexes(db, database, schemas);
|
|
1021
1232
|
const grouped = buildTables(tables, columns, constraints, indexes);
|
|
1022
1233
|
const schemaTs = emitSchema(grouped, {
|
|
1023
1234
|
useCustomTimeTypes: opts.useCustomTimeTypes ?? true,
|
|
@@ -1031,27 +1242,41 @@ async function introspect(db, opts = {}) {
|
|
|
1031
1242
|
}
|
|
1032
1243
|
};
|
|
1033
1244
|
}
|
|
1034
|
-
async function
|
|
1245
|
+
async function resolveDatabase(db, targetDatabase, allDatabases) {
|
|
1246
|
+
if (allDatabases) {
|
|
1247
|
+
return null;
|
|
1248
|
+
}
|
|
1249
|
+
if (targetDatabase) {
|
|
1250
|
+
return targetDatabase;
|
|
1251
|
+
}
|
|
1252
|
+
const rows = await db.execute(sql5`SELECT current_database() as current_database`);
|
|
1253
|
+
return rows[0]?.current_database ?? null;
|
|
1254
|
+
}
|
|
1255
|
+
async function resolveSchemas(db, database, targetSchemas) {
|
|
1035
1256
|
if (targetSchemas?.length) {
|
|
1036
1257
|
return targetSchemas;
|
|
1037
1258
|
}
|
|
1038
|
-
const
|
|
1259
|
+
const databaseFilter = database ? sql5`catalog_name = ${database}` : sql5`1 = 1`;
|
|
1260
|
+
const rows = await db.execute(sql5`SELECT schema_name FROM information_schema.schemata WHERE ${databaseFilter}`);
|
|
1039
1261
|
return rows.map((row) => row.schema_name).filter((name) => !SYSTEM_SCHEMAS.has(name));
|
|
1040
1262
|
}
|
|
1041
|
-
async function loadTables(db, schemas, includeViews) {
|
|
1263
|
+
async function loadTables(db, database, schemas, includeViews) {
|
|
1042
1264
|
const schemaFragments = schemas.map((schema) => sql5`${schema}`);
|
|
1265
|
+
const databaseFilter = database ? sql5`table_catalog = ${database}` : sql5`1 = 1`;
|
|
1043
1266
|
return await db.execute(sql5`
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1267
|
+
SELECT table_schema as schema_name, table_name, table_type
|
|
1268
|
+
FROM information_schema.tables
|
|
1269
|
+
WHERE ${databaseFilter}
|
|
1270
|
+
AND table_schema IN (${sql5.join(schemaFragments, sql5.raw(", "))})
|
|
1271
|
+
AND ${includeViews ? sql5`1 = 1` : sql5`table_type = 'BASE TABLE'`}
|
|
1272
|
+
ORDER BY table_schema, table_name
|
|
1049
1273
|
`);
|
|
1050
1274
|
}
|
|
1051
|
-
async function loadColumns(db, schemas) {
|
|
1275
|
+
async function loadColumns(db, database, schemas) {
|
|
1052
1276
|
const schemaFragments = schemas.map((schema) => sql5`${schema}`);
|
|
1277
|
+
const databaseFilter = database ? sql5`database_name = ${database}` : sql5`1 = 1`;
|
|
1053
1278
|
return await db.execute(sql5`
|
|
1054
|
-
|
|
1279
|
+
SELECT
|
|
1055
1280
|
schema_name,
|
|
1056
1281
|
table_name,
|
|
1057
1282
|
column_name,
|
|
@@ -1063,15 +1288,17 @@ async function loadColumns(db, schemas) {
|
|
|
1063
1288
|
numeric_precision,
|
|
1064
1289
|
numeric_scale,
|
|
1065
1290
|
internal
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1291
|
+
FROM duckdb_columns()
|
|
1292
|
+
WHERE ${databaseFilter}
|
|
1293
|
+
AND schema_name IN (${sql5.join(schemaFragments, sql5.raw(", "))})
|
|
1294
|
+
ORDER BY schema_name, table_name, column_index
|
|
1069
1295
|
`);
|
|
1070
1296
|
}
|
|
1071
|
-
async function loadConstraints(db, schemas) {
|
|
1297
|
+
async function loadConstraints(db, database, schemas) {
|
|
1072
1298
|
const schemaFragments = schemas.map((schema) => sql5`${schema}`);
|
|
1299
|
+
const databaseFilter = database ? sql5`database_name = ${database}` : sql5`1 = 1`;
|
|
1073
1300
|
return await db.execute(sql5`
|
|
1074
|
-
|
|
1301
|
+
SELECT
|
|
1075
1302
|
schema_name,
|
|
1076
1303
|
table_name,
|
|
1077
1304
|
constraint_name,
|
|
@@ -1080,23 +1307,26 @@ async function loadConstraints(db, schemas) {
|
|
|
1080
1307
|
constraint_column_names,
|
|
1081
1308
|
referenced_table,
|
|
1082
1309
|
referenced_column_names
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1310
|
+
FROM duckdb_constraints()
|
|
1311
|
+
WHERE ${databaseFilter}
|
|
1312
|
+
AND schema_name IN (${sql5.join(schemaFragments, sql5.raw(", "))})
|
|
1313
|
+
ORDER BY schema_name, table_name, constraint_index
|
|
1086
1314
|
`);
|
|
1087
1315
|
}
|
|
1088
|
-
async function loadIndexes(db, schemas) {
|
|
1316
|
+
async function loadIndexes(db, database, schemas) {
|
|
1089
1317
|
const schemaFragments = schemas.map((schema) => sql5`${schema}`);
|
|
1318
|
+
const databaseFilter = database ? sql5`database_name = ${database}` : sql5`1 = 1`;
|
|
1090
1319
|
return await db.execute(sql5`
|
|
1091
|
-
|
|
1320
|
+
SELECT
|
|
1092
1321
|
schema_name,
|
|
1093
1322
|
table_name,
|
|
1094
1323
|
index_name,
|
|
1095
1324
|
is_unique,
|
|
1096
1325
|
expressions
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1326
|
+
FROM duckdb_indexes()
|
|
1327
|
+
WHERE ${databaseFilter}
|
|
1328
|
+
AND schema_name IN (${sql5.join(schemaFragments, sql5.raw(", "))})
|
|
1329
|
+
ORDER BY schema_name, table_name, index_name
|
|
1100
1330
|
`);
|
|
1101
1331
|
}
|
|
1102
1332
|
function buildTables(tables, columns, constraints, indexes) {
|
|
@@ -1291,7 +1521,9 @@ function mapDuckDbType(column, imports, options) {
|
|
|
1291
1521
|
}
|
|
1292
1522
|
if (upper === "BIGINT" || upper === "INT8" || upper === "UBIGINT") {
|
|
1293
1523
|
imports.pgCore.add("bigint");
|
|
1294
|
-
return {
|
|
1524
|
+
return {
|
|
1525
|
+
builder: `bigint(${columnName(column.name)}, { mode: 'number' })`
|
|
1526
|
+
};
|
|
1295
1527
|
}
|
|
1296
1528
|
const decimalMatch = /^DECIMAL\((\d+),(\d+)\)/i.exec(upper);
|
|
1297
1529
|
const numericMatch = /^NUMERIC\((\d+),(\d+)\)/i.exec(upper);
|
|
@@ -1538,9 +1770,140 @@ function renderImports(imports, importBasePath) {
|
|
|
1538
1770
|
return lines.join(`
|
|
1539
1771
|
`);
|
|
1540
1772
|
}
|
|
1773
|
+
// src/olap.ts
|
|
1774
|
+
import { is as is5 } from "drizzle-orm/entity";
|
|
1775
|
+
import { sql as sql6 } from "drizzle-orm";
|
|
1776
|
+
import { SQL as SQL5 } from "drizzle-orm/sql/sql";
|
|
1777
|
+
import { Column as Column3, getTableName as getTableName3 } from "drizzle-orm";
|
|
1778
|
+
var countN = (expr = sql6`*`) => sql6`count(${expr})`.mapWith(Number);
|
|
1779
|
+
var sumN = (expr) => sql6`sum(${expr})`.mapWith(Number);
|
|
1780
|
+
var avgN = (expr) => sql6`avg(${expr})`.mapWith(Number);
|
|
1781
|
+
var sumDistinctN = (expr) => sql6`sum(distinct ${expr})`.mapWith(Number);
|
|
1782
|
+
var percentileCont = (p, expr) => sql6`percentile_cont(${p}) within group (order by ${expr})`.mapWith(Number);
|
|
1783
|
+
var median = (expr) => percentileCont(0.5, expr);
|
|
1784
|
+
var anyValue = (expr) => sql6`any_value(${expr})`;
|
|
1785
|
+
function normalizeArray(value) {
|
|
1786
|
+
if (!value)
|
|
1787
|
+
return [];
|
|
1788
|
+
return Array.isArray(value) ? value : [value];
|
|
1789
|
+
}
|
|
1790
|
+
function overClause(options) {
|
|
1791
|
+
const partitions = normalizeArray(options?.partitionBy);
|
|
1792
|
+
const orders = normalizeArray(options?.orderBy);
|
|
1793
|
+
const chunks = [];
|
|
1794
|
+
if (partitions.length > 0) {
|
|
1795
|
+
chunks.push(sql6`partition by ${sql6.join(partitions, sql6`, `)}`);
|
|
1796
|
+
}
|
|
1797
|
+
if (orders.length > 0) {
|
|
1798
|
+
chunks.push(sql6`order by ${sql6.join(orders, sql6`, `)}`);
|
|
1799
|
+
}
|
|
1800
|
+
if (chunks.length === 0) {
|
|
1801
|
+
return sql6``;
|
|
1802
|
+
}
|
|
1803
|
+
return sql6`over (${sql6.join(chunks, sql6` `)})`;
|
|
1804
|
+
}
|
|
1805
|
+
var rowNumber = (options) => sql6`row_number() ${overClause(options)}`.mapWith(Number);
|
|
1806
|
+
var rank = (options) => sql6`rank() ${overClause(options)}`.mapWith(Number);
|
|
1807
|
+
var denseRank = (options) => sql6`dense_rank() ${overClause(options)}`.mapWith(Number);
|
|
1808
|
+
var lag = (expr, offset = 1, defaultValue, options) => defaultValue ? sql6`lag(${expr}, ${offset}, ${defaultValue}) ${overClause(options)}` : sql6`lag(${expr}, ${offset}) ${overClause(options)}`;
|
|
1809
|
+
var lead = (expr, offset = 1, defaultValue, options) => defaultValue ? sql6`lead(${expr}, ${offset}, ${defaultValue}) ${overClause(options)}` : sql6`lead(${expr}, ${offset}) ${overClause(options)}`;
|
|
1810
|
+
function keyAlias(key, fallback) {
|
|
1811
|
+
if (is5(key, SQL5.Aliased)) {
|
|
1812
|
+
return key.fieldAlias ?? fallback;
|
|
1813
|
+
}
|
|
1814
|
+
if (is5(key, Column3)) {
|
|
1815
|
+
return `${getTableName3(key.table)}.${key.name}`;
|
|
1816
|
+
}
|
|
1817
|
+
return fallback;
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
class OlapBuilder {
|
|
1821
|
+
db;
|
|
1822
|
+
source;
|
|
1823
|
+
keys = [];
|
|
1824
|
+
measureMap = {};
|
|
1825
|
+
nonAggregates = {};
|
|
1826
|
+
wrapNonAggWithAnyValue = false;
|
|
1827
|
+
orderByClauses = [];
|
|
1828
|
+
constructor(db) {
|
|
1829
|
+
this.db = db;
|
|
1830
|
+
}
|
|
1831
|
+
from(source) {
|
|
1832
|
+
this.source = source;
|
|
1833
|
+
return this;
|
|
1834
|
+
}
|
|
1835
|
+
groupBy(keys) {
|
|
1836
|
+
this.keys = keys;
|
|
1837
|
+
return this;
|
|
1838
|
+
}
|
|
1839
|
+
measures(measures) {
|
|
1840
|
+
this.measureMap = measures;
|
|
1841
|
+
return this;
|
|
1842
|
+
}
|
|
1843
|
+
selectNonAggregates(fields, options = {}) {
|
|
1844
|
+
this.nonAggregates = fields;
|
|
1845
|
+
this.wrapNonAggWithAnyValue = options.anyValue ?? false;
|
|
1846
|
+
return this;
|
|
1847
|
+
}
|
|
1848
|
+
orderBy(...clauses) {
|
|
1849
|
+
this.orderByClauses = clauses;
|
|
1850
|
+
return this;
|
|
1851
|
+
}
|
|
1852
|
+
build() {
|
|
1853
|
+
if (!this.source) {
|
|
1854
|
+
throw new Error("olap: .from() is required");
|
|
1855
|
+
}
|
|
1856
|
+
if (this.keys.length === 0) {
|
|
1857
|
+
throw new Error("olap: .groupBy() is required");
|
|
1858
|
+
}
|
|
1859
|
+
if (Object.keys(this.measureMap).length === 0) {
|
|
1860
|
+
throw new Error("olap: .measures() is required");
|
|
1861
|
+
}
|
|
1862
|
+
const selection = {};
|
|
1863
|
+
this.keys.forEach((key, idx) => {
|
|
1864
|
+
const alias = keyAlias(key, `key_${idx}`);
|
|
1865
|
+
selection[alias] = key;
|
|
1866
|
+
});
|
|
1867
|
+
Object.entries(this.nonAggregates).forEach(([alias, expr]) => {
|
|
1868
|
+
selection[alias] = this.wrapNonAggWithAnyValue ? anyValue(expr) : expr;
|
|
1869
|
+
});
|
|
1870
|
+
Object.assign(selection, this.measureMap);
|
|
1871
|
+
let query = this.db.select(selection).from(this.source).groupBy(...this.keys);
|
|
1872
|
+
if (this.orderByClauses.length > 0) {
|
|
1873
|
+
query = query.orderBy(...this.orderByClauses);
|
|
1874
|
+
}
|
|
1875
|
+
return query;
|
|
1876
|
+
}
|
|
1877
|
+
run() {
|
|
1878
|
+
return this.build();
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
var olap = (db) => new OlapBuilder(db);
|
|
1541
1882
|
export {
|
|
1883
|
+
wrapperToNodeApiValue,
|
|
1884
|
+
wrapTimestamp,
|
|
1885
|
+
wrapStruct,
|
|
1886
|
+
wrapMap,
|
|
1887
|
+
wrapList,
|
|
1888
|
+
wrapJson,
|
|
1889
|
+
wrapBlob,
|
|
1890
|
+
wrapArray,
|
|
1891
|
+
sumN,
|
|
1892
|
+
sumDistinctN,
|
|
1893
|
+
rowNumber,
|
|
1894
|
+
rank,
|
|
1895
|
+
prepareParams,
|
|
1896
|
+
percentileCont,
|
|
1897
|
+
olap,
|
|
1542
1898
|
migrate,
|
|
1899
|
+
median,
|
|
1900
|
+
lead,
|
|
1901
|
+
lag,
|
|
1902
|
+
isDuckDBWrapper,
|
|
1543
1903
|
introspect,
|
|
1904
|
+
executeOnClient,
|
|
1905
|
+
executeInBatches,
|
|
1906
|
+
executeArrowOnClient,
|
|
1544
1907
|
duckDbTimestamp,
|
|
1545
1908
|
duckDbTime,
|
|
1546
1909
|
duckDbStruct,
|
|
@@ -1556,9 +1919,16 @@ export {
|
|
|
1556
1919
|
duckDbArrayContained,
|
|
1557
1920
|
duckDbArray,
|
|
1558
1921
|
drizzle,
|
|
1922
|
+
denseRank,
|
|
1923
|
+
countN,
|
|
1924
|
+
closeClientConnection,
|
|
1925
|
+
avgN,
|
|
1926
|
+
anyValue,
|
|
1927
|
+
OlapBuilder,
|
|
1559
1928
|
DuckDBTransaction,
|
|
1560
1929
|
DuckDBSession,
|
|
1561
1930
|
DuckDBPreparedQuery,
|
|
1562
1931
|
DuckDBDriver,
|
|
1563
|
-
DuckDBDatabase
|
|
1932
|
+
DuckDBDatabase,
|
|
1933
|
+
DUCKDB_VALUE_MARKER
|
|
1564
1934
|
};
|
package/dist/introspect.d.ts
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
import type { RowData } from './client.ts';
|
|
2
2
|
import type { DuckDBDatabase } from './driver.ts';
|
|
3
3
|
export interface IntrospectOptions {
|
|
4
|
+
/**
|
|
5
|
+
* Database/catalog to introspect. If not specified, uses the current database
|
|
6
|
+
* (via `SELECT current_database()`). This prevents returning tables from all
|
|
7
|
+
* attached databases in MotherDuck workspaces.
|
|
8
|
+
*/
|
|
9
|
+
database?: string;
|
|
10
|
+
/**
|
|
11
|
+
* When true, introspects all attached databases instead of just the current one.
|
|
12
|
+
* Ignored if `database` is explicitly set.
|
|
13
|
+
* @default false
|
|
14
|
+
*/
|
|
15
|
+
allDatabases?: boolean;
|
|
4
16
|
schemas?: string[];
|
|
5
17
|
includeViews?: boolean;
|
|
6
18
|
useCustomTimeTypes?: boolean;
|