@leonardovida-md/drizzle-neo-duckdb 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/columns.d.ts +5 -0
- package/dist/dialect.d.ts +21 -0
- package/dist/duckdb-introspect.mjs +414 -92
- package/dist/helpers.mjs +4 -4
- package/dist/index.mjs +416 -96
- package/dist/introspect.d.ts +8 -0
- package/dist/pool.d.ts +8 -0
- package/dist/session.d.ts +7 -1
- package/dist/sql/query-rewriters.d.ts +1 -0
- package/dist/sql/result-mapper.d.ts +7 -0
- package/dist/value-wrappers-core.d.ts +2 -2
- package/package.json +1 -1
- package/src/bin/duckdb-introspect.ts +27 -0
- package/src/client.ts +13 -10
- package/src/columns.ts +9 -9
- package/src/dialect.ts +51 -3
- package/src/driver.ts +15 -2
- package/src/introspect.ts +23 -6
- package/src/pool.ts +182 -12
- package/src/session.ts +98 -5
- package/src/sql/query-rewriters.ts +183 -75
- package/src/sql/result-mapper.ts +7 -7
- package/src/value-wrappers-core.ts +2 -2
- package/src/value-wrappers.ts +13 -3
|
@@ -24,83 +24,159 @@ import { PgPreparedQuery, PgSession } from "drizzle-orm/pg-core/session";
|
|
|
24
24
|
import { fillPlaceholders, sql } from "drizzle-orm/sql/sql";
|
|
25
25
|
|
|
26
26
|
// src/sql/query-rewriters.ts
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
break;
|
|
45
|
-
if (ch === "'" && source[idx - 1] !== "\\") {
|
|
46
|
-
inString = !inString;
|
|
27
|
+
var OPERATORS = [
|
|
28
|
+
{ token: "@>", fn: "array_has_all" },
|
|
29
|
+
{ token: "<@", fn: "array_has_all", swap: true },
|
|
30
|
+
{ token: "&&", fn: "array_has_any" }
|
|
31
|
+
];
|
|
32
|
+
var isWhitespace = (char) => char !== undefined && /\s/.test(char);
|
|
33
|
+
function scrubForRewrite(query) {
|
|
34
|
+
let scrubbed = "";
|
|
35
|
+
let state = "code";
|
|
36
|
+
for (let i = 0;i < query.length; i += 1) {
|
|
37
|
+
const char = query[i];
|
|
38
|
+
const next = query[i + 1];
|
|
39
|
+
if (state === "code") {
|
|
40
|
+
if (char === "'") {
|
|
41
|
+
scrubbed += "'";
|
|
42
|
+
state = "single";
|
|
43
|
+
continue;
|
|
47
44
|
}
|
|
48
|
-
if (
|
|
45
|
+
if (char === '"') {
|
|
46
|
+
scrubbed += '"';
|
|
47
|
+
state = "double";
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
if (char === "-" && next === "-") {
|
|
51
|
+
scrubbed += " ";
|
|
52
|
+
i += 1;
|
|
53
|
+
state = "lineComment";
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
if (char === "/" && next === "*") {
|
|
57
|
+
scrubbed += " ";
|
|
58
|
+
i += 1;
|
|
59
|
+
state = "blockComment";
|
|
49
60
|
continue;
|
|
50
|
-
if (ch === ")" || ch === "]") {
|
|
51
|
-
depth++;
|
|
52
|
-
} else if (ch === "(" || ch === "[") {
|
|
53
|
-
depth--;
|
|
54
|
-
if (depth < 0) {
|
|
55
|
-
return [idx + 1, source.slice(idx + 1, start + 1)];
|
|
56
|
-
}
|
|
57
|
-
} else if (depth === 0 && isWhitespace(ch)) {
|
|
58
|
-
return [idx + 1, source.slice(idx + 1, start + 1)];
|
|
59
61
|
}
|
|
62
|
+
scrubbed += char;
|
|
63
|
+
continue;
|
|
60
64
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
idx++;
|
|
67
|
-
}
|
|
68
|
-
let depth = 0;
|
|
69
|
-
let inString = false;
|
|
70
|
-
for (;idx < source.length; idx++) {
|
|
71
|
-
const ch = source[idx];
|
|
72
|
-
if (ch === undefined)
|
|
73
|
-
break;
|
|
74
|
-
if (ch === "'" && source[idx - 1] !== "\\") {
|
|
75
|
-
inString = !inString;
|
|
65
|
+
if (state === "single") {
|
|
66
|
+
if (char === "'" && next === "'") {
|
|
67
|
+
scrubbed += "''";
|
|
68
|
+
i += 1;
|
|
69
|
+
continue;
|
|
76
70
|
}
|
|
77
|
-
|
|
71
|
+
scrubbed += char === "'" ? "'" : ".";
|
|
72
|
+
if (char === "'") {
|
|
73
|
+
state = "code";
|
|
74
|
+
}
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
if (state === "double") {
|
|
78
|
+
if (char === '"' && next === '"') {
|
|
79
|
+
scrubbed += '""';
|
|
80
|
+
i += 1;
|
|
78
81
|
continue;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
82
|
+
}
|
|
83
|
+
scrubbed += char === '"' ? '"' : ".";
|
|
84
|
+
if (char === '"') {
|
|
85
|
+
state = "code";
|
|
86
|
+
}
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (state === "lineComment") {
|
|
90
|
+
scrubbed += char === `
|
|
91
|
+
` ? `
|
|
92
|
+
` : " ";
|
|
93
|
+
if (char === `
|
|
94
|
+
`) {
|
|
95
|
+
state = "code";
|
|
96
|
+
}
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
if (state === "blockComment") {
|
|
100
|
+
if (char === "*" && next === "/") {
|
|
101
|
+
scrubbed += " ";
|
|
102
|
+
i += 1;
|
|
103
|
+
state = "code";
|
|
104
|
+
} else {
|
|
105
|
+
scrubbed += " ";
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return scrubbed;
|
|
110
|
+
}
|
|
111
|
+
function findNextOperator(scrubbed, start) {
|
|
112
|
+
for (let idx = start;idx < scrubbed.length; idx += 1) {
|
|
113
|
+
for (const operator of OPERATORS) {
|
|
114
|
+
if (scrubbed.startsWith(operator.token, idx)) {
|
|
115
|
+
return { index: idx, operator };
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
function walkLeft(source, scrubbed, start) {
|
|
122
|
+
let idx = start;
|
|
123
|
+
while (idx >= 0 && isWhitespace(scrubbed[idx])) {
|
|
124
|
+
idx -= 1;
|
|
125
|
+
}
|
|
126
|
+
let depth = 0;
|
|
127
|
+
for (;idx >= 0; idx -= 1) {
|
|
128
|
+
const ch = scrubbed[idx];
|
|
129
|
+
if (ch === ")" || ch === "]") {
|
|
130
|
+
depth += 1;
|
|
131
|
+
} else if (ch === "(" || ch === "[") {
|
|
132
|
+
if (depth === 0) {
|
|
133
|
+
return [idx + 1, source.slice(idx + 1, start + 1)];
|
|
134
|
+
}
|
|
135
|
+
depth = Math.max(0, depth - 1);
|
|
136
|
+
} else if (depth === 0 && isWhitespace(ch)) {
|
|
137
|
+
return [idx + 1, source.slice(idx + 1, start + 1)];
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return [0, source.slice(0, start + 1)];
|
|
141
|
+
}
|
|
142
|
+
function walkRight(source, scrubbed, start) {
|
|
143
|
+
let idx = start;
|
|
144
|
+
while (idx < scrubbed.length && isWhitespace(scrubbed[idx])) {
|
|
145
|
+
idx += 1;
|
|
146
|
+
}
|
|
147
|
+
let depth = 0;
|
|
148
|
+
for (;idx < scrubbed.length; idx += 1) {
|
|
149
|
+
const ch = scrubbed[idx];
|
|
150
|
+
if (ch === "(" || ch === "[") {
|
|
151
|
+
depth += 1;
|
|
152
|
+
} else if (ch === ")" || ch === "]") {
|
|
153
|
+
if (depth === 0) {
|
|
87
154
|
return [idx, source.slice(start, idx)];
|
|
88
155
|
}
|
|
156
|
+
depth = Math.max(0, depth - 1);
|
|
157
|
+
} else if (depth === 0 && isWhitespace(ch)) {
|
|
158
|
+
return [idx, source.slice(start, idx)];
|
|
89
159
|
}
|
|
90
|
-
|
|
91
|
-
|
|
160
|
+
}
|
|
161
|
+
return [scrubbed.length, source.slice(start)];
|
|
162
|
+
}
|
|
163
|
+
function adaptArrayOperators(query) {
|
|
92
164
|
let rewritten = query;
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
165
|
+
let scrubbed = scrubForRewrite(query);
|
|
166
|
+
let searchStart = 0;
|
|
167
|
+
while (true) {
|
|
168
|
+
const next = findNextOperator(scrubbed, searchStart);
|
|
169
|
+
if (!next)
|
|
170
|
+
break;
|
|
171
|
+
const { index, operator } = next;
|
|
172
|
+
const [leftStart, leftExpr] = walkLeft(rewritten, scrubbed, index - 1);
|
|
173
|
+
const [rightEnd, rightExpr] = walkRight(rewritten, scrubbed, index + operator.token.length);
|
|
174
|
+
const left = leftExpr.trim();
|
|
175
|
+
const right = rightExpr.trim();
|
|
176
|
+
const replacement = `${operator.fn}(${operator.swap ? right : left}, ${operator.swap ? left : right})`;
|
|
177
|
+
rewritten = rewritten.slice(0, leftStart) + replacement + rewritten.slice(rightEnd);
|
|
178
|
+
scrubbed = scrubForRewrite(rewritten);
|
|
179
|
+
searchStart = leftStart + replacement.length;
|
|
104
180
|
}
|
|
105
181
|
return rewritten;
|
|
106
182
|
}
|
|
@@ -335,6 +411,12 @@ function dateToMicros(value) {
|
|
|
335
411
|
if (value instanceof Date) {
|
|
336
412
|
return BigInt(value.getTime()) * 1000n;
|
|
337
413
|
}
|
|
414
|
+
if (typeof value === "bigint") {
|
|
415
|
+
return value;
|
|
416
|
+
}
|
|
417
|
+
if (typeof value === "number") {
|
|
418
|
+
return BigInt(Math.trunc(value)) * 1000n;
|
|
419
|
+
}
|
|
338
420
|
let normalized = value;
|
|
339
421
|
if (!value.includes("T") && value.includes(" ")) {
|
|
340
422
|
normalized = value.replace(" ", "T");
|
|
@@ -404,14 +486,17 @@ function parsePgArrayLiteral(value) {
|
|
|
404
486
|
}
|
|
405
487
|
function prepareParams(params, options = {}) {
|
|
406
488
|
return params.map((param) => {
|
|
407
|
-
if (typeof param === "string"
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
489
|
+
if (typeof param === "string") {
|
|
490
|
+
const trimmed = param.trim();
|
|
491
|
+
if (trimmed && isPgArrayLiteral(trimmed)) {
|
|
492
|
+
if (options.rejectStringArrayLiterals) {
|
|
493
|
+
throw new Error("Stringified array literals are not supported. Use duckDbList()/duckDbArray() or pass native arrays.");
|
|
494
|
+
}
|
|
495
|
+
if (options.warnOnStringArrayLiteral) {
|
|
496
|
+
options.warnOnStringArrayLiteral();
|
|
497
|
+
}
|
|
498
|
+
return parsePgArrayLiteral(trimmed);
|
|
413
499
|
}
|
|
414
|
-
return parsePgArrayLiteral(param);
|
|
415
500
|
}
|
|
416
501
|
return param;
|
|
417
502
|
});
|
|
@@ -529,6 +614,13 @@ async function executeArrowOnClient(client, query, params) {
|
|
|
529
614
|
}
|
|
530
615
|
|
|
531
616
|
// src/session.ts
|
|
617
|
+
function isSavepointSyntaxError(error) {
|
|
618
|
+
if (!(error instanceof Error) || !error.message) {
|
|
619
|
+
return false;
|
|
620
|
+
}
|
|
621
|
+
return error.message.toLowerCase().includes("savepoint") && error.message.toLowerCase().includes("syntax error");
|
|
622
|
+
}
|
|
623
|
+
|
|
532
624
|
class DuckDBPreparedQuery extends PgPreparedQuery {
|
|
533
625
|
client;
|
|
534
626
|
dialect;
|
|
@@ -593,6 +685,7 @@ class DuckDBSession extends PgSession {
|
|
|
593
685
|
rewriteArrays;
|
|
594
686
|
rejectStringArrayLiterals;
|
|
595
687
|
hasWarnedArrayLiteral = false;
|
|
688
|
+
rollbackOnly = false;
|
|
596
689
|
constructor(client, dialect, schema, options = {}) {
|
|
597
690
|
super(dialect);
|
|
598
691
|
this.client = client;
|
|
@@ -606,7 +699,15 @@ class DuckDBSession extends PgSession {
|
|
|
606
699
|
prepareQuery(query, fields, name, isResponseInArrayMode, customResultMapper) {
|
|
607
700
|
return new DuckDBPreparedQuery(this.client, this.dialect, query.sql, query.params, this.logger, fields, isResponseInArrayMode, customResultMapper, this.rewriteArrays, this.rejectStringArrayLiterals, this.rejectStringArrayLiterals ? undefined : this.warnOnStringArrayLiteral);
|
|
608
701
|
}
|
|
609
|
-
|
|
702
|
+
execute(query) {
|
|
703
|
+
this.dialect.resetPgJsonFlag();
|
|
704
|
+
return super.execute(query);
|
|
705
|
+
}
|
|
706
|
+
all(query) {
|
|
707
|
+
this.dialect.resetPgJsonFlag();
|
|
708
|
+
return super.all(query);
|
|
709
|
+
}
|
|
710
|
+
async transaction(transaction, config) {
|
|
610
711
|
let pinnedConnection;
|
|
611
712
|
let pool;
|
|
612
713
|
let clientForTx = this.client;
|
|
@@ -619,8 +720,15 @@ class DuckDBSession extends PgSession {
|
|
|
619
720
|
const tx = new DuckDBTransaction(this.dialect, session, this.schema);
|
|
620
721
|
try {
|
|
621
722
|
await tx.execute(sql`BEGIN TRANSACTION;`);
|
|
723
|
+
if (config) {
|
|
724
|
+
await tx.setTransaction(config);
|
|
725
|
+
}
|
|
622
726
|
try {
|
|
623
727
|
const result = await transaction(tx);
|
|
728
|
+
if (session.isRollbackOnly()) {
|
|
729
|
+
await tx.execute(sql`rollback`);
|
|
730
|
+
throw new TransactionRollbackError;
|
|
731
|
+
}
|
|
624
732
|
await tx.execute(sql`commit`);
|
|
625
733
|
return result;
|
|
626
734
|
} catch (error) {
|
|
@@ -642,7 +750,9 @@ class DuckDBSession extends PgSession {
|
|
|
642
750
|
query: ${query}`, []);
|
|
643
751
|
};
|
|
644
752
|
executeBatches(query, options = {}) {
|
|
753
|
+
this.dialect.resetPgJsonFlag();
|
|
645
754
|
const builtQuery = this.dialect.sqlToQuery(query);
|
|
755
|
+
this.dialect.assertNoPgJsonColumns();
|
|
646
756
|
const params = prepareParams(builtQuery.params, {
|
|
647
757
|
rejectStringArrayLiterals: this.rejectStringArrayLiterals,
|
|
648
758
|
warnOnStringArrayLiteral: this.rejectStringArrayLiterals ? undefined : () => this.warnOnStringArrayLiteral(builtQuery.sql)
|
|
@@ -655,7 +765,9 @@ query: ${query}`, []);
|
|
|
655
765
|
return executeInBatches(this.client, rewrittenQuery, params, options);
|
|
656
766
|
}
|
|
657
767
|
async executeArrow(query) {
|
|
768
|
+
this.dialect.resetPgJsonFlag();
|
|
658
769
|
const builtQuery = this.dialect.sqlToQuery(query);
|
|
770
|
+
this.dialect.assertNoPgJsonColumns();
|
|
659
771
|
const params = prepareParams(builtQuery.params, {
|
|
660
772
|
rejectStringArrayLiterals: this.rejectStringArrayLiterals,
|
|
661
773
|
warnOnStringArrayLiteral: this.rejectStringArrayLiterals ? undefined : () => this.warnOnStringArrayLiteral(builtQuery.sql)
|
|
@@ -667,6 +779,12 @@ query: ${query}`, []);
|
|
|
667
779
|
this.logger.logQuery(rewrittenQuery, params);
|
|
668
780
|
return executeArrowOnClient(this.client, rewrittenQuery, params);
|
|
669
781
|
}
|
|
782
|
+
markRollbackOnly() {
|
|
783
|
+
this.rollbackOnly = true;
|
|
784
|
+
}
|
|
785
|
+
isRollbackOnly() {
|
|
786
|
+
return this.rollbackOnly;
|
|
787
|
+
}
|
|
670
788
|
}
|
|
671
789
|
|
|
672
790
|
class DuckDBTransaction extends PgTransaction {
|
|
@@ -697,8 +815,46 @@ class DuckDBTransaction extends PgTransaction {
|
|
|
697
815
|
return this.session.executeArrow(query);
|
|
698
816
|
}
|
|
699
817
|
async transaction(transaction) {
|
|
700
|
-
const
|
|
701
|
-
|
|
818
|
+
const internals = this;
|
|
819
|
+
const savepoint = `drizzle_savepoint_${this.nestedIndex + 1}`;
|
|
820
|
+
const savepointSql = sql.raw(`savepoint ${savepoint}`);
|
|
821
|
+
const releaseSql = sql.raw(`release savepoint ${savepoint}`);
|
|
822
|
+
const rollbackSql = sql.raw(`rollback to savepoint ${savepoint}`);
|
|
823
|
+
const nestedTx = new DuckDBTransaction(internals.dialect, internals.session, this.schema, this.nestedIndex + 1);
|
|
824
|
+
if (internals.dialect.areSavepointsUnsupported()) {
|
|
825
|
+
return this.runNestedWithoutSavepoint(transaction, nestedTx, internals);
|
|
826
|
+
}
|
|
827
|
+
let createdSavepoint = false;
|
|
828
|
+
try {
|
|
829
|
+
await internals.session.execute(savepointSql);
|
|
830
|
+
internals.dialect.markSavepointsSupported();
|
|
831
|
+
createdSavepoint = true;
|
|
832
|
+
} catch (error) {
|
|
833
|
+
if (!isSavepointSyntaxError(error)) {
|
|
834
|
+
throw error;
|
|
835
|
+
}
|
|
836
|
+
internals.dialect.markSavepointsUnsupported();
|
|
837
|
+
return this.runNestedWithoutSavepoint(transaction, nestedTx, internals);
|
|
838
|
+
}
|
|
839
|
+
try {
|
|
840
|
+
const result = await transaction(nestedTx);
|
|
841
|
+
if (createdSavepoint) {
|
|
842
|
+
await internals.session.execute(releaseSql);
|
|
843
|
+
}
|
|
844
|
+
return result;
|
|
845
|
+
} catch (error) {
|
|
846
|
+
if (createdSavepoint) {
|
|
847
|
+
await internals.session.execute(rollbackSql);
|
|
848
|
+
}
|
|
849
|
+
internals.session.markRollbackOnly();
|
|
850
|
+
throw error;
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
runNestedWithoutSavepoint(transaction, nestedTx, internals) {
|
|
854
|
+
return transaction(nestedTx).catch((error) => {
|
|
855
|
+
internals.session.markRollbackOnly();
|
|
856
|
+
throw error;
|
|
857
|
+
});
|
|
702
858
|
}
|
|
703
859
|
}
|
|
704
860
|
var arrayLiteralWarning = "Received a stringified Postgres-style array literal. Use duckDbList()/duckDbArray() or pass native arrays instead. You can also set rejectStringArrayLiterals=true to throw.";
|
|
@@ -720,15 +876,30 @@ import {
|
|
|
720
876
|
import {
|
|
721
877
|
sql as sql2
|
|
722
878
|
} from "drizzle-orm";
|
|
723
|
-
|
|
724
879
|
class DuckDBDialect extends PgDialect {
|
|
725
880
|
static [entityKind2] = "DuckDBPgDialect";
|
|
726
881
|
hasPgJsonColumn = false;
|
|
882
|
+
savepointsSupported = 0 /* Unknown */;
|
|
883
|
+
resetPgJsonFlag() {
|
|
884
|
+
this.hasPgJsonColumn = false;
|
|
885
|
+
}
|
|
886
|
+
markPgJsonDetected() {
|
|
887
|
+
this.hasPgJsonColumn = true;
|
|
888
|
+
}
|
|
727
889
|
assertNoPgJsonColumns() {
|
|
728
890
|
if (this.hasPgJsonColumn) {
|
|
729
|
-
throw new Error("Pg JSON/JSONB columns are not supported in DuckDB. Replace them with duckDbJson() to use DuckDB
|
|
891
|
+
throw new Error("Pg JSON/JSONB columns are not supported in DuckDB. Replace them with duckDbJson() to use DuckDB's native JSON type.");
|
|
730
892
|
}
|
|
731
893
|
}
|
|
894
|
+
areSavepointsUnsupported() {
|
|
895
|
+
return this.savepointsSupported === 2 /* No */;
|
|
896
|
+
}
|
|
897
|
+
markSavepointsSupported() {
|
|
898
|
+
this.savepointsSupported = 1 /* Yes */;
|
|
899
|
+
}
|
|
900
|
+
markSavepointsUnsupported() {
|
|
901
|
+
this.savepointsSupported = 2 /* No */;
|
|
902
|
+
}
|
|
732
903
|
async migrate(migrations, session, config) {
|
|
733
904
|
const migrationConfig = typeof config === "string" ? { migrationsFolder: config } : config;
|
|
734
905
|
const migrationsSchema = migrationConfig.migrationsSchema ?? "drizzle";
|
|
@@ -765,8 +936,8 @@ class DuckDBDialect extends PgDialect {
|
|
|
765
936
|
}
|
|
766
937
|
prepareTyping(encoder) {
|
|
767
938
|
if (is2(encoder, PgJsonb) || is2(encoder, PgJson)) {
|
|
768
|
-
this.
|
|
769
|
-
|
|
939
|
+
this.markPgJsonDetected();
|
|
940
|
+
throw new Error("Pg JSON/JSONB columns are not supported in DuckDB. Replace them with duckDbJson() to use DuckDB's native JSON type.");
|
|
770
941
|
} else if (is2(encoder, PgNumeric)) {
|
|
771
942
|
return "decimal";
|
|
772
943
|
} else if (is2(encoder, PgTime2)) {
|
|
@@ -900,41 +1071,148 @@ function resolvePoolSize(pool) {
|
|
|
900
1071
|
}
|
|
901
1072
|
function createDuckDBConnectionPool(instance, options = {}) {
|
|
902
1073
|
const size = options.size && options.size > 0 ? options.size : 4;
|
|
1074
|
+
const acquireTimeout = options.acquireTimeout ?? 30000;
|
|
1075
|
+
const maxWaitingRequests = options.maxWaitingRequests ?? 100;
|
|
1076
|
+
const maxLifetimeMs = options.maxLifetimeMs;
|
|
1077
|
+
const idleTimeoutMs = options.idleTimeoutMs;
|
|
1078
|
+
const metadata = new WeakMap;
|
|
903
1079
|
const idle = [];
|
|
904
1080
|
const waiting = [];
|
|
905
1081
|
let total = 0;
|
|
906
1082
|
let closed = false;
|
|
1083
|
+
let pendingAcquires = 0;
|
|
1084
|
+
const shouldRecycle = (conn, now) => {
|
|
1085
|
+
if (maxLifetimeMs !== undefined && now - conn.createdAt >= maxLifetimeMs) {
|
|
1086
|
+
return true;
|
|
1087
|
+
}
|
|
1088
|
+
if (idleTimeoutMs !== undefined && now - conn.lastUsedAt >= idleTimeoutMs) {
|
|
1089
|
+
return true;
|
|
1090
|
+
}
|
|
1091
|
+
return false;
|
|
1092
|
+
};
|
|
907
1093
|
const acquire = async () => {
|
|
908
1094
|
if (closed) {
|
|
909
1095
|
throw new Error("DuckDB connection pool is closed");
|
|
910
1096
|
}
|
|
911
|
-
|
|
912
|
-
|
|
1097
|
+
while (idle.length > 0) {
|
|
1098
|
+
const pooled = idle.pop();
|
|
1099
|
+
const now = Date.now();
|
|
1100
|
+
if (shouldRecycle(pooled, now)) {
|
|
1101
|
+
await closeClientConnection(pooled.connection);
|
|
1102
|
+
total = Math.max(0, total - 1);
|
|
1103
|
+
metadata.delete(pooled.connection);
|
|
1104
|
+
continue;
|
|
1105
|
+
}
|
|
1106
|
+
pooled.lastUsedAt = now;
|
|
1107
|
+
metadata.set(pooled.connection, {
|
|
1108
|
+
createdAt: pooled.createdAt,
|
|
1109
|
+
lastUsedAt: pooled.lastUsedAt
|
|
1110
|
+
});
|
|
1111
|
+
return pooled.connection;
|
|
913
1112
|
}
|
|
914
1113
|
if (total < size) {
|
|
1114
|
+
pendingAcquires += 1;
|
|
915
1115
|
total += 1;
|
|
916
|
-
|
|
1116
|
+
try {
|
|
1117
|
+
const connection = await DuckDBConnection.create(instance);
|
|
1118
|
+
if (closed) {
|
|
1119
|
+
await closeClientConnection(connection);
|
|
1120
|
+
total -= 1;
|
|
1121
|
+
throw new Error("DuckDB connection pool is closed");
|
|
1122
|
+
}
|
|
1123
|
+
const now = Date.now();
|
|
1124
|
+
metadata.set(connection, { createdAt: now, lastUsedAt: now });
|
|
1125
|
+
return connection;
|
|
1126
|
+
} catch (error) {
|
|
1127
|
+
total -= 1;
|
|
1128
|
+
throw error;
|
|
1129
|
+
} finally {
|
|
1130
|
+
pendingAcquires -= 1;
|
|
1131
|
+
}
|
|
917
1132
|
}
|
|
918
|
-
|
|
919
|
-
waiting
|
|
1133
|
+
if (waiting.length >= maxWaitingRequests) {
|
|
1134
|
+
throw new Error(`DuckDB connection pool queue is full (max ${maxWaitingRequests} waiting requests)`);
|
|
1135
|
+
}
|
|
1136
|
+
return await new Promise((resolve, reject) => {
|
|
1137
|
+
const timeoutId = setTimeout(() => {
|
|
1138
|
+
const idx = waiting.findIndex((w) => w.timeoutId === timeoutId);
|
|
1139
|
+
if (idx !== -1) {
|
|
1140
|
+
waiting.splice(idx, 1);
|
|
1141
|
+
}
|
|
1142
|
+
reject(new Error(`DuckDB connection pool acquire timeout after ${acquireTimeout}ms`));
|
|
1143
|
+
}, acquireTimeout);
|
|
1144
|
+
waiting.push({ resolve, reject, timeoutId });
|
|
920
1145
|
});
|
|
921
1146
|
};
|
|
922
1147
|
const release = async (connection) => {
|
|
1148
|
+
const waiter = waiting.shift();
|
|
1149
|
+
if (waiter) {
|
|
1150
|
+
clearTimeout(waiter.timeoutId);
|
|
1151
|
+
const now2 = Date.now();
|
|
1152
|
+
const meta = metadata.get(connection) ?? { createdAt: now2, lastUsedAt: now2 };
|
|
1153
|
+
const expired = maxLifetimeMs !== undefined && now2 - meta.createdAt >= maxLifetimeMs;
|
|
1154
|
+
if (closed) {
|
|
1155
|
+
await closeClientConnection(connection);
|
|
1156
|
+
total = Math.max(0, total - 1);
|
|
1157
|
+
metadata.delete(connection);
|
|
1158
|
+
waiter.reject(new Error("DuckDB connection pool is closed"));
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
if (expired) {
|
|
1162
|
+
await closeClientConnection(connection);
|
|
1163
|
+
total = Math.max(0, total - 1);
|
|
1164
|
+
metadata.delete(connection);
|
|
1165
|
+
try {
|
|
1166
|
+
const replacement = await acquire();
|
|
1167
|
+
waiter.resolve(replacement);
|
|
1168
|
+
} catch (error) {
|
|
1169
|
+
waiter.reject(error);
|
|
1170
|
+
}
|
|
1171
|
+
return;
|
|
1172
|
+
}
|
|
1173
|
+
meta.lastUsedAt = now2;
|
|
1174
|
+
metadata.set(connection, meta);
|
|
1175
|
+
waiter.resolve(connection);
|
|
1176
|
+
return;
|
|
1177
|
+
}
|
|
923
1178
|
if (closed) {
|
|
924
1179
|
await closeClientConnection(connection);
|
|
1180
|
+
metadata.delete(connection);
|
|
1181
|
+
total = Math.max(0, total - 1);
|
|
925
1182
|
return;
|
|
926
1183
|
}
|
|
927
|
-
const
|
|
928
|
-
|
|
929
|
-
|
|
1184
|
+
const now = Date.now();
|
|
1185
|
+
const existingMeta = metadata.get(connection) ?? { createdAt: now, lastUsedAt: now };
|
|
1186
|
+
existingMeta.lastUsedAt = now;
|
|
1187
|
+
metadata.set(connection, existingMeta);
|
|
1188
|
+
if (maxLifetimeMs !== undefined && now - existingMeta.createdAt >= maxLifetimeMs) {
|
|
1189
|
+
await closeClientConnection(connection);
|
|
1190
|
+
total -= 1;
|
|
1191
|
+
metadata.delete(connection);
|
|
930
1192
|
return;
|
|
931
1193
|
}
|
|
932
|
-
idle.push(
|
|
1194
|
+
idle.push({
|
|
1195
|
+
connection,
|
|
1196
|
+
createdAt: existingMeta.createdAt,
|
|
1197
|
+
lastUsedAt: existingMeta.lastUsedAt
|
|
1198
|
+
});
|
|
933
1199
|
};
|
|
934
1200
|
const close = async () => {
|
|
935
1201
|
closed = true;
|
|
1202
|
+
const waiters = waiting.splice(0, waiting.length);
|
|
1203
|
+
for (const waiter of waiters) {
|
|
1204
|
+
clearTimeout(waiter.timeoutId);
|
|
1205
|
+
waiter.reject(new Error("DuckDB connection pool is closed"));
|
|
1206
|
+
}
|
|
936
1207
|
const toClose = idle.splice(0, idle.length);
|
|
937
|
-
await Promise.
|
|
1208
|
+
await Promise.allSettled(toClose.map((item) => closeClientConnection(item.connection)));
|
|
1209
|
+
total = Math.max(0, total - toClose.length);
|
|
1210
|
+
toClose.forEach((item) => metadata.delete(item.connection));
|
|
1211
|
+
const maxWait = 5000;
|
|
1212
|
+
const start = Date.now();
|
|
1213
|
+
while (pendingAcquires > 0 && Date.now() - start < maxWait) {
|
|
1214
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
1215
|
+
}
|
|
938
1216
|
};
|
|
939
1217
|
return {
|
|
940
1218
|
acquire,
|
|
@@ -1042,6 +1320,17 @@ class DuckDBDatabase extends PgDatabase {
|
|
|
1042
1320
|
if (isPool(this.$client) && this.$client.close) {
|
|
1043
1321
|
await this.$client.close();
|
|
1044
1322
|
}
|
|
1323
|
+
if (!isPool(this.$client)) {
|
|
1324
|
+
await closeClientConnection(this.$client);
|
|
1325
|
+
}
|
|
1326
|
+
if (this.$instance) {
|
|
1327
|
+
const maybeClosable = this.$instance;
|
|
1328
|
+
if (typeof maybeClosable.close === "function") {
|
|
1329
|
+
await maybeClosable.close();
|
|
1330
|
+
} else if (typeof maybeClosable.closeSync === "function") {
|
|
1331
|
+
maybeClosable.closeSync();
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1045
1334
|
}
|
|
1046
1335
|
select(fields) {
|
|
1047
1336
|
const selectedFields = fields ? aliasFields(fields) : undefined;
|
|
@@ -1445,6 +1734,20 @@ function mapDuckDbType(column, imports, options) {
|
|
|
1445
1734
|
imports.pgCore.add("text");
|
|
1446
1735
|
return { builder: `text(${columnName(column.name)}) /* JSON */` };
|
|
1447
1736
|
}
|
|
1737
|
+
if (upper.startsWith("ENUM")) {
|
|
1738
|
+
imports.pgCore.add("text");
|
|
1739
|
+
const enumLiteral = raw.replace(/^ENUM\s*/i, "").trim();
|
|
1740
|
+
return {
|
|
1741
|
+
builder: `text(${columnName(column.name)}) /* ENUM ${enumLiteral} */`
|
|
1742
|
+
};
|
|
1743
|
+
}
|
|
1744
|
+
if (upper.startsWith("UNION")) {
|
|
1745
|
+
imports.pgCore.add("text");
|
|
1746
|
+
const unionLiteral = raw.replace(/^UNION\s*/i, "").trim();
|
|
1747
|
+
return {
|
|
1748
|
+
builder: `text(${columnName(column.name)}) /* UNION ${unionLiteral} */`
|
|
1749
|
+
};
|
|
1750
|
+
}
|
|
1448
1751
|
if (upper === "INET") {
|
|
1449
1752
|
imports.local.add("duckDbInet");
|
|
1450
1753
|
return { builder: `duckDbInet(${columnName(column.name)})` };
|
|
@@ -1510,7 +1813,7 @@ function mapDuckDbType(column, imports, options) {
|
|
|
1510
1813
|
}
|
|
1511
1814
|
imports.pgCore.add("text");
|
|
1512
1815
|
return {
|
|
1513
|
-
builder: `text(${columnName(column.name)}) /*
|
|
1816
|
+
builder: `text(${columnName(column.name)}) /* unsupported DuckDB type: ${upper} */`
|
|
1514
1817
|
};
|
|
1515
1818
|
}
|
|
1516
1819
|
function parseStructFields(inner) {
|
|
@@ -1620,6 +1923,7 @@ function renderImports(imports, importBasePath) {
|
|
|
1620
1923
|
function parseArgs(argv) {
|
|
1621
1924
|
const options = {
|
|
1622
1925
|
outFile: path.resolve(process.cwd(), "drizzle/schema.ts"),
|
|
1926
|
+
outMeta: undefined,
|
|
1623
1927
|
allDatabases: false,
|
|
1624
1928
|
includeViews: false,
|
|
1625
1929
|
useCustomTimeTypes: true
|
|
@@ -1645,6 +1949,11 @@ function parseArgs(argv) {
|
|
|
1645
1949
|
case "--outFile":
|
|
1646
1950
|
options.outFile = path.resolve(process.cwd(), argv[++i] ?? "drizzle/schema.ts");
|
|
1647
1951
|
break;
|
|
1952
|
+
case "--out-json":
|
|
1953
|
+
case "--outJson":
|
|
1954
|
+
case "--json":
|
|
1955
|
+
options.outMeta = path.resolve(process.cwd(), argv[++i] ?? "drizzle/schema.meta.json");
|
|
1956
|
+
break;
|
|
1648
1957
|
case "--include-views":
|
|
1649
1958
|
case "--includeViews":
|
|
1650
1959
|
options.includeViews = true;
|
|
@@ -1679,6 +1988,7 @@ Options:
|
|
|
1679
1988
|
--all-databases Introspect all attached databases (not just current)
|
|
1680
1989
|
--schema Comma separated schema list (defaults to all non-system schemas)
|
|
1681
1990
|
--out Output file (default: ./drizzle/schema.ts)
|
|
1991
|
+
--json Optional JSON metadata output (default: ./drizzle/schema.meta.json)
|
|
1682
1992
|
--include-views Include views in the generated schema
|
|
1683
1993
|
--use-pg-time Use pg-core timestamp/date/time instead of DuckDB custom helpers
|
|
1684
1994
|
--import-base Override import path for duckdb helpers (default: package name)
|
|
@@ -1719,11 +2029,23 @@ async function main() {
|
|
|
1719
2029
|
});
|
|
1720
2030
|
await mkdir(path.dirname(options.outFile), { recursive: true });
|
|
1721
2031
|
await writeFile(options.outFile, result.files.schemaTs, "utf8");
|
|
2032
|
+
if (options.outMeta) {
|
|
2033
|
+
await mkdir(path.dirname(options.outMeta), { recursive: true });
|
|
2034
|
+
await writeFile(options.outMeta, JSON.stringify(result.files.metaJson, null, 2), "utf8");
|
|
2035
|
+
}
|
|
1722
2036
|
console.log(`Wrote schema to ${options.outFile}`);
|
|
2037
|
+
if (options.outMeta) {
|
|
2038
|
+
console.log(`Wrote metadata to ${options.outMeta}`);
|
|
2039
|
+
}
|
|
1723
2040
|
} finally {
|
|
1724
2041
|
if ("closeSync" in connection && typeof connection.closeSync === "function") {
|
|
1725
2042
|
connection.closeSync();
|
|
1726
2043
|
}
|
|
2044
|
+
if ("closeSync" in instance && typeof instance.closeSync === "function") {
|
|
2045
|
+
instance.closeSync();
|
|
2046
|
+
} else if ("close" in instance && typeof instance.close === "function") {
|
|
2047
|
+
await instance.close();
|
|
2048
|
+
}
|
|
1727
2049
|
}
|
|
1728
2050
|
}
|
|
1729
2051
|
main().catch((err) => {
|