@leonardovida-md/drizzle-neo-duckdb 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/columns.d.ts +5 -0
- package/dist/dialect.d.ts +21 -0
- package/dist/duckdb-introspect.mjs +414 -92
- package/dist/helpers.mjs +4 -4
- package/dist/index.mjs +416 -96
- package/dist/introspect.d.ts +8 -0
- package/dist/pool.d.ts +8 -0
- package/dist/session.d.ts +7 -1
- package/dist/sql/query-rewriters.d.ts +1 -0
- package/dist/sql/result-mapper.d.ts +7 -0
- package/dist/value-wrappers-core.d.ts +2 -2
- package/package.json +1 -1
- package/src/bin/duckdb-introspect.ts +27 -0
- package/src/client.ts +13 -10
- package/src/columns.ts +9 -9
- package/src/dialect.ts +51 -3
- package/src/driver.ts +15 -2
- package/src/introspect.ts +23 -6
- package/src/pool.ts +182 -12
- package/src/session.ts +98 -5
- package/src/sql/query-rewriters.ts +183 -75
- package/src/sql/result-mapper.ts +7 -7
- package/src/value-wrappers-core.ts +2 -2
- package/src/value-wrappers.ts +13 -3
package/dist/index.mjs
CHANGED
|
@@ -16,83 +16,159 @@ import { PgPreparedQuery, PgSession } from "drizzle-orm/pg-core/session";
|
|
|
16
16
|
import { fillPlaceholders, sql } from "drizzle-orm/sql/sql";
|
|
17
17
|
|
|
18
18
|
// src/sql/query-rewriters.ts
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
break;
|
|
37
|
-
if (ch === "'" && source[idx - 1] !== "\\") {
|
|
38
|
-
inString = !inString;
|
|
19
|
+
var OPERATORS = [
|
|
20
|
+
{ token: "@>", fn: "array_has_all" },
|
|
21
|
+
{ token: "<@", fn: "array_has_all", swap: true },
|
|
22
|
+
{ token: "&&", fn: "array_has_any" }
|
|
23
|
+
];
|
|
24
|
+
var isWhitespace = (char) => char !== undefined && /\s/.test(char);
|
|
25
|
+
function scrubForRewrite(query) {
|
|
26
|
+
let scrubbed = "";
|
|
27
|
+
let state = "code";
|
|
28
|
+
for (let i = 0;i < query.length; i += 1) {
|
|
29
|
+
const char = query[i];
|
|
30
|
+
const next = query[i + 1];
|
|
31
|
+
if (state === "code") {
|
|
32
|
+
if (char === "'") {
|
|
33
|
+
scrubbed += "'";
|
|
34
|
+
state = "single";
|
|
35
|
+
continue;
|
|
39
36
|
}
|
|
40
|
-
if (
|
|
37
|
+
if (char === '"') {
|
|
38
|
+
scrubbed += '"';
|
|
39
|
+
state = "double";
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (char === "-" && next === "-") {
|
|
43
|
+
scrubbed += " ";
|
|
44
|
+
i += 1;
|
|
45
|
+
state = "lineComment";
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if (char === "/" && next === "*") {
|
|
49
|
+
scrubbed += " ";
|
|
50
|
+
i += 1;
|
|
51
|
+
state = "blockComment";
|
|
41
52
|
continue;
|
|
42
|
-
if (ch === ")" || ch === "]") {
|
|
43
|
-
depth++;
|
|
44
|
-
} else if (ch === "(" || ch === "[") {
|
|
45
|
-
depth--;
|
|
46
|
-
if (depth < 0) {
|
|
47
|
-
return [idx + 1, source.slice(idx + 1, start + 1)];
|
|
48
|
-
}
|
|
49
|
-
} else if (depth === 0 && isWhitespace(ch)) {
|
|
50
|
-
return [idx + 1, source.slice(idx + 1, start + 1)];
|
|
51
53
|
}
|
|
54
|
+
scrubbed += char;
|
|
55
|
+
continue;
|
|
52
56
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
for (;idx < source.length; idx++) {
|
|
63
|
-
const ch = source[idx];
|
|
64
|
-
if (ch === undefined)
|
|
65
|
-
break;
|
|
66
|
-
if (ch === "'" && source[idx - 1] !== "\\") {
|
|
67
|
-
inString = !inString;
|
|
57
|
+
if (state === "single") {
|
|
58
|
+
if (char === "'" && next === "'") {
|
|
59
|
+
scrubbed += "''";
|
|
60
|
+
i += 1;
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
scrubbed += char === "'" ? "'" : ".";
|
|
64
|
+
if (char === "'") {
|
|
65
|
+
state = "code";
|
|
68
66
|
}
|
|
69
|
-
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (state === "double") {
|
|
70
|
+
if (char === '"' && next === '"') {
|
|
71
|
+
scrubbed += '""';
|
|
72
|
+
i += 1;
|
|
70
73
|
continue;
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
74
|
+
}
|
|
75
|
+
scrubbed += char === '"' ? '"' : ".";
|
|
76
|
+
if (char === '"') {
|
|
77
|
+
state = "code";
|
|
78
|
+
}
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (state === "lineComment") {
|
|
82
|
+
scrubbed += char === `
|
|
83
|
+
` ? `
|
|
84
|
+
` : " ";
|
|
85
|
+
if (char === `
|
|
86
|
+
`) {
|
|
87
|
+
state = "code";
|
|
88
|
+
}
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (state === "blockComment") {
|
|
92
|
+
if (char === "*" && next === "/") {
|
|
93
|
+
scrubbed += " ";
|
|
94
|
+
i += 1;
|
|
95
|
+
state = "code";
|
|
96
|
+
} else {
|
|
97
|
+
scrubbed += " ";
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return scrubbed;
|
|
102
|
+
}
|
|
103
|
+
function findNextOperator(scrubbed, start) {
|
|
104
|
+
for (let idx = start;idx < scrubbed.length; idx += 1) {
|
|
105
|
+
for (const operator of OPERATORS) {
|
|
106
|
+
if (scrubbed.startsWith(operator.token, idx)) {
|
|
107
|
+
return { index: idx, operator };
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
function walkLeft(source, scrubbed, start) {
|
|
114
|
+
let idx = start;
|
|
115
|
+
while (idx >= 0 && isWhitespace(scrubbed[idx])) {
|
|
116
|
+
idx -= 1;
|
|
117
|
+
}
|
|
118
|
+
let depth = 0;
|
|
119
|
+
for (;idx >= 0; idx -= 1) {
|
|
120
|
+
const ch = scrubbed[idx];
|
|
121
|
+
if (ch === ")" || ch === "]") {
|
|
122
|
+
depth += 1;
|
|
123
|
+
} else if (ch === "(" || ch === "[") {
|
|
124
|
+
if (depth === 0) {
|
|
125
|
+
return [idx + 1, source.slice(idx + 1, start + 1)];
|
|
126
|
+
}
|
|
127
|
+
depth = Math.max(0, depth - 1);
|
|
128
|
+
} else if (depth === 0 && isWhitespace(ch)) {
|
|
129
|
+
return [idx + 1, source.slice(idx + 1, start + 1)];
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return [0, source.slice(0, start + 1)];
|
|
133
|
+
}
|
|
134
|
+
function walkRight(source, scrubbed, start) {
|
|
135
|
+
let idx = start;
|
|
136
|
+
while (idx < scrubbed.length && isWhitespace(scrubbed[idx])) {
|
|
137
|
+
idx += 1;
|
|
138
|
+
}
|
|
139
|
+
let depth = 0;
|
|
140
|
+
for (;idx < scrubbed.length; idx += 1) {
|
|
141
|
+
const ch = scrubbed[idx];
|
|
142
|
+
if (ch === "(" || ch === "[") {
|
|
143
|
+
depth += 1;
|
|
144
|
+
} else if (ch === ")" || ch === "]") {
|
|
145
|
+
if (depth === 0) {
|
|
79
146
|
return [idx, source.slice(start, idx)];
|
|
80
147
|
}
|
|
148
|
+
depth = Math.max(0, depth - 1);
|
|
149
|
+
} else if (depth === 0 && isWhitespace(ch)) {
|
|
150
|
+
return [idx, source.slice(start, idx)];
|
|
81
151
|
}
|
|
82
|
-
|
|
83
|
-
|
|
152
|
+
}
|
|
153
|
+
return [scrubbed.length, source.slice(start)];
|
|
154
|
+
}
|
|
155
|
+
function adaptArrayOperators(query) {
|
|
84
156
|
let rewritten = query;
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
157
|
+
let scrubbed = scrubForRewrite(query);
|
|
158
|
+
let searchStart = 0;
|
|
159
|
+
while (true) {
|
|
160
|
+
const next = findNextOperator(scrubbed, searchStart);
|
|
161
|
+
if (!next)
|
|
162
|
+
break;
|
|
163
|
+
const { index, operator } = next;
|
|
164
|
+
const [leftStart, leftExpr] = walkLeft(rewritten, scrubbed, index - 1);
|
|
165
|
+
const [rightEnd, rightExpr] = walkRight(rewritten, scrubbed, index + operator.token.length);
|
|
166
|
+
const left = leftExpr.trim();
|
|
167
|
+
const right = rightExpr.trim();
|
|
168
|
+
const replacement = `${operator.fn}(${operator.swap ? right : left}, ${operator.swap ? left : right})`;
|
|
169
|
+
rewritten = rewritten.slice(0, leftStart) + replacement + rewritten.slice(rightEnd);
|
|
170
|
+
scrubbed = scrubForRewrite(rewritten);
|
|
171
|
+
searchStart = leftStart + replacement.length;
|
|
96
172
|
}
|
|
97
173
|
return rewritten;
|
|
98
174
|
}
|
|
@@ -386,6 +462,12 @@ function dateToMicros(value) {
|
|
|
386
462
|
if (value instanceof Date) {
|
|
387
463
|
return BigInt(value.getTime()) * 1000n;
|
|
388
464
|
}
|
|
465
|
+
if (typeof value === "bigint") {
|
|
466
|
+
return value;
|
|
467
|
+
}
|
|
468
|
+
if (typeof value === "number") {
|
|
469
|
+
return BigInt(Math.trunc(value)) * 1000n;
|
|
470
|
+
}
|
|
389
471
|
let normalized = value;
|
|
390
472
|
if (!value.includes("T") && value.includes(" ")) {
|
|
391
473
|
normalized = value.replace(" ", "T");
|
|
@@ -455,14 +537,17 @@ function parsePgArrayLiteral(value) {
|
|
|
455
537
|
}
|
|
456
538
|
function prepareParams(params, options = {}) {
|
|
457
539
|
return params.map((param) => {
|
|
458
|
-
if (typeof param === "string"
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
540
|
+
if (typeof param === "string") {
|
|
541
|
+
const trimmed = param.trim();
|
|
542
|
+
if (trimmed && isPgArrayLiteral(trimmed)) {
|
|
543
|
+
if (options.rejectStringArrayLiterals) {
|
|
544
|
+
throw new Error("Stringified array literals are not supported. Use duckDbList()/duckDbArray() or pass native arrays.");
|
|
545
|
+
}
|
|
546
|
+
if (options.warnOnStringArrayLiteral) {
|
|
547
|
+
options.warnOnStringArrayLiteral();
|
|
548
|
+
}
|
|
549
|
+
return parsePgArrayLiteral(trimmed);
|
|
464
550
|
}
|
|
465
|
-
return parsePgArrayLiteral(param);
|
|
466
551
|
}
|
|
467
552
|
return param;
|
|
468
553
|
});
|
|
@@ -580,6 +665,13 @@ async function executeArrowOnClient(client, query, params) {
|
|
|
580
665
|
}
|
|
581
666
|
|
|
582
667
|
// src/session.ts
|
|
668
|
+
function isSavepointSyntaxError(error) {
|
|
669
|
+
if (!(error instanceof Error) || !error.message) {
|
|
670
|
+
return false;
|
|
671
|
+
}
|
|
672
|
+
return error.message.toLowerCase().includes("savepoint") && error.message.toLowerCase().includes("syntax error");
|
|
673
|
+
}
|
|
674
|
+
|
|
583
675
|
class DuckDBPreparedQuery extends PgPreparedQuery {
|
|
584
676
|
client;
|
|
585
677
|
dialect;
|
|
@@ -644,6 +736,7 @@ class DuckDBSession extends PgSession {
|
|
|
644
736
|
rewriteArrays;
|
|
645
737
|
rejectStringArrayLiterals;
|
|
646
738
|
hasWarnedArrayLiteral = false;
|
|
739
|
+
rollbackOnly = false;
|
|
647
740
|
constructor(client, dialect, schema, options = {}) {
|
|
648
741
|
super(dialect);
|
|
649
742
|
this.client = client;
|
|
@@ -657,7 +750,15 @@ class DuckDBSession extends PgSession {
|
|
|
657
750
|
prepareQuery(query, fields, name, isResponseInArrayMode, customResultMapper) {
|
|
658
751
|
return new DuckDBPreparedQuery(this.client, this.dialect, query.sql, query.params, this.logger, fields, isResponseInArrayMode, customResultMapper, this.rewriteArrays, this.rejectStringArrayLiterals, this.rejectStringArrayLiterals ? undefined : this.warnOnStringArrayLiteral);
|
|
659
752
|
}
|
|
660
|
-
|
|
753
|
+
execute(query) {
|
|
754
|
+
this.dialect.resetPgJsonFlag();
|
|
755
|
+
return super.execute(query);
|
|
756
|
+
}
|
|
757
|
+
all(query) {
|
|
758
|
+
this.dialect.resetPgJsonFlag();
|
|
759
|
+
return super.all(query);
|
|
760
|
+
}
|
|
761
|
+
async transaction(transaction, config) {
|
|
661
762
|
let pinnedConnection;
|
|
662
763
|
let pool;
|
|
663
764
|
let clientForTx = this.client;
|
|
@@ -670,8 +771,15 @@ class DuckDBSession extends PgSession {
|
|
|
670
771
|
const tx = new DuckDBTransaction(this.dialect, session, this.schema);
|
|
671
772
|
try {
|
|
672
773
|
await tx.execute(sql`BEGIN TRANSACTION;`);
|
|
774
|
+
if (config) {
|
|
775
|
+
await tx.setTransaction(config);
|
|
776
|
+
}
|
|
673
777
|
try {
|
|
674
778
|
const result = await transaction(tx);
|
|
779
|
+
if (session.isRollbackOnly()) {
|
|
780
|
+
await tx.execute(sql`rollback`);
|
|
781
|
+
throw new TransactionRollbackError;
|
|
782
|
+
}
|
|
675
783
|
await tx.execute(sql`commit`);
|
|
676
784
|
return result;
|
|
677
785
|
} catch (error) {
|
|
@@ -693,7 +801,9 @@ class DuckDBSession extends PgSession {
|
|
|
693
801
|
query: ${query}`, []);
|
|
694
802
|
};
|
|
695
803
|
executeBatches(query, options = {}) {
|
|
804
|
+
this.dialect.resetPgJsonFlag();
|
|
696
805
|
const builtQuery = this.dialect.sqlToQuery(query);
|
|
806
|
+
this.dialect.assertNoPgJsonColumns();
|
|
697
807
|
const params = prepareParams(builtQuery.params, {
|
|
698
808
|
rejectStringArrayLiterals: this.rejectStringArrayLiterals,
|
|
699
809
|
warnOnStringArrayLiteral: this.rejectStringArrayLiterals ? undefined : () => this.warnOnStringArrayLiteral(builtQuery.sql)
|
|
@@ -706,7 +816,9 @@ query: ${query}`, []);
|
|
|
706
816
|
return executeInBatches(this.client, rewrittenQuery, params, options);
|
|
707
817
|
}
|
|
708
818
|
async executeArrow(query) {
|
|
819
|
+
this.dialect.resetPgJsonFlag();
|
|
709
820
|
const builtQuery = this.dialect.sqlToQuery(query);
|
|
821
|
+
this.dialect.assertNoPgJsonColumns();
|
|
710
822
|
const params = prepareParams(builtQuery.params, {
|
|
711
823
|
rejectStringArrayLiterals: this.rejectStringArrayLiterals,
|
|
712
824
|
warnOnStringArrayLiteral: this.rejectStringArrayLiterals ? undefined : () => this.warnOnStringArrayLiteral(builtQuery.sql)
|
|
@@ -718,6 +830,12 @@ query: ${query}`, []);
|
|
|
718
830
|
this.logger.logQuery(rewrittenQuery, params);
|
|
719
831
|
return executeArrowOnClient(this.client, rewrittenQuery, params);
|
|
720
832
|
}
|
|
833
|
+
markRollbackOnly() {
|
|
834
|
+
this.rollbackOnly = true;
|
|
835
|
+
}
|
|
836
|
+
isRollbackOnly() {
|
|
837
|
+
return this.rollbackOnly;
|
|
838
|
+
}
|
|
721
839
|
}
|
|
722
840
|
|
|
723
841
|
class DuckDBTransaction extends PgTransaction {
|
|
@@ -748,8 +866,46 @@ class DuckDBTransaction extends PgTransaction {
|
|
|
748
866
|
return this.session.executeArrow(query);
|
|
749
867
|
}
|
|
750
868
|
async transaction(transaction) {
|
|
751
|
-
const
|
|
752
|
-
|
|
869
|
+
const internals = this;
|
|
870
|
+
const savepoint = `drizzle_savepoint_${this.nestedIndex + 1}`;
|
|
871
|
+
const savepointSql = sql.raw(`savepoint ${savepoint}`);
|
|
872
|
+
const releaseSql = sql.raw(`release savepoint ${savepoint}`);
|
|
873
|
+
const rollbackSql = sql.raw(`rollback to savepoint ${savepoint}`);
|
|
874
|
+
const nestedTx = new DuckDBTransaction(internals.dialect, internals.session, this.schema, this.nestedIndex + 1);
|
|
875
|
+
if (internals.dialect.areSavepointsUnsupported()) {
|
|
876
|
+
return this.runNestedWithoutSavepoint(transaction, nestedTx, internals);
|
|
877
|
+
}
|
|
878
|
+
let createdSavepoint = false;
|
|
879
|
+
try {
|
|
880
|
+
await internals.session.execute(savepointSql);
|
|
881
|
+
internals.dialect.markSavepointsSupported();
|
|
882
|
+
createdSavepoint = true;
|
|
883
|
+
} catch (error) {
|
|
884
|
+
if (!isSavepointSyntaxError(error)) {
|
|
885
|
+
throw error;
|
|
886
|
+
}
|
|
887
|
+
internals.dialect.markSavepointsUnsupported();
|
|
888
|
+
return this.runNestedWithoutSavepoint(transaction, nestedTx, internals);
|
|
889
|
+
}
|
|
890
|
+
try {
|
|
891
|
+
const result = await transaction(nestedTx);
|
|
892
|
+
if (createdSavepoint) {
|
|
893
|
+
await internals.session.execute(releaseSql);
|
|
894
|
+
}
|
|
895
|
+
return result;
|
|
896
|
+
} catch (error) {
|
|
897
|
+
if (createdSavepoint) {
|
|
898
|
+
await internals.session.execute(rollbackSql);
|
|
899
|
+
}
|
|
900
|
+
internals.session.markRollbackOnly();
|
|
901
|
+
throw error;
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
runNestedWithoutSavepoint(transaction, nestedTx, internals) {
|
|
905
|
+
return transaction(nestedTx).catch((error) => {
|
|
906
|
+
internals.session.markRollbackOnly();
|
|
907
|
+
throw error;
|
|
908
|
+
});
|
|
753
909
|
}
|
|
754
910
|
}
|
|
755
911
|
var arrayLiteralWarning = "Received a stringified Postgres-style array literal. Use duckDbList()/duckDbArray() or pass native arrays instead. You can also set rejectStringArrayLiterals=true to throw.";
|
|
@@ -771,15 +927,30 @@ import {
|
|
|
771
927
|
import {
|
|
772
928
|
sql as sql2
|
|
773
929
|
} from "drizzle-orm";
|
|
774
|
-
|
|
775
930
|
class DuckDBDialect extends PgDialect {
|
|
776
931
|
static [entityKind2] = "DuckDBPgDialect";
|
|
777
932
|
hasPgJsonColumn = false;
|
|
933
|
+
savepointsSupported = 0 /* Unknown */;
|
|
934
|
+
resetPgJsonFlag() {
|
|
935
|
+
this.hasPgJsonColumn = false;
|
|
936
|
+
}
|
|
937
|
+
markPgJsonDetected() {
|
|
938
|
+
this.hasPgJsonColumn = true;
|
|
939
|
+
}
|
|
778
940
|
assertNoPgJsonColumns() {
|
|
779
941
|
if (this.hasPgJsonColumn) {
|
|
780
|
-
throw new Error("Pg JSON/JSONB columns are not supported in DuckDB. Replace them with duckDbJson() to use DuckDB
|
|
942
|
+
throw new Error("Pg JSON/JSONB columns are not supported in DuckDB. Replace them with duckDbJson() to use DuckDB's native JSON type.");
|
|
781
943
|
}
|
|
782
944
|
}
|
|
945
|
+
areSavepointsUnsupported() {
|
|
946
|
+
return this.savepointsSupported === 2 /* No */;
|
|
947
|
+
}
|
|
948
|
+
markSavepointsSupported() {
|
|
949
|
+
this.savepointsSupported = 1 /* Yes */;
|
|
950
|
+
}
|
|
951
|
+
markSavepointsUnsupported() {
|
|
952
|
+
this.savepointsSupported = 2 /* No */;
|
|
953
|
+
}
|
|
783
954
|
async migrate(migrations, session, config) {
|
|
784
955
|
const migrationConfig = typeof config === "string" ? { migrationsFolder: config } : config;
|
|
785
956
|
const migrationsSchema = migrationConfig.migrationsSchema ?? "drizzle";
|
|
@@ -816,8 +987,8 @@ class DuckDBDialect extends PgDialect {
|
|
|
816
987
|
}
|
|
817
988
|
prepareTyping(encoder) {
|
|
818
989
|
if (is2(encoder, PgJsonb) || is2(encoder, PgJson)) {
|
|
819
|
-
this.
|
|
820
|
-
|
|
990
|
+
this.markPgJsonDetected();
|
|
991
|
+
throw new Error("Pg JSON/JSONB columns are not supported in DuckDB. Replace them with duckDbJson() to use DuckDB's native JSON type.");
|
|
821
992
|
} else if (is2(encoder, PgNumeric)) {
|
|
822
993
|
return "decimal";
|
|
823
994
|
} else if (is2(encoder, PgTime2)) {
|
|
@@ -951,41 +1122,148 @@ function resolvePoolSize(pool) {
|
|
|
951
1122
|
}
|
|
952
1123
|
function createDuckDBConnectionPool(instance, options = {}) {
|
|
953
1124
|
const size = options.size && options.size > 0 ? options.size : 4;
|
|
1125
|
+
const acquireTimeout = options.acquireTimeout ?? 30000;
|
|
1126
|
+
const maxWaitingRequests = options.maxWaitingRequests ?? 100;
|
|
1127
|
+
const maxLifetimeMs = options.maxLifetimeMs;
|
|
1128
|
+
const idleTimeoutMs = options.idleTimeoutMs;
|
|
1129
|
+
const metadata = new WeakMap;
|
|
954
1130
|
const idle = [];
|
|
955
1131
|
const waiting = [];
|
|
956
1132
|
let total = 0;
|
|
957
1133
|
let closed = false;
|
|
1134
|
+
let pendingAcquires = 0;
|
|
1135
|
+
const shouldRecycle = (conn, now) => {
|
|
1136
|
+
if (maxLifetimeMs !== undefined && now - conn.createdAt >= maxLifetimeMs) {
|
|
1137
|
+
return true;
|
|
1138
|
+
}
|
|
1139
|
+
if (idleTimeoutMs !== undefined && now - conn.lastUsedAt >= idleTimeoutMs) {
|
|
1140
|
+
return true;
|
|
1141
|
+
}
|
|
1142
|
+
return false;
|
|
1143
|
+
};
|
|
958
1144
|
const acquire = async () => {
|
|
959
1145
|
if (closed) {
|
|
960
1146
|
throw new Error("DuckDB connection pool is closed");
|
|
961
1147
|
}
|
|
962
|
-
|
|
963
|
-
|
|
1148
|
+
while (idle.length > 0) {
|
|
1149
|
+
const pooled = idle.pop();
|
|
1150
|
+
const now = Date.now();
|
|
1151
|
+
if (shouldRecycle(pooled, now)) {
|
|
1152
|
+
await closeClientConnection(pooled.connection);
|
|
1153
|
+
total = Math.max(0, total - 1);
|
|
1154
|
+
metadata.delete(pooled.connection);
|
|
1155
|
+
continue;
|
|
1156
|
+
}
|
|
1157
|
+
pooled.lastUsedAt = now;
|
|
1158
|
+
metadata.set(pooled.connection, {
|
|
1159
|
+
createdAt: pooled.createdAt,
|
|
1160
|
+
lastUsedAt: pooled.lastUsedAt
|
|
1161
|
+
});
|
|
1162
|
+
return pooled.connection;
|
|
964
1163
|
}
|
|
965
1164
|
if (total < size) {
|
|
1165
|
+
pendingAcquires += 1;
|
|
966
1166
|
total += 1;
|
|
967
|
-
|
|
1167
|
+
try {
|
|
1168
|
+
const connection = await DuckDBConnection.create(instance);
|
|
1169
|
+
if (closed) {
|
|
1170
|
+
await closeClientConnection(connection);
|
|
1171
|
+
total -= 1;
|
|
1172
|
+
throw new Error("DuckDB connection pool is closed");
|
|
1173
|
+
}
|
|
1174
|
+
const now = Date.now();
|
|
1175
|
+
metadata.set(connection, { createdAt: now, lastUsedAt: now });
|
|
1176
|
+
return connection;
|
|
1177
|
+
} catch (error) {
|
|
1178
|
+
total -= 1;
|
|
1179
|
+
throw error;
|
|
1180
|
+
} finally {
|
|
1181
|
+
pendingAcquires -= 1;
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
if (waiting.length >= maxWaitingRequests) {
|
|
1185
|
+
throw new Error(`DuckDB connection pool queue is full (max ${maxWaitingRequests} waiting requests)`);
|
|
968
1186
|
}
|
|
969
|
-
return await new Promise((resolve) => {
|
|
970
|
-
|
|
1187
|
+
return await new Promise((resolve, reject) => {
|
|
1188
|
+
const timeoutId = setTimeout(() => {
|
|
1189
|
+
const idx = waiting.findIndex((w) => w.timeoutId === timeoutId);
|
|
1190
|
+
if (idx !== -1) {
|
|
1191
|
+
waiting.splice(idx, 1);
|
|
1192
|
+
}
|
|
1193
|
+
reject(new Error(`DuckDB connection pool acquire timeout after ${acquireTimeout}ms`));
|
|
1194
|
+
}, acquireTimeout);
|
|
1195
|
+
waiting.push({ resolve, reject, timeoutId });
|
|
971
1196
|
});
|
|
972
1197
|
};
|
|
973
1198
|
const release = async (connection) => {
|
|
1199
|
+
const waiter = waiting.shift();
|
|
1200
|
+
if (waiter) {
|
|
1201
|
+
clearTimeout(waiter.timeoutId);
|
|
1202
|
+
const now2 = Date.now();
|
|
1203
|
+
const meta = metadata.get(connection) ?? { createdAt: now2, lastUsedAt: now2 };
|
|
1204
|
+
const expired = maxLifetimeMs !== undefined && now2 - meta.createdAt >= maxLifetimeMs;
|
|
1205
|
+
if (closed) {
|
|
1206
|
+
await closeClientConnection(connection);
|
|
1207
|
+
total = Math.max(0, total - 1);
|
|
1208
|
+
metadata.delete(connection);
|
|
1209
|
+
waiter.reject(new Error("DuckDB connection pool is closed"));
|
|
1210
|
+
return;
|
|
1211
|
+
}
|
|
1212
|
+
if (expired) {
|
|
1213
|
+
await closeClientConnection(connection);
|
|
1214
|
+
total = Math.max(0, total - 1);
|
|
1215
|
+
metadata.delete(connection);
|
|
1216
|
+
try {
|
|
1217
|
+
const replacement = await acquire();
|
|
1218
|
+
waiter.resolve(replacement);
|
|
1219
|
+
} catch (error) {
|
|
1220
|
+
waiter.reject(error);
|
|
1221
|
+
}
|
|
1222
|
+
return;
|
|
1223
|
+
}
|
|
1224
|
+
meta.lastUsedAt = now2;
|
|
1225
|
+
metadata.set(connection, meta);
|
|
1226
|
+
waiter.resolve(connection);
|
|
1227
|
+
return;
|
|
1228
|
+
}
|
|
974
1229
|
if (closed) {
|
|
975
1230
|
await closeClientConnection(connection);
|
|
1231
|
+
metadata.delete(connection);
|
|
1232
|
+
total = Math.max(0, total - 1);
|
|
976
1233
|
return;
|
|
977
1234
|
}
|
|
978
|
-
const
|
|
979
|
-
|
|
980
|
-
|
|
1235
|
+
const now = Date.now();
|
|
1236
|
+
const existingMeta = metadata.get(connection) ?? { createdAt: now, lastUsedAt: now };
|
|
1237
|
+
existingMeta.lastUsedAt = now;
|
|
1238
|
+
metadata.set(connection, existingMeta);
|
|
1239
|
+
if (maxLifetimeMs !== undefined && now - existingMeta.createdAt >= maxLifetimeMs) {
|
|
1240
|
+
await closeClientConnection(connection);
|
|
1241
|
+
total -= 1;
|
|
1242
|
+
metadata.delete(connection);
|
|
981
1243
|
return;
|
|
982
1244
|
}
|
|
983
|
-
idle.push(
|
|
1245
|
+
idle.push({
|
|
1246
|
+
connection,
|
|
1247
|
+
createdAt: existingMeta.createdAt,
|
|
1248
|
+
lastUsedAt: existingMeta.lastUsedAt
|
|
1249
|
+
});
|
|
984
1250
|
};
|
|
985
1251
|
const close = async () => {
|
|
986
1252
|
closed = true;
|
|
1253
|
+
const waiters = waiting.splice(0, waiting.length);
|
|
1254
|
+
for (const waiter of waiters) {
|
|
1255
|
+
clearTimeout(waiter.timeoutId);
|
|
1256
|
+
waiter.reject(new Error("DuckDB connection pool is closed"));
|
|
1257
|
+
}
|
|
987
1258
|
const toClose = idle.splice(0, idle.length);
|
|
988
|
-
await Promise.
|
|
1259
|
+
await Promise.allSettled(toClose.map((item) => closeClientConnection(item.connection)));
|
|
1260
|
+
total = Math.max(0, total - toClose.length);
|
|
1261
|
+
toClose.forEach((item) => metadata.delete(item.connection));
|
|
1262
|
+
const maxWait = 5000;
|
|
1263
|
+
const start = Date.now();
|
|
1264
|
+
while (pendingAcquires > 0 && Date.now() - start < maxWait) {
|
|
1265
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
1266
|
+
}
|
|
989
1267
|
};
|
|
990
1268
|
return {
|
|
991
1269
|
acquire,
|
|
@@ -1093,6 +1371,17 @@ class DuckDBDatabase extends PgDatabase {
|
|
|
1093
1371
|
if (isPool(this.$client) && this.$client.close) {
|
|
1094
1372
|
await this.$client.close();
|
|
1095
1373
|
}
|
|
1374
|
+
if (!isPool(this.$client)) {
|
|
1375
|
+
await closeClientConnection(this.$client);
|
|
1376
|
+
}
|
|
1377
|
+
if (this.$instance) {
|
|
1378
|
+
const maybeClosable = this.$instance;
|
|
1379
|
+
if (typeof maybeClosable.close === "function") {
|
|
1380
|
+
await maybeClosable.close();
|
|
1381
|
+
} else if (typeof maybeClosable.closeSync === "function") {
|
|
1382
|
+
maybeClosable.closeSync();
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1096
1385
|
}
|
|
1097
1386
|
select(fields) {
|
|
1098
1387
|
const selectedFields = fields ? aliasFields(fields) : undefined;
|
|
@@ -1176,6 +1465,13 @@ function buildStructLiteral(value, schema) {
|
|
|
1176
1465
|
});
|
|
1177
1466
|
return sql4`struct_pack(${sql4.join(parts, sql4.raw(", "))})`;
|
|
1178
1467
|
}
|
|
1468
|
+
function buildMapLiteral(value, valueType) {
|
|
1469
|
+
const keys = Object.keys(value);
|
|
1470
|
+
const vals = Object.values(value);
|
|
1471
|
+
const keyList = buildListLiteral(keys, "TEXT");
|
|
1472
|
+
const valList = buildListLiteral(vals, valueType?.endsWith("[]") ? valueType.slice(0, -2) : valueType);
|
|
1473
|
+
return sql4`map(${keyList}, ${valList})`;
|
|
1474
|
+
}
|
|
1179
1475
|
var duckDbList = (name, elementType) => customType({
|
|
1180
1476
|
dataType() {
|
|
1181
1477
|
return `${elementType}[]`;
|
|
@@ -1189,11 +1485,11 @@ var duckDbList = (name, elementType) => customType({
|
|
|
1189
1485
|
}
|
|
1190
1486
|
if (typeof value === "string") {
|
|
1191
1487
|
const parsed = coerceArrayString(value);
|
|
1192
|
-
if (parsed) {
|
|
1488
|
+
if (parsed !== undefined) {
|
|
1193
1489
|
return parsed;
|
|
1194
1490
|
}
|
|
1195
1491
|
}
|
|
1196
|
-
return
|
|
1492
|
+
return value;
|
|
1197
1493
|
}
|
|
1198
1494
|
})(name);
|
|
1199
1495
|
var duckDbArray = (name, elementType, fixedLength) => customType({
|
|
@@ -1209,11 +1505,11 @@ var duckDbArray = (name, elementType, fixedLength) => customType({
|
|
|
1209
1505
|
}
|
|
1210
1506
|
if (typeof value === "string") {
|
|
1211
1507
|
const parsed = coerceArrayString(value);
|
|
1212
|
-
if (parsed) {
|
|
1508
|
+
if (parsed !== undefined) {
|
|
1213
1509
|
return parsed;
|
|
1214
1510
|
}
|
|
1215
1511
|
}
|
|
1216
|
-
return
|
|
1512
|
+
return value;
|
|
1217
1513
|
}
|
|
1218
1514
|
})(name);
|
|
1219
1515
|
var duckDbMap = (name, valueType) => customType({
|
|
@@ -1761,6 +2057,20 @@ function mapDuckDbType(column, imports, options) {
|
|
|
1761
2057
|
imports.pgCore.add("text");
|
|
1762
2058
|
return { builder: `text(${columnName(column.name)}) /* JSON */` };
|
|
1763
2059
|
}
|
|
2060
|
+
if (upper.startsWith("ENUM")) {
|
|
2061
|
+
imports.pgCore.add("text");
|
|
2062
|
+
const enumLiteral = raw.replace(/^ENUM\s*/i, "").trim();
|
|
2063
|
+
return {
|
|
2064
|
+
builder: `text(${columnName(column.name)}) /* ENUM ${enumLiteral} */`
|
|
2065
|
+
};
|
|
2066
|
+
}
|
|
2067
|
+
if (upper.startsWith("UNION")) {
|
|
2068
|
+
imports.pgCore.add("text");
|
|
2069
|
+
const unionLiteral = raw.replace(/^UNION\s*/i, "").trim();
|
|
2070
|
+
return {
|
|
2071
|
+
builder: `text(${columnName(column.name)}) /* UNION ${unionLiteral} */`
|
|
2072
|
+
};
|
|
2073
|
+
}
|
|
1764
2074
|
if (upper === "INET") {
|
|
1765
2075
|
imports.local.add("duckDbInet");
|
|
1766
2076
|
return { builder: `duckDbInet(${columnName(column.name)})` };
|
|
@@ -1826,7 +2136,7 @@ function mapDuckDbType(column, imports, options) {
|
|
|
1826
2136
|
}
|
|
1827
2137
|
imports.pgCore.add("text");
|
|
1828
2138
|
return {
|
|
1829
|
-
builder: `text(${columnName(column.name)}) /*
|
|
2139
|
+
builder: `text(${columnName(column.name)}) /* unsupported DuckDB type: ${upper} */`
|
|
1830
2140
|
};
|
|
1831
2141
|
}
|
|
1832
2142
|
function parseStructFields(inner) {
|
|
@@ -2049,13 +2359,17 @@ export {
|
|
|
2049
2359
|
wrapJson,
|
|
2050
2360
|
wrapBlob,
|
|
2051
2361
|
wrapArray,
|
|
2362
|
+
toIdentifier,
|
|
2052
2363
|
sumN,
|
|
2053
2364
|
sumDistinctN,
|
|
2365
|
+
splitTopLevel,
|
|
2054
2366
|
rowNumber,
|
|
2055
2367
|
resolvePoolSize,
|
|
2056
2368
|
rank,
|
|
2057
2369
|
prepareParams,
|
|
2058
2370
|
percentileCont,
|
|
2371
|
+
parseStructFields,
|
|
2372
|
+
parseMapValue,
|
|
2059
2373
|
olap,
|
|
2060
2374
|
migrate,
|
|
2061
2375
|
median,
|
|
@@ -2064,6 +2378,7 @@ export {
|
|
|
2064
2378
|
isPool,
|
|
2065
2379
|
isDuckDBWrapper,
|
|
2066
2380
|
introspect,
|
|
2381
|
+
formatLiteral,
|
|
2067
2382
|
executeOnClient,
|
|
2068
2383
|
executeInBatches,
|
|
2069
2384
|
executeArrowOnClient,
|
|
@@ -2085,7 +2400,12 @@ export {
|
|
|
2085
2400
|
denseRank,
|
|
2086
2401
|
createDuckDBConnectionPool,
|
|
2087
2402
|
countN,
|
|
2403
|
+
coerceArrayString,
|
|
2088
2404
|
closeClientConnection,
|
|
2405
|
+
buildStructLiteral,
|
|
2406
|
+
buildMapLiteral,
|
|
2407
|
+
buildListLiteral,
|
|
2408
|
+
buildDefault,
|
|
2089
2409
|
avgN,
|
|
2090
2410
|
anyValue,
|
|
2091
2411
|
POOL_PRESETS,
|