@leonardovida-md/drizzle-neo-duckdb 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -18
- package/dist/client.d.ts +19 -1
- package/dist/columns.d.ts +18 -10
- package/dist/driver.d.ts +37 -1
- package/dist/duckdb-introspect.mjs +382 -60
- package/dist/helpers.d.ts +1 -0
- package/dist/helpers.mjs +319 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.mjs +588 -72
- package/dist/introspect.d.ts +1 -0
- package/dist/olap.d.ts +46 -0
- package/dist/pool.d.ts +22 -0
- package/dist/session.d.ts +5 -0
- package/dist/sql/query-rewriters.d.ts +0 -1
- package/dist/utils.d.ts +1 -1
- package/dist/value-wrappers-core.d.ts +42 -0
- package/dist/value-wrappers.d.ts +8 -0
- package/package.json +12 -4
- package/src/bin/duckdb-introspect.ts +12 -3
- package/src/client.ts +178 -23
- package/src/columns.ts +65 -36
- package/src/dialect.ts +2 -2
- package/src/driver.ts +211 -13
- package/src/helpers.ts +18 -0
- package/src/index.ts +4 -0
- package/src/introspect.ts +39 -33
- package/src/migrator.ts +2 -4
- package/src/olap.ts +190 -0
- package/src/pool.ts +104 -0
- package/src/select-builder.ts +3 -7
- package/src/session.ts +123 -28
- package/src/sql/query-rewriters.ts +4 -54
- package/src/sql/result-mapper.ts +6 -6
- package/src/sql/selection.ts +2 -9
- package/src/utils.ts +1 -1
- package/src/value-wrappers-core.ts +156 -0
- package/src/value-wrappers.ts +155 -0
package/src/olap.ts
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { is } from 'drizzle-orm/entity';
|
|
2
|
+
import { sql, Subquery, type SQLWrapper } from 'drizzle-orm';
|
|
3
|
+
import type { AnyPgColumn, PgTable } from 'drizzle-orm/pg-core';
|
|
4
|
+
import type { PgViewBase } from 'drizzle-orm/pg-core/view-base';
|
|
5
|
+
import type { SelectedFields } from 'drizzle-orm/pg-core/query-builders';
|
|
6
|
+
import { SQL } from 'drizzle-orm/sql/sql';
|
|
7
|
+
import { Column, getTableName } from 'drizzle-orm';
|
|
8
|
+
import type { DuckDBDatabase } from './driver.ts';
|
|
9
|
+
|
|
10
|
+
export const countN = (expr: SQLWrapper = sql`*`) =>
|
|
11
|
+
sql<number>`count(${expr})`.mapWith(Number);
|
|
12
|
+
|
|
13
|
+
export const sumN = (expr: SQLWrapper) =>
|
|
14
|
+
sql<number>`sum(${expr})`.mapWith(Number);
|
|
15
|
+
|
|
16
|
+
export const avgN = (expr: SQLWrapper) =>
|
|
17
|
+
sql<number>`avg(${expr})`.mapWith(Number);
|
|
18
|
+
|
|
19
|
+
export const sumDistinctN = (expr: SQLWrapper) =>
|
|
20
|
+
sql<number>`sum(distinct ${expr})`.mapWith(Number);
|
|
21
|
+
|
|
22
|
+
export const percentileCont = (p: number, expr: SQLWrapper) =>
|
|
23
|
+
sql<number>`percentile_cont(${p}) within group (order by ${expr})`.mapWith(
|
|
24
|
+
Number
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
export const median = (expr: SQLWrapper) => percentileCont(0.5, expr);
|
|
28
|
+
|
|
29
|
+
export const anyValue = <T = unknown>(expr: SQLWrapper) =>
|
|
30
|
+
sql<T>`any_value(${expr})`;
|
|
31
|
+
|
|
32
|
+
type PartitionOrder =
|
|
33
|
+
| {
|
|
34
|
+
partitionBy?: SQLWrapper | SQLWrapper[];
|
|
35
|
+
orderBy?: SQLWrapper | SQLWrapper[];
|
|
36
|
+
}
|
|
37
|
+
| undefined;
|
|
38
|
+
|
|
39
|
+
function normalizeArray<T>(value?: T | T[]): T[] {
|
|
40
|
+
if (!value) return [];
|
|
41
|
+
return Array.isArray(value) ? value : [value];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function overClause(options?: PartitionOrder) {
|
|
45
|
+
const partitions = normalizeArray(options?.partitionBy);
|
|
46
|
+
const orders = normalizeArray(options?.orderBy);
|
|
47
|
+
|
|
48
|
+
const chunks: SQLWrapper[] = [];
|
|
49
|
+
|
|
50
|
+
if (partitions.length > 0) {
|
|
51
|
+
chunks.push(sql`partition by ${sql.join(partitions, sql`, `)}`);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
if (orders.length > 0) {
|
|
55
|
+
chunks.push(sql`order by ${sql.join(orders, sql`, `)}`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (chunks.length === 0) {
|
|
59
|
+
return sql``;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return sql`over (${sql.join(chunks, sql` `)})`;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export const rowNumber = (options?: PartitionOrder) =>
|
|
66
|
+
sql<number>`row_number() ${overClause(options)}`.mapWith(Number);
|
|
67
|
+
|
|
68
|
+
export const rank = (options?: PartitionOrder) =>
|
|
69
|
+
sql<number>`rank() ${overClause(options)}`.mapWith(Number);
|
|
70
|
+
|
|
71
|
+
export const denseRank = (options?: PartitionOrder) =>
|
|
72
|
+
sql<number>`dense_rank() ${overClause(options)}`.mapWith(Number);
|
|
73
|
+
|
|
74
|
+
export const lag = <T = unknown>(
|
|
75
|
+
expr: SQLWrapper,
|
|
76
|
+
offset = 1,
|
|
77
|
+
defaultValue?: SQLWrapper,
|
|
78
|
+
options?: PartitionOrder
|
|
79
|
+
) =>
|
|
80
|
+
defaultValue
|
|
81
|
+
? sql<T>`lag(${expr}, ${offset}, ${defaultValue}) ${overClause(options)}`
|
|
82
|
+
: sql<T>`lag(${expr}, ${offset}) ${overClause(options)}`;
|
|
83
|
+
|
|
84
|
+
export const lead = <T = unknown>(
|
|
85
|
+
expr: SQLWrapper,
|
|
86
|
+
offset = 1,
|
|
87
|
+
defaultValue?: SQLWrapper,
|
|
88
|
+
options?: PartitionOrder
|
|
89
|
+
) =>
|
|
90
|
+
defaultValue
|
|
91
|
+
? sql<T>`lead(${expr}, ${offset}, ${defaultValue}) ${overClause(options)}`
|
|
92
|
+
: sql<T>`lead(${expr}, ${offset}) ${overClause(options)}`;
|
|
93
|
+
|
|
94
|
+
type ValueExpr = SQL | SQL.Aliased | AnyPgColumn;
|
|
95
|
+
type GroupKey = ValueExpr;
|
|
96
|
+
type MeasureMap = Record<string, ValueExpr>;
|
|
97
|
+
type NonAggMap = Record<string, ValueExpr>;
|
|
98
|
+
|
|
99
|
+
function keyAlias(key: SQLWrapper, fallback: string): string {
|
|
100
|
+
if (is(key, SQL.Aliased)) {
|
|
101
|
+
return key.fieldAlias ?? fallback;
|
|
102
|
+
}
|
|
103
|
+
if (is(key, Column)) {
|
|
104
|
+
return `${getTableName(key.table)}.${key.name}`;
|
|
105
|
+
}
|
|
106
|
+
return fallback;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export class OlapBuilder {
|
|
110
|
+
private source?: PgTable | Subquery | PgViewBase | SQL;
|
|
111
|
+
private keys: GroupKey[] = [];
|
|
112
|
+
private measureMap: MeasureMap = {};
|
|
113
|
+
private nonAggregates: NonAggMap = {};
|
|
114
|
+
private wrapNonAggWithAnyValue = false;
|
|
115
|
+
private orderByClauses: ValueExpr[] = [];
|
|
116
|
+
|
|
117
|
+
constructor(private db: DuckDBDatabase) {}
|
|
118
|
+
|
|
119
|
+
from(source: PgTable | Subquery | PgViewBase | SQL): this {
|
|
120
|
+
this.source = source;
|
|
121
|
+
return this;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
groupBy(keys: GroupKey[]): this {
|
|
125
|
+
this.keys = keys;
|
|
126
|
+
return this;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
measures(measures: MeasureMap): this {
|
|
130
|
+
this.measureMap = measures;
|
|
131
|
+
return this;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
selectNonAggregates(
|
|
135
|
+
fields: NonAggMap,
|
|
136
|
+
options: { anyValue?: boolean } = {}
|
|
137
|
+
): this {
|
|
138
|
+
this.nonAggregates = fields;
|
|
139
|
+
this.wrapNonAggWithAnyValue = options.anyValue ?? false;
|
|
140
|
+
return this;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
orderBy(...clauses: ValueExpr[]): this {
|
|
144
|
+
this.orderByClauses = clauses;
|
|
145
|
+
return this;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
build() {
|
|
149
|
+
if (!this.source) {
|
|
150
|
+
throw new Error('olap: .from() is required');
|
|
151
|
+
}
|
|
152
|
+
if (this.keys.length === 0) {
|
|
153
|
+
throw new Error('olap: .groupBy() is required');
|
|
154
|
+
}
|
|
155
|
+
if (Object.keys(this.measureMap).length === 0) {
|
|
156
|
+
throw new Error('olap: .measures() is required');
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const selection: Record<string, ValueExpr> = {};
|
|
160
|
+
|
|
161
|
+
this.keys.forEach((key, idx) => {
|
|
162
|
+
const alias = keyAlias(key, `key_${idx}`);
|
|
163
|
+
selection[alias] = key;
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
Object.entries(this.nonAggregates).forEach(([alias, expr]) => {
|
|
167
|
+
selection[alias] = this.wrapNonAggWithAnyValue ? anyValue(expr) : expr;
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
Object.assign(selection, this.measureMap);
|
|
171
|
+
|
|
172
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Drizzle's query builder types don't allow reassignment after groupBy
|
|
173
|
+
let query: any = this.db
|
|
174
|
+
.select(selection as SelectedFields)
|
|
175
|
+
.from(this.source!)
|
|
176
|
+
.groupBy(...this.keys);
|
|
177
|
+
|
|
178
|
+
if (this.orderByClauses.length > 0) {
|
|
179
|
+
query = query.orderBy(...this.orderByClauses);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return query;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
run() {
|
|
186
|
+
return this.build();
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export const olap = (db: DuckDBDatabase) => new OlapBuilder(db);
|
package/src/pool.ts
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
|
|
2
|
+
import { closeClientConnection, type DuckDBConnectionPool } from './client.ts';
|
|
3
|
+
|
|
4
|
+
/** Pool size presets for different MotherDuck instance types */
|
|
5
|
+
export type PoolPreset =
|
|
6
|
+
| 'pulse'
|
|
7
|
+
| 'standard'
|
|
8
|
+
| 'jumbo'
|
|
9
|
+
| 'mega'
|
|
10
|
+
| 'giga'
|
|
11
|
+
| 'local'
|
|
12
|
+
| 'memory';
|
|
13
|
+
|
|
14
|
+
/** Pool sizes optimized for each MotherDuck instance type */
|
|
15
|
+
export const POOL_PRESETS: Record<PoolPreset, number> = {
|
|
16
|
+
pulse: 4, // Auto-scaling, ad-hoc analytics
|
|
17
|
+
standard: 6, // Balanced ETL/ELT workloads
|
|
18
|
+
jumbo: 8, // Complex queries, high-volume
|
|
19
|
+
mega: 12, // Large-scale transformations
|
|
20
|
+
giga: 16, // Maximum parallelism
|
|
21
|
+
local: 8, // Local DuckDB file
|
|
22
|
+
memory: 4, // In-memory testing
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
export interface DuckDBPoolConfig {
|
|
26
|
+
/** Maximum concurrent connections. Defaults to 4. */
|
|
27
|
+
size?: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Resolve pool configuration to a concrete size.
|
|
32
|
+
* Returns false if pooling is disabled.
|
|
33
|
+
*/
|
|
34
|
+
export function resolvePoolSize(
|
|
35
|
+
pool: DuckDBPoolConfig | PoolPreset | false | undefined
|
|
36
|
+
): number | false {
|
|
37
|
+
if (pool === false) return false;
|
|
38
|
+
if (pool === undefined) return 4;
|
|
39
|
+
if (typeof pool === 'string') return POOL_PRESETS[pool];
|
|
40
|
+
return pool.size ?? 4;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface DuckDBConnectionPoolOptions {
|
|
44
|
+
/** Maximum concurrent connections. Defaults to 4. */
|
|
45
|
+
size?: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function createDuckDBConnectionPool(
|
|
49
|
+
instance: DuckDBInstance,
|
|
50
|
+
options: DuckDBConnectionPoolOptions = {}
|
|
51
|
+
): DuckDBConnectionPool & { size: number } {
|
|
52
|
+
const size = options.size && options.size > 0 ? options.size : 4;
|
|
53
|
+
const idle: DuckDBConnection[] = [];
|
|
54
|
+
const waiting: Array<(conn: DuckDBConnection) => void> = [];
|
|
55
|
+
let total = 0;
|
|
56
|
+
let closed = false;
|
|
57
|
+
|
|
58
|
+
const acquire = async (): Promise<DuckDBConnection> => {
|
|
59
|
+
if (closed) {
|
|
60
|
+
throw new Error('DuckDB connection pool is closed');
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (idle.length > 0) {
|
|
64
|
+
return idle.pop() as DuckDBConnection;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (total < size) {
|
|
68
|
+
total += 1;
|
|
69
|
+
return await DuckDBConnection.create(instance);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return await new Promise((resolve) => {
|
|
73
|
+
waiting.push(resolve);
|
|
74
|
+
});
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const release = async (connection: DuckDBConnection): Promise<void> => {
|
|
78
|
+
if (closed) {
|
|
79
|
+
await closeClientConnection(connection);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const waiter = waiting.shift();
|
|
84
|
+
if (waiter) {
|
|
85
|
+
waiter(connection);
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
idle.push(connection);
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const close = async (): Promise<void> => {
|
|
93
|
+
closed = true;
|
|
94
|
+
const toClose = idle.splice(0, idle.length);
|
|
95
|
+
await Promise.all(toClose.map((conn) => closeClientConnection(conn)));
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
acquire,
|
|
100
|
+
release,
|
|
101
|
+
close,
|
|
102
|
+
size,
|
|
103
|
+
};
|
|
104
|
+
}
|
package/src/select-builder.ts
CHANGED
|
@@ -6,11 +6,7 @@ import {
|
|
|
6
6
|
type SelectedFields,
|
|
7
7
|
type TableLikeHasEmptySelection,
|
|
8
8
|
} from 'drizzle-orm/pg-core/query-builders';
|
|
9
|
-
import {
|
|
10
|
-
PgColumn,
|
|
11
|
-
PgTable,
|
|
12
|
-
type PgSession,
|
|
13
|
-
} from 'drizzle-orm/pg-core';
|
|
9
|
+
import { PgColumn, PgTable, type PgSession } from 'drizzle-orm/pg-core';
|
|
14
10
|
import { Subquery, ViewBaseConfig, type SQLWrapper } from 'drizzle-orm';
|
|
15
11
|
import { PgViewBase } from 'drizzle-orm/pg-core/view-base';
|
|
16
12
|
import type {
|
|
@@ -25,7 +21,7 @@ import { getTableColumns, type DrizzleTypeError } from 'drizzle-orm/utils';
|
|
|
25
21
|
interface PgViewBaseInternal<
|
|
26
22
|
TName extends string = string,
|
|
27
23
|
TExisting extends boolean = boolean,
|
|
28
|
-
TSelectedFields extends ColumnsSelection = ColumnsSelection
|
|
24
|
+
TSelectedFields extends ColumnsSelection = ColumnsSelection,
|
|
29
25
|
> extends PgViewBase<TName, TExisting, TSelectedFields> {
|
|
30
26
|
[ViewBaseConfig]?: {
|
|
31
27
|
selectedFields: SelectedFields;
|
|
@@ -34,7 +30,7 @@ interface PgViewBaseInternal<
|
|
|
34
30
|
|
|
35
31
|
export class DuckDBSelectBuilder<
|
|
36
32
|
TSelection extends SelectedFields | undefined,
|
|
37
|
-
TBuilderMode extends 'db' | 'qb' = 'db'
|
|
33
|
+
TBuilderMode extends 'db' | 'qb' = 'db',
|
|
38
34
|
> extends PgSelectBuilder<TSelection, TBuilderMode> {
|
|
39
35
|
private _fields: TSelection;
|
|
40
36
|
private _session: PgSession | undefined;
|
package/src/session.ts
CHANGED
|
@@ -19,13 +19,25 @@ import { adaptArrayOperators } from './sql/query-rewriters.ts';
|
|
|
19
19
|
import { mapResultRow } from './sql/result-mapper.ts';
|
|
20
20
|
import type { DuckDBDialect } from './dialect.ts';
|
|
21
21
|
import { TransactionRollbackError } from 'drizzle-orm/errors';
|
|
22
|
-
import type {
|
|
23
|
-
|
|
22
|
+
import type {
|
|
23
|
+
DuckDBClientLike,
|
|
24
|
+
DuckDBConnectionPool,
|
|
25
|
+
RowData,
|
|
26
|
+
} from './client.ts';
|
|
27
|
+
import {
|
|
28
|
+
executeArrowOnClient,
|
|
29
|
+
executeInBatches,
|
|
30
|
+
executeOnClient,
|
|
31
|
+
prepareParams,
|
|
32
|
+
type ExecuteInBatchesOptions,
|
|
33
|
+
} from './client.ts';
|
|
34
|
+
import { isPool } from './client.ts';
|
|
35
|
+
import type { DuckDBConnection } from '@duckdb/node-api';
|
|
24
36
|
|
|
25
37
|
export type { DuckDBClientLike, RowData } from './client.ts';
|
|
26
38
|
|
|
27
39
|
export class DuckDBPreparedQuery<
|
|
28
|
-
T extends PreparedQueryConfig
|
|
40
|
+
T extends PreparedQueryConfig,
|
|
29
41
|
> extends PgPreparedQuery<T> {
|
|
30
42
|
static readonly [entityKind]: string = 'DuckDBPreparedQuery';
|
|
31
43
|
|
|
@@ -37,7 +49,9 @@ export class DuckDBPreparedQuery<
|
|
|
37
49
|
private logger: Logger,
|
|
38
50
|
private fields: SelectedFieldsOrdered | undefined,
|
|
39
51
|
private _isResponseInArrayMode: boolean,
|
|
40
|
-
private customResultMapper:
|
|
52
|
+
private customResultMapper:
|
|
53
|
+
| ((rows: unknown[][]) => T['execute'])
|
|
54
|
+
| undefined,
|
|
41
55
|
private rewriteArrays: boolean,
|
|
42
56
|
private rejectStringArrayLiterals: boolean,
|
|
43
57
|
private warnOnStringArrayLiteral?: (sql: string) => void
|
|
@@ -71,17 +85,10 @@ export class DuckDBPreparedQuery<
|
|
|
71
85
|
|
|
72
86
|
this.logger.logQuery(rewrittenQuery, params);
|
|
73
87
|
|
|
74
|
-
const {
|
|
75
|
-
|
|
76
|
-
joinsNotNullableMap,
|
|
77
|
-
customResultMapper,
|
|
78
|
-
} = this as typeof this & { joinsNotNullableMap?: Record<string, boolean> };
|
|
88
|
+
const { fields, joinsNotNullableMap, customResultMapper } =
|
|
89
|
+
this as typeof this & { joinsNotNullableMap?: Record<string, boolean> };
|
|
79
90
|
|
|
80
|
-
const rows = await executeOnClient(
|
|
81
|
-
this.client,
|
|
82
|
-
rewrittenQuery,
|
|
83
|
-
params
|
|
84
|
-
);
|
|
91
|
+
const rows = await executeOnClient(this.client, rewrittenQuery, params);
|
|
85
92
|
|
|
86
93
|
if (rows.length === 0 || !fields) {
|
|
87
94
|
return rows as T['execute'];
|
|
@@ -115,7 +122,7 @@ export interface DuckDBSessionOptions {
|
|
|
115
122
|
|
|
116
123
|
export class DuckDBSession<
|
|
117
124
|
TFullSchema extends Record<string, unknown> = Record<string, never>,
|
|
118
|
-
TSchema extends TablesRelationalConfig = Record<string, never
|
|
125
|
+
TSchema extends TablesRelationalConfig = Record<string, never>,
|
|
119
126
|
> extends PgSession<DuckDBQueryResultHKT, TFullSchema, TSchema> {
|
|
120
127
|
static readonly [entityKind]: string = 'DuckDBSession';
|
|
121
128
|
|
|
@@ -164,8 +171,18 @@ export class DuckDBSession<
|
|
|
164
171
|
override async transaction<T>(
|
|
165
172
|
transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>
|
|
166
173
|
): Promise<T> {
|
|
174
|
+
let pinnedConnection: DuckDBConnection | undefined;
|
|
175
|
+
let pool: DuckDBConnectionPool | undefined;
|
|
176
|
+
|
|
177
|
+
let clientForTx: DuckDBClientLike = this.client;
|
|
178
|
+
if (isPool(this.client)) {
|
|
179
|
+
pool = this.client;
|
|
180
|
+
pinnedConnection = await pool.acquire();
|
|
181
|
+
clientForTx = pinnedConnection;
|
|
182
|
+
}
|
|
183
|
+
|
|
167
184
|
const session = new DuckDBSession(
|
|
168
|
-
|
|
185
|
+
clientForTx,
|
|
169
186
|
this.dialect,
|
|
170
187
|
this.schema,
|
|
171
188
|
this.options
|
|
@@ -177,15 +194,21 @@ export class DuckDBSession<
|
|
|
177
194
|
this.schema
|
|
178
195
|
);
|
|
179
196
|
|
|
180
|
-
await tx.execute(sql`BEGIN TRANSACTION;`);
|
|
181
|
-
|
|
182
197
|
try {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
198
|
+
await tx.execute(sql`BEGIN TRANSACTION;`);
|
|
199
|
+
|
|
200
|
+
try {
|
|
201
|
+
const result = await transaction(tx);
|
|
202
|
+
await tx.execute(sql`commit`);
|
|
203
|
+
return result;
|
|
204
|
+
} catch (error) {
|
|
205
|
+
await tx.execute(sql`rollback`);
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
} finally {
|
|
209
|
+
if (pinnedConnection && pool) {
|
|
210
|
+
await pool.release(pinnedConnection);
|
|
211
|
+
}
|
|
189
212
|
}
|
|
190
213
|
}
|
|
191
214
|
|
|
@@ -199,11 +222,67 @@ export class DuckDBSession<
|
|
|
199
222
|
[]
|
|
200
223
|
);
|
|
201
224
|
};
|
|
225
|
+
|
|
226
|
+
executeBatches<T extends RowData = RowData>(
|
|
227
|
+
query: SQL,
|
|
228
|
+
options: ExecuteInBatchesOptions = {}
|
|
229
|
+
): AsyncGenerator<GenericRowData<T>[], void, void> {
|
|
230
|
+
const builtQuery = this.dialect.sqlToQuery(query);
|
|
231
|
+
const params = prepareParams(builtQuery.params, {
|
|
232
|
+
rejectStringArrayLiterals: this.rejectStringArrayLiterals,
|
|
233
|
+
warnOnStringArrayLiteral: this.rejectStringArrayLiterals
|
|
234
|
+
? undefined
|
|
235
|
+
: () => this.warnOnStringArrayLiteral(builtQuery.sql),
|
|
236
|
+
});
|
|
237
|
+
const rewrittenQuery = this.rewriteArrays
|
|
238
|
+
? adaptArrayOperators(builtQuery.sql)
|
|
239
|
+
: builtQuery.sql;
|
|
240
|
+
|
|
241
|
+
if (this.rewriteArrays && rewrittenQuery !== builtQuery.sql) {
|
|
242
|
+
this.logger.logQuery(
|
|
243
|
+
`[duckdb] original query before array rewrite: ${builtQuery.sql}`,
|
|
244
|
+
params
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
this.logger.logQuery(rewrittenQuery, params);
|
|
249
|
+
|
|
250
|
+
return executeInBatches(
|
|
251
|
+
this.client,
|
|
252
|
+
rewrittenQuery,
|
|
253
|
+
params,
|
|
254
|
+
options
|
|
255
|
+
) as AsyncGenerator<GenericRowData<T>[], void, void>;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async executeArrow(query: SQL): Promise<unknown> {
|
|
259
|
+
const builtQuery = this.dialect.sqlToQuery(query);
|
|
260
|
+
const params = prepareParams(builtQuery.params, {
|
|
261
|
+
rejectStringArrayLiterals: this.rejectStringArrayLiterals,
|
|
262
|
+
warnOnStringArrayLiteral: this.rejectStringArrayLiterals
|
|
263
|
+
? undefined
|
|
264
|
+
: () => this.warnOnStringArrayLiteral(builtQuery.sql),
|
|
265
|
+
});
|
|
266
|
+
const rewrittenQuery = this.rewriteArrays
|
|
267
|
+
? adaptArrayOperators(builtQuery.sql)
|
|
268
|
+
: builtQuery.sql;
|
|
269
|
+
|
|
270
|
+
if (this.rewriteArrays && rewrittenQuery !== builtQuery.sql) {
|
|
271
|
+
this.logger.logQuery(
|
|
272
|
+
`[duckdb] original query before array rewrite: ${builtQuery.sql}`,
|
|
273
|
+
params
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
this.logger.logQuery(rewrittenQuery, params);
|
|
278
|
+
|
|
279
|
+
return executeArrowOnClient(this.client, rewrittenQuery, params);
|
|
280
|
+
}
|
|
202
281
|
}
|
|
203
282
|
|
|
204
283
|
type PgTransactionInternals<
|
|
205
284
|
TFullSchema extends Record<string, unknown> = Record<string, never>,
|
|
206
|
-
TSchema extends TablesRelationalConfig = Record<string, never
|
|
285
|
+
TSchema extends TablesRelationalConfig = Record<string, never>,
|
|
207
286
|
> = {
|
|
208
287
|
dialect: DuckDBDialect;
|
|
209
288
|
session: DuckDBSession<TFullSchema, TSchema>;
|
|
@@ -211,13 +290,13 @@ type PgTransactionInternals<
|
|
|
211
290
|
|
|
212
291
|
type DuckDBTransactionWithInternals<
|
|
213
292
|
TFullSchema extends Record<string, unknown> = Record<string, never>,
|
|
214
|
-
TSchema extends TablesRelationalConfig = Record<string, never
|
|
293
|
+
TSchema extends TablesRelationalConfig = Record<string, never>,
|
|
215
294
|
> = PgTransactionInternals<TFullSchema, TSchema> &
|
|
216
295
|
DuckDBTransaction<TFullSchema, TSchema>;
|
|
217
296
|
|
|
218
297
|
export class DuckDBTransaction<
|
|
219
298
|
TFullSchema extends Record<string, unknown>,
|
|
220
|
-
TSchema extends TablesRelationalConfig
|
|
299
|
+
TSchema extends TablesRelationalConfig,
|
|
221
300
|
> extends PgTransaction<DuckDBQueryResultHKT, TFullSchema, TSchema> {
|
|
222
301
|
static readonly [entityKind]: string = 'DuckDBTransaction';
|
|
223
302
|
|
|
@@ -240,15 +319,32 @@ export class DuckDBTransaction<
|
|
|
240
319
|
}
|
|
241
320
|
|
|
242
321
|
setTransaction(config: PgTransactionConfig): Promise<void> {
|
|
322
|
+
// Cast needed: PgTransaction doesn't expose dialect/session properties in public API
|
|
243
323
|
type Tx = DuckDBTransactionWithInternals<TFullSchema, TSchema>;
|
|
244
324
|
return (this as unknown as Tx).session.execute(
|
|
245
325
|
sql`set transaction ${this.getTransactionConfigSQL(config)}`
|
|
246
326
|
);
|
|
247
327
|
}
|
|
248
328
|
|
|
329
|
+
executeBatches<T extends RowData = RowData>(
|
|
330
|
+
query: SQL,
|
|
331
|
+
options: ExecuteInBatchesOptions = {}
|
|
332
|
+
): AsyncGenerator<GenericRowData<T>[], void, void> {
|
|
333
|
+
// Cast needed: PgTransaction doesn't expose session property in public API
|
|
334
|
+
type Tx = DuckDBTransactionWithInternals<TFullSchema, TSchema>;
|
|
335
|
+
return (this as unknown as Tx).session.executeBatches<T>(query, options);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
executeArrow(query: SQL): Promise<unknown> {
|
|
339
|
+
// Cast needed: PgTransaction doesn't expose session property in public API
|
|
340
|
+
type Tx = DuckDBTransactionWithInternals<TFullSchema, TSchema>;
|
|
341
|
+
return (this as unknown as Tx).session.executeArrow(query);
|
|
342
|
+
}
|
|
343
|
+
|
|
249
344
|
override async transaction<T>(
|
|
250
345
|
transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>
|
|
251
346
|
): Promise<T> {
|
|
347
|
+
// Cast needed: PgTransaction doesn't expose dialect/session properties in public API
|
|
252
348
|
type Tx = DuckDBTransactionWithInternals<TFullSchema, TSchema>;
|
|
253
349
|
const nestedTx = new DuckDBTransaction<TFullSchema, TSchema>(
|
|
254
350
|
(this as unknown as Tx).dialect,
|
|
@@ -268,7 +364,6 @@ export type GenericTableData<T = RowData> = T[];
|
|
|
268
364
|
const arrayLiteralWarning =
|
|
269
365
|
'Received a stringified Postgres-style array literal. Use duckDbList()/duckDbArray() or pass native arrays instead. You can also set rejectStringArrayLiterals=true to throw.';
|
|
270
366
|
|
|
271
|
-
|
|
272
367
|
export interface DuckDBQueryResultHKT extends PgQueryResultHKT {
|
|
273
368
|
type: GenericTableData<Assume<this['row'], RowData>>;
|
|
274
369
|
}
|
|
@@ -1,13 +1,3 @@
|
|
|
1
|
-
const selectionRegex = /select\s+(.+)\s+from/i;
|
|
2
|
-
const tableIdPropSelectionRegex = new RegExp(
|
|
3
|
-
[
|
|
4
|
-
`("(.+)"\\."(.+)")`, // table identifier + property
|
|
5
|
-
`(\\s+as\\s+'?(.+?)'?\\.'?(.+?)'?)?`, // optional AS clause
|
|
6
|
-
].join(''),
|
|
7
|
-
'i'
|
|
8
|
-
);
|
|
9
|
-
const noTableIdPropSelectionRegex = /"(.+)"(\s+as\s+'?\1'?)?/i;
|
|
10
|
-
|
|
11
1
|
export function adaptArrayOperators(query: string): string {
|
|
12
2
|
type ArrayOperator = {
|
|
13
3
|
token: '@>' | '<@' | '&&';
|
|
@@ -34,6 +24,7 @@ export function adaptArrayOperators(query: string): string {
|
|
|
34
24
|
let inString = false;
|
|
35
25
|
for (; idx >= 0; idx--) {
|
|
36
26
|
const ch = source[idx];
|
|
27
|
+
if (ch === undefined) break;
|
|
37
28
|
if (ch === "'" && source[idx - 1] !== '\\') {
|
|
38
29
|
inString = !inString;
|
|
39
30
|
}
|
|
@@ -62,6 +53,7 @@ export function adaptArrayOperators(query: string): string {
|
|
|
62
53
|
let inString = false;
|
|
63
54
|
for (; idx < source.length; idx++) {
|
|
64
55
|
const ch = source[idx];
|
|
56
|
+
if (ch === undefined) break;
|
|
65
57
|
if (ch === "'" && source[idx - 1] !== '\\') {
|
|
66
58
|
inString = !inString;
|
|
67
59
|
}
|
|
@@ -85,10 +77,7 @@ export function adaptArrayOperators(query: string): string {
|
|
|
85
77
|
let idx = rewritten.indexOf(token);
|
|
86
78
|
while (idx !== -1) {
|
|
87
79
|
const [leftStart, leftExpr] = walkLeft(rewritten, idx - 1);
|
|
88
|
-
const [rightEnd, rightExpr] = walkRight(
|
|
89
|
-
rewritten,
|
|
90
|
-
idx + token.length
|
|
91
|
-
);
|
|
80
|
+
const [rightEnd, rightExpr] = walkRight(rewritten, idx + token.length);
|
|
92
81
|
|
|
93
82
|
const left = leftExpr.trim();
|
|
94
83
|
const right = rightExpr.trim();
|
|
@@ -98,9 +87,7 @@ export function adaptArrayOperators(query: string): string {
|
|
|
98
87
|
})`;
|
|
99
88
|
|
|
100
89
|
rewritten =
|
|
101
|
-
rewritten.slice(0, leftStart) +
|
|
102
|
-
replacement +
|
|
103
|
-
rewritten.slice(rightEnd);
|
|
90
|
+
rewritten.slice(0, leftStart) + replacement + rewritten.slice(rightEnd);
|
|
104
91
|
|
|
105
92
|
idx = rewritten.indexOf(token, leftStart + replacement.length);
|
|
106
93
|
}
|
|
@@ -108,40 +95,3 @@ export function adaptArrayOperators(query: string): string {
|
|
|
108
95
|
|
|
109
96
|
return rewritten;
|
|
110
97
|
}
|
|
111
|
-
|
|
112
|
-
export function queryAdapter(query: string): string {
|
|
113
|
-
const selection = selectionRegex.exec(query);
|
|
114
|
-
|
|
115
|
-
if (selection?.length !== 2) {
|
|
116
|
-
return query;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
const fields = selection[1]
|
|
120
|
-
.split(',')
|
|
121
|
-
.map((field) => {
|
|
122
|
-
const trimmedField = field.trim();
|
|
123
|
-
const tableProp = tableIdPropSelectionRegex.exec(trimmedField);
|
|
124
|
-
if (tableProp) {
|
|
125
|
-
const [, identifier, table, column, , aliasTable, aliasColumn] =
|
|
126
|
-
tableProp;
|
|
127
|
-
|
|
128
|
-
const asAlias = `'${aliasTable ?? table}.${aliasColumn ?? column}'`;
|
|
129
|
-
if (tableProp[4]) {
|
|
130
|
-
return trimmedField.replace(tableProp[4], ` as ${asAlias}`);
|
|
131
|
-
}
|
|
132
|
-
return `${identifier} as ${asAlias}`;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const noTableProp = noTableIdPropSelectionRegex.exec(trimmedField);
|
|
136
|
-
if (noTableProp) {
|
|
137
|
-
const [, column, alias] = noTableProp;
|
|
138
|
-
const asAlias = ` as '${column}'`;
|
|
139
|
-
return alias ? trimmedField.replace(alias, asAlias) : `${trimmedField}${asAlias}`;
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
return trimmedField;
|
|
143
|
-
})
|
|
144
|
-
.filter(Boolean) as string[];
|
|
145
|
-
|
|
146
|
-
return query.replace(selection[1], fields.join(', '));
|
|
147
|
-
}
|
package/src/sql/result-mapper.ts
CHANGED
|
@@ -88,7 +88,10 @@ function normalizeTimestampString(
|
|
|
88
88
|
return value;
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
-
function normalizeTimestamp(
|
|
91
|
+
function normalizeTimestamp(
|
|
92
|
+
value: unknown,
|
|
93
|
+
withTimezone: boolean
|
|
94
|
+
): Date | unknown {
|
|
92
95
|
if (value instanceof Date) {
|
|
93
96
|
return value;
|
|
94
97
|
}
|
|
@@ -96,8 +99,7 @@ function normalizeTimestamp(value: unknown, withTimezone: boolean): Date | unkno
|
|
|
96
99
|
const hasOffset =
|
|
97
100
|
value.endsWith('Z') || /[+-]\d{2}:?\d{2}$/.test(value.trim());
|
|
98
101
|
const spaced = value.replace(' ', 'T');
|
|
99
|
-
const normalized =
|
|
100
|
-
withTimezone || hasOffset ? spaced : `${spaced}+00`;
|
|
102
|
+
const normalized = withTimezone || hasOffset ? spaced : `${spaced}+00`;
|
|
101
103
|
return new Date(normalized);
|
|
102
104
|
}
|
|
103
105
|
return value;
|
|
@@ -177,9 +179,7 @@ function mapDriverValue(
|
|
|
177
179
|
if (normalized instanceof Date) {
|
|
178
180
|
return normalized;
|
|
179
181
|
}
|
|
180
|
-
return decoder.mapFromDriverValue(
|
|
181
|
-
toDecoderInput(decoder, normalized)
|
|
182
|
-
);
|
|
182
|
+
return decoder.mapFromDriverValue(toDecoderInput(decoder, normalized));
|
|
183
183
|
}
|
|
184
184
|
|
|
185
185
|
if (is(decoder, PgDateString)) {
|