@leonardovida-md/drizzle-neo-duckdb 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -85
- package/dist/client.d.ts +15 -1
- package/dist/columns.d.ts +8 -2
- package/dist/dialect.d.ts +21 -0
- package/dist/driver.d.ts +7 -3
- package/dist/duckdb-introspect.mjs +667 -133
- package/dist/helpers.mjs +35 -4
- package/dist/index.d.ts +1 -0
- package/dist/index.mjs +695 -137
- package/dist/introspect.d.ts +8 -0
- package/dist/options.d.ts +10 -0
- package/dist/pool.d.ts +8 -0
- package/dist/session.d.ts +18 -6
- package/dist/sql/query-rewriters.d.ts +1 -0
- package/dist/sql/result-mapper.d.ts +7 -0
- package/dist/value-wrappers-core.d.ts +2 -2
- package/package.json +1 -1
- package/src/bin/duckdb-introspect.ts +27 -0
- package/src/client.ts +301 -38
- package/src/columns.ts +60 -13
- package/src/dialect.ts +51 -3
- package/src/driver.ts +45 -7
- package/src/index.ts +1 -0
- package/src/introspect.ts +23 -6
- package/src/options.ts +40 -0
- package/src/pool.ts +182 -12
- package/src/session.ts +206 -31
- package/src/sql/query-rewriters.ts +191 -75
- package/src/sql/result-mapper.ts +7 -7
- package/src/value-wrappers-core.ts +2 -2
- package/src/value-wrappers.ts +13 -3
package/dist/introspect.d.ts
CHANGED
|
@@ -63,4 +63,12 @@ export interface IntrospectResult {
|
|
|
63
63
|
}
|
|
64
64
|
export declare const DEFAULT_IMPORT_BASE = "@leonardovida-md/drizzle-neo-duckdb/helpers";
|
|
65
65
|
export declare function introspect(db: DuckDBDatabase, opts?: IntrospectOptions): Promise<IntrospectResult>;
|
|
66
|
+
export declare function buildDefault(defaultValue: string | null): string;
|
|
67
|
+
export declare function parseStructFields(inner: string): Array<{
|
|
68
|
+
name: string;
|
|
69
|
+
type: string;
|
|
70
|
+
}>;
|
|
71
|
+
export declare function parseMapValue(raw: string): string;
|
|
72
|
+
export declare function splitTopLevel(input: string, delimiter: string): string[];
|
|
73
|
+
export declare function toIdentifier(name: string): string;
|
|
66
74
|
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export type RewriteArraysMode = 'auto' | 'always' | 'never';
|
|
2
|
+
export type RewriteArraysOption = boolean | RewriteArraysMode;
|
|
3
|
+
export declare function resolveRewriteArraysOption(value?: RewriteArraysOption): RewriteArraysMode;
|
|
4
|
+
export type PrepareCacheOption = boolean | number | {
|
|
5
|
+
size?: number;
|
|
6
|
+
};
|
|
7
|
+
export interface PreparedStatementCacheConfig {
|
|
8
|
+
size: number;
|
|
9
|
+
}
|
|
10
|
+
export declare function resolvePrepareCacheOption(option?: PrepareCacheOption): PreparedStatementCacheConfig | undefined;
|
package/dist/pool.d.ts
CHANGED
|
@@ -16,6 +16,14 @@ export declare function resolvePoolSize(pool: DuckDBPoolConfig | PoolPreset | fa
|
|
|
16
16
|
export interface DuckDBConnectionPoolOptions {
|
|
17
17
|
/** Maximum concurrent connections. Defaults to 4. */
|
|
18
18
|
size?: number;
|
|
19
|
+
/** Timeout in milliseconds to wait for a connection. Defaults to 30000 (30s). */
|
|
20
|
+
acquireTimeout?: number;
|
|
21
|
+
/** Maximum number of requests waiting for a connection. Defaults to 100. */
|
|
22
|
+
maxWaitingRequests?: number;
|
|
23
|
+
/** Max time (ms) a connection may live before being recycled. */
|
|
24
|
+
maxLifetimeMs?: number;
|
|
25
|
+
/** Max idle time (ms) before an idle connection is discarded. */
|
|
26
|
+
idleTimeoutMs?: number;
|
|
19
27
|
}
|
|
20
28
|
export declare function createDuckDBConnectionPool(instance: DuckDBInstance, options?: DuckDBConnectionPoolOptions): DuckDBConnectionPool & {
|
|
21
29
|
size: number;
|
package/dist/session.d.ts
CHANGED
|
@@ -9,7 +9,8 @@ import { type Query, SQL } from 'drizzle-orm/sql/sql';
|
|
|
9
9
|
import type { Assume } from 'drizzle-orm/utils';
|
|
10
10
|
import type { DuckDBDialect } from './dialect.ts';
|
|
11
11
|
import type { DuckDBClientLike, RowData } from './client.ts';
|
|
12
|
-
import { type ExecuteInBatchesOptions } from './client.ts';
|
|
12
|
+
import { type ExecuteBatchesRawChunk, type ExecuteInBatchesOptions } from './client.ts';
|
|
13
|
+
import type { PreparedStatementCacheConfig, RewriteArraysMode } from './options.ts';
|
|
13
14
|
export type { DuckDBClientLike, RowData } from './client.ts';
|
|
14
15
|
export declare class DuckDBPreparedQuery<T extends PreparedQueryConfig> extends PgPreparedQuery<T> {
|
|
15
16
|
private client;
|
|
@@ -20,19 +21,21 @@ export declare class DuckDBPreparedQuery<T extends PreparedQueryConfig> extends
|
|
|
20
21
|
private fields;
|
|
21
22
|
private _isResponseInArrayMode;
|
|
22
23
|
private customResultMapper;
|
|
23
|
-
private
|
|
24
|
+
private rewriteArraysMode;
|
|
24
25
|
private rejectStringArrayLiterals;
|
|
26
|
+
private prepareCache;
|
|
25
27
|
private warnOnStringArrayLiteral?;
|
|
26
28
|
static readonly [entityKind]: string;
|
|
27
|
-
constructor(client: DuckDBClientLike, dialect: DuckDBDialect, queryString: string, params: unknown[], logger: Logger, fields: SelectedFieldsOrdered | undefined, _isResponseInArrayMode: boolean, customResultMapper: ((rows: unknown[][]) => T['execute']) | undefined,
|
|
29
|
+
constructor(client: DuckDBClientLike, dialect: DuckDBDialect, queryString: string, params: unknown[], logger: Logger, fields: SelectedFieldsOrdered | undefined, _isResponseInArrayMode: boolean, customResultMapper: ((rows: unknown[][]) => T['execute']) | undefined, rewriteArraysMode: RewriteArraysMode, rejectStringArrayLiterals: boolean, prepareCache: PreparedStatementCacheConfig | undefined, warnOnStringArrayLiteral?: ((sql: string) => void) | undefined);
|
|
28
30
|
execute(placeholderValues?: Record<string, unknown> | undefined): Promise<T['execute']>;
|
|
29
31
|
all(placeholderValues?: Record<string, unknown> | undefined): Promise<T['all']>;
|
|
30
32
|
isResponseInArrayMode(): boolean;
|
|
31
33
|
}
|
|
32
34
|
export interface DuckDBSessionOptions {
|
|
33
35
|
logger?: Logger;
|
|
34
|
-
rewriteArrays?:
|
|
36
|
+
rewriteArrays?: RewriteArraysMode;
|
|
35
37
|
rejectStringArrayLiterals?: boolean;
|
|
38
|
+
prepareCache?: PreparedStatementCacheConfig;
|
|
36
39
|
}
|
|
37
40
|
export declare class DuckDBSession<TFullSchema extends Record<string, unknown> = Record<string, never>, TSchema extends TablesRelationalConfig = Record<string, never>> extends PgSession<DuckDBQueryResultHKT, TFullSchema, TSchema> {
|
|
38
41
|
private client;
|
|
@@ -41,15 +44,22 @@ export declare class DuckDBSession<TFullSchema extends Record<string, unknown> =
|
|
|
41
44
|
static readonly [entityKind]: string;
|
|
42
45
|
protected dialect: DuckDBDialect;
|
|
43
46
|
private logger;
|
|
44
|
-
private
|
|
47
|
+
private rewriteArraysMode;
|
|
45
48
|
private rejectStringArrayLiterals;
|
|
49
|
+
private prepareCache;
|
|
46
50
|
private hasWarnedArrayLiteral;
|
|
51
|
+
private rollbackOnly;
|
|
47
52
|
constructor(client: DuckDBClientLike, dialect: DuckDBDialect, schema: RelationalSchemaConfig<TSchema> | undefined, options?: DuckDBSessionOptions);
|
|
48
53
|
prepareQuery<T extends PreparedQueryConfig = PreparedQueryConfig>(query: Query, fields: SelectedFieldsOrdered | undefined, name: string | undefined, isResponseInArrayMode: boolean, customResultMapper?: (rows: unknown[][]) => T['execute']): PgPreparedQuery<T>;
|
|
49
|
-
|
|
54
|
+
execute<T>(query: SQL): Promise<T>;
|
|
55
|
+
all<T = unknown>(query: SQL): Promise<T[]>;
|
|
56
|
+
transaction<T>(transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>, config?: PgTransactionConfig): Promise<T>;
|
|
50
57
|
private warnOnStringArrayLiteral;
|
|
51
58
|
executeBatches<T extends RowData = RowData>(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<GenericRowData<T>[], void, void>;
|
|
59
|
+
executeBatchesRaw(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<ExecuteBatchesRawChunk, void, void>;
|
|
52
60
|
executeArrow(query: SQL): Promise<unknown>;
|
|
61
|
+
markRollbackOnly(): void;
|
|
62
|
+
isRollbackOnly(): boolean;
|
|
53
63
|
}
|
|
54
64
|
export declare class DuckDBTransaction<TFullSchema extends Record<string, unknown>, TSchema extends TablesRelationalConfig> extends PgTransaction<DuckDBQueryResultHKT, TFullSchema, TSchema> {
|
|
55
65
|
static readonly [entityKind]: string;
|
|
@@ -57,8 +67,10 @@ export declare class DuckDBTransaction<TFullSchema extends Record<string, unknow
|
|
|
57
67
|
getTransactionConfigSQL(config: PgTransactionConfig): SQL;
|
|
58
68
|
setTransaction(config: PgTransactionConfig): Promise<void>;
|
|
59
69
|
executeBatches<T extends RowData = RowData>(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<GenericRowData<T>[], void, void>;
|
|
70
|
+
executeBatchesRaw(query: SQL, options?: ExecuteInBatchesOptions): AsyncGenerator<ExecuteBatchesRawChunk, void, void>;
|
|
60
71
|
executeArrow(query: SQL): Promise<unknown>;
|
|
61
72
|
transaction<T>(transaction: (tx: DuckDBTransaction<TFullSchema, TSchema>) => Promise<T>): Promise<T>;
|
|
73
|
+
private runNestedWithoutSavepoint;
|
|
62
74
|
}
|
|
63
75
|
export type GenericRowData<T extends RowData = RowData> = T;
|
|
64
76
|
export type GenericTableData<T = RowData> = T[];
|
|
@@ -1,2 +1,9 @@
|
|
|
1
1
|
import { type AnyColumn, type SelectedFieldsOrdered } from 'drizzle-orm';
|
|
2
|
+
export declare function normalizeInet(value: unknown): unknown;
|
|
3
|
+
export declare function normalizeTimestampString(value: unknown, withTimezone: boolean): string | unknown;
|
|
4
|
+
export declare function normalizeTimestamp(value: unknown, withTimezone: boolean): Date | unknown;
|
|
5
|
+
export declare function normalizeDateString(value: unknown): string | unknown;
|
|
6
|
+
export declare function normalizeDateValue(value: unknown): Date | unknown;
|
|
7
|
+
export declare function normalizeTime(value: unknown): string | unknown;
|
|
8
|
+
export declare function normalizeInterval(value: unknown): string | unknown;
|
|
2
9
|
export declare function mapResultRow<TResult>(columns: SelectedFieldsOrdered<AnyColumn>, row: unknown[], joinsNotNullableMap: Record<string, boolean> | undefined): TResult;
|
|
@@ -23,7 +23,7 @@ export interface StructValueWrapper extends DuckDBValueWrapper<'struct', Record<
|
|
|
23
23
|
export interface MapValueWrapper extends DuckDBValueWrapper<'map', Record<string, unknown>> {
|
|
24
24
|
readonly valueType?: string;
|
|
25
25
|
}
|
|
26
|
-
export interface TimestampValueWrapper extends DuckDBValueWrapper<'timestamp', Date | string> {
|
|
26
|
+
export interface TimestampValueWrapper extends DuckDBValueWrapper<'timestamp', Date | string | number | bigint> {
|
|
27
27
|
readonly withTimezone: boolean;
|
|
28
28
|
readonly precision?: number;
|
|
29
29
|
}
|
|
@@ -37,6 +37,6 @@ export declare function wrapList(data: unknown[], elementType?: string): ListVal
|
|
|
37
37
|
export declare function wrapArray(data: unknown[], elementType?: string, fixedLength?: number): ArrayValueWrapper;
|
|
38
38
|
export declare function wrapStruct(data: Record<string, unknown>, schema?: Record<string, string>): StructValueWrapper;
|
|
39
39
|
export declare function wrapMap(data: Record<string, unknown>, valueType?: string): MapValueWrapper;
|
|
40
|
-
export declare function wrapTimestamp(data: Date | string, withTimezone: boolean, precision?: number): TimestampValueWrapper;
|
|
40
|
+
export declare function wrapTimestamp(data: Date | string | number | bigint, withTimezone: boolean, precision?: number): TimestampValueWrapper;
|
|
41
41
|
export declare function wrapBlob(data: Buffer | Uint8Array): BlobValueWrapper;
|
|
42
42
|
export declare function wrapJson(data: unknown): JsonValueWrapper;
|
package/package.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"module": "./dist/index.mjs",
|
|
4
4
|
"main": "./dist/index.mjs",
|
|
5
5
|
"types": "./dist/index.d.ts",
|
|
6
|
-
"version": "1.1.
|
|
6
|
+
"version": "1.1.2",
|
|
7
7
|
"description": "A drizzle ORM client for use with DuckDB. Based on drizzle's Postgres client.",
|
|
8
8
|
"type": "module",
|
|
9
9
|
"scripts": {
|
|
@@ -12,6 +12,7 @@ interface CliOptions {
|
|
|
12
12
|
allDatabases: boolean;
|
|
13
13
|
schemas?: string[];
|
|
14
14
|
outFile: string;
|
|
15
|
+
outMeta?: string;
|
|
15
16
|
includeViews: boolean;
|
|
16
17
|
useCustomTimeTypes: boolean;
|
|
17
18
|
importBasePath?: string;
|
|
@@ -20,6 +21,7 @@ interface CliOptions {
|
|
|
20
21
|
function parseArgs(argv: string[]): CliOptions {
|
|
21
22
|
const options: CliOptions = {
|
|
22
23
|
outFile: path.resolve(process.cwd(), 'drizzle/schema.ts'),
|
|
24
|
+
outMeta: undefined,
|
|
23
25
|
allDatabases: false,
|
|
24
26
|
includeViews: false,
|
|
25
27
|
useCustomTimeTypes: true,
|
|
@@ -52,6 +54,14 @@ function parseArgs(argv: string[]): CliOptions {
|
|
|
52
54
|
argv[++i] ?? 'drizzle/schema.ts'
|
|
53
55
|
);
|
|
54
56
|
break;
|
|
57
|
+
case '--out-json':
|
|
58
|
+
case '--outJson':
|
|
59
|
+
case '--json':
|
|
60
|
+
options.outMeta = path.resolve(
|
|
61
|
+
process.cwd(),
|
|
62
|
+
argv[++i] ?? 'drizzle/schema.meta.json'
|
|
63
|
+
);
|
|
64
|
+
break;
|
|
55
65
|
case '--include-views':
|
|
56
66
|
case '--includeViews':
|
|
57
67
|
options.includeViews = true;
|
|
@@ -88,6 +98,7 @@ Options:
|
|
|
88
98
|
--all-databases Introspect all attached databases (not just current)
|
|
89
99
|
--schema Comma separated schema list (defaults to all non-system schemas)
|
|
90
100
|
--out Output file (default: ./drizzle/schema.ts)
|
|
101
|
+
--json Optional JSON metadata output (default: ./drizzle/schema.meta.json)
|
|
91
102
|
--include-views Include views in the generated schema
|
|
92
103
|
--use-pg-time Use pg-core timestamp/date/time instead of DuckDB custom helpers
|
|
93
104
|
--import-base Override import path for duckdb helpers (default: package name)
|
|
@@ -136,8 +147,19 @@ async function main() {
|
|
|
136
147
|
|
|
137
148
|
await mkdir(path.dirname(options.outFile), { recursive: true });
|
|
138
149
|
await writeFile(options.outFile, result.files.schemaTs, 'utf8');
|
|
150
|
+
if (options.outMeta) {
|
|
151
|
+
await mkdir(path.dirname(options.outMeta), { recursive: true });
|
|
152
|
+
await writeFile(
|
|
153
|
+
options.outMeta,
|
|
154
|
+
JSON.stringify(result.files.metaJson, null, 2),
|
|
155
|
+
'utf8'
|
|
156
|
+
);
|
|
157
|
+
}
|
|
139
158
|
|
|
140
159
|
console.log(`Wrote schema to ${options.outFile}`);
|
|
160
|
+
if (options.outMeta) {
|
|
161
|
+
console.log(`Wrote metadata to ${options.outMeta}`);
|
|
162
|
+
}
|
|
141
163
|
} finally {
|
|
142
164
|
if (
|
|
143
165
|
'closeSync' in connection &&
|
|
@@ -145,6 +167,11 @@ async function main() {
|
|
|
145
167
|
) {
|
|
146
168
|
connection.closeSync();
|
|
147
169
|
}
|
|
170
|
+
if ('closeSync' in instance && typeof instance.closeSync === 'function') {
|
|
171
|
+
instance.closeSync();
|
|
172
|
+
} else if ('close' in instance && typeof instance.close === 'function') {
|
|
173
|
+
await instance.close();
|
|
174
|
+
}
|
|
148
175
|
}
|
|
149
176
|
}
|
|
150
177
|
|
package/src/client.ts
CHANGED
|
@@ -2,6 +2,7 @@ import {
|
|
|
2
2
|
listValue,
|
|
3
3
|
timestampValue,
|
|
4
4
|
type DuckDBConnection,
|
|
5
|
+
type DuckDBPreparedStatement,
|
|
5
6
|
type DuckDBValue,
|
|
6
7
|
} from '@duckdb/node-api';
|
|
7
8
|
import {
|
|
@@ -9,6 +10,7 @@ import {
|
|
|
9
10
|
wrapperToNodeApiValue,
|
|
10
11
|
type AnyDuckDBValueWrapper,
|
|
11
12
|
} from './value-wrappers.ts';
|
|
13
|
+
import type { PreparedStatementCacheConfig } from './options.ts';
|
|
12
14
|
|
|
13
15
|
export type DuckDBClientLike = DuckDBConnection | DuckDBConnectionPool;
|
|
14
16
|
export type RowData = Record<string, unknown>;
|
|
@@ -25,6 +27,25 @@ export function isPool(
|
|
|
25
27
|
return typeof (client as DuckDBConnectionPool).acquire === 'function';
|
|
26
28
|
}
|
|
27
29
|
|
|
30
|
+
export interface ExecuteClientOptions {
|
|
31
|
+
prepareCache?: PreparedStatementCacheConfig;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export type ExecuteArraysResult = { columns: string[]; rows: unknown[][] };
|
|
35
|
+
|
|
36
|
+
type MaterializedRows = ExecuteArraysResult;
|
|
37
|
+
|
|
38
|
+
type PreparedCacheEntry = {
|
|
39
|
+
statement: DuckDBPreparedStatement;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
type PreparedStatementCache = {
|
|
43
|
+
size: number;
|
|
44
|
+
entries: Map<string, PreparedCacheEntry>;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const PREPARED_CACHE = Symbol.for('drizzle-duckdb:prepared-cache');
|
|
48
|
+
|
|
28
49
|
export interface PrepareParamsOptions {
|
|
29
50
|
rejectStringArrayLiterals?: boolean;
|
|
30
51
|
warnOnStringArrayLiteral?: () => void;
|
|
@@ -49,17 +70,31 @@ export function prepareParams(
|
|
|
49
70
|
options: PrepareParamsOptions = {}
|
|
50
71
|
): unknown[] {
|
|
51
72
|
return params.map((param) => {
|
|
52
|
-
if (typeof param === 'string' &&
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
73
|
+
if (typeof param === 'string' && param.length > 0) {
|
|
74
|
+
const firstChar = param[0];
|
|
75
|
+
const maybeArrayLiteral =
|
|
76
|
+
firstChar === '{' ||
|
|
77
|
+
firstChar === '[' ||
|
|
78
|
+
firstChar === ' ' ||
|
|
79
|
+
firstChar === '\t';
|
|
80
|
+
|
|
81
|
+
if (maybeArrayLiteral) {
|
|
82
|
+
const trimmed =
|
|
83
|
+
firstChar === '{' || firstChar === '[' ? param : param.trim();
|
|
84
|
+
|
|
85
|
+
if (trimmed && isPgArrayLiteral(trimmed)) {
|
|
86
|
+
if (options.rejectStringArrayLiterals) {
|
|
87
|
+
throw new Error(
|
|
88
|
+
'Stringified array literals are not supported. Use duckDbList()/duckDbArray() or pass native arrays.'
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (options.warnOnStringArrayLiteral) {
|
|
93
|
+
options.warnOnStringArrayLiteral();
|
|
94
|
+
}
|
|
95
|
+
return parsePgArrayLiteral(trimmed);
|
|
96
|
+
}
|
|
61
97
|
}
|
|
62
|
-
return parsePgArrayLiteral(param);
|
|
63
98
|
}
|
|
64
99
|
return param;
|
|
65
100
|
});
|
|
@@ -103,14 +138,177 @@ function toNodeApiValue(value: unknown): DuckDBValue {
|
|
|
103
138
|
}
|
|
104
139
|
|
|
105
140
|
function deduplicateColumns(columns: string[]): string[] {
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
141
|
+
const counts = new Map<string, number>();
|
|
142
|
+
let hasDuplicates = false;
|
|
143
|
+
|
|
144
|
+
for (const column of columns) {
|
|
145
|
+
const next = (counts.get(column) ?? 0) + 1;
|
|
146
|
+
counts.set(column, next);
|
|
147
|
+
if (next > 1) {
|
|
148
|
+
hasDuplicates = true;
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (!hasDuplicates) {
|
|
154
|
+
return columns;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
counts.clear();
|
|
158
|
+
return columns.map((column) => {
|
|
159
|
+
const count = counts.get(column) ?? 0;
|
|
160
|
+
counts.set(column, count + 1);
|
|
161
|
+
return count === 0 ? column : `${column}_${count}`;
|
|
111
162
|
});
|
|
112
163
|
}
|
|
113
164
|
|
|
165
|
+
function destroyPreparedStatement(entry: PreparedCacheEntry | undefined): void {
|
|
166
|
+
if (!entry) return;
|
|
167
|
+
try {
|
|
168
|
+
entry.statement.destroySync();
|
|
169
|
+
} catch {
|
|
170
|
+
// Ignore cleanup errors
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function getPreparedCache(
|
|
175
|
+
connection: DuckDBConnection,
|
|
176
|
+
size: number
|
|
177
|
+
): PreparedStatementCache {
|
|
178
|
+
const store = connection as unknown as Record<
|
|
179
|
+
symbol,
|
|
180
|
+
PreparedStatementCache | undefined
|
|
181
|
+
>;
|
|
182
|
+
const existing = store[PREPARED_CACHE];
|
|
183
|
+
if (existing) {
|
|
184
|
+
existing.size = size;
|
|
185
|
+
return existing;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const cache: PreparedStatementCache = { size, entries: new Map() };
|
|
189
|
+
store[PREPARED_CACHE] = cache;
|
|
190
|
+
return cache;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function evictOldest(cache: PreparedStatementCache): void {
|
|
194
|
+
const oldest = cache.entries.keys().next();
|
|
195
|
+
if (!oldest.done) {
|
|
196
|
+
const key = oldest.value as string;
|
|
197
|
+
const entry = cache.entries.get(key);
|
|
198
|
+
cache.entries.delete(key);
|
|
199
|
+
destroyPreparedStatement(entry);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function evictCacheEntry(cache: PreparedStatementCache, key: string): void {
|
|
204
|
+
const entry = cache.entries.get(key);
|
|
205
|
+
cache.entries.delete(key);
|
|
206
|
+
destroyPreparedStatement(entry);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async function getOrPrepareStatement(
|
|
210
|
+
connection: DuckDBConnection,
|
|
211
|
+
query: string,
|
|
212
|
+
cacheConfig: PreparedStatementCacheConfig
|
|
213
|
+
): Promise<DuckDBPreparedStatement> {
|
|
214
|
+
const cache = getPreparedCache(connection, cacheConfig.size);
|
|
215
|
+
const cached = cache.entries.get(query);
|
|
216
|
+
if (cached) {
|
|
217
|
+
cache.entries.delete(query);
|
|
218
|
+
cache.entries.set(query, cached);
|
|
219
|
+
return cached.statement;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const statement = await connection.prepare(query);
|
|
223
|
+
cache.entries.set(query, { statement });
|
|
224
|
+
|
|
225
|
+
while (cache.entries.size > cache.size) {
|
|
226
|
+
evictOldest(cache);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
return statement;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
async function materializeResultRows(result: {
|
|
233
|
+
getRowsJS: () => Promise<unknown[][] | undefined>;
|
|
234
|
+
columnNames: () => string[];
|
|
235
|
+
deduplicatedColumnNames?: () => string[];
|
|
236
|
+
}): Promise<MaterializedRows> {
|
|
237
|
+
const rows = (await result.getRowsJS()) ?? [];
|
|
238
|
+
const baseColumns =
|
|
239
|
+
typeof result.deduplicatedColumnNames === 'function'
|
|
240
|
+
? result.deduplicatedColumnNames()
|
|
241
|
+
: result.columnNames();
|
|
242
|
+
const columns =
|
|
243
|
+
typeof result.deduplicatedColumnNames === 'function'
|
|
244
|
+
? baseColumns
|
|
245
|
+
: deduplicateColumns(baseColumns);
|
|
246
|
+
|
|
247
|
+
return { columns, rows };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async function materializeRows(
|
|
251
|
+
client: DuckDBClientLike,
|
|
252
|
+
query: string,
|
|
253
|
+
params: unknown[],
|
|
254
|
+
options: ExecuteClientOptions = {}
|
|
255
|
+
): Promise<MaterializedRows> {
|
|
256
|
+
if (isPool(client)) {
|
|
257
|
+
const connection = await client.acquire();
|
|
258
|
+
try {
|
|
259
|
+
return await materializeRows(connection, query, params, options);
|
|
260
|
+
} finally {
|
|
261
|
+
await client.release(connection);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const values =
|
|
266
|
+
params.length > 0
|
|
267
|
+
? (params.map((param) => toNodeApiValue(param)) as DuckDBValue[])
|
|
268
|
+
: undefined;
|
|
269
|
+
|
|
270
|
+
const connection = client as DuckDBConnection;
|
|
271
|
+
|
|
272
|
+
if (options.prepareCache && typeof connection.prepare === 'function') {
|
|
273
|
+
const cache = getPreparedCache(connection, options.prepareCache.size);
|
|
274
|
+
try {
|
|
275
|
+
const statement = await getOrPrepareStatement(
|
|
276
|
+
connection,
|
|
277
|
+
query,
|
|
278
|
+
options.prepareCache
|
|
279
|
+
);
|
|
280
|
+
if (values) {
|
|
281
|
+
statement.bind(values as DuckDBValue[]);
|
|
282
|
+
} else {
|
|
283
|
+
statement.clearBindings?.();
|
|
284
|
+
}
|
|
285
|
+
const result = await statement.run();
|
|
286
|
+
cache.entries.delete(query);
|
|
287
|
+
cache.entries.set(query, { statement });
|
|
288
|
+
return await materializeResultRows(result);
|
|
289
|
+
} catch (error) {
|
|
290
|
+
evictCacheEntry(cache, query);
|
|
291
|
+
throw error;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const result = await connection.run(query, values);
|
|
296
|
+
return await materializeResultRows(result);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function clearPreparedCache(connection: DuckDBConnection): void {
|
|
300
|
+
const store = connection as unknown as Record<
|
|
301
|
+
symbol,
|
|
302
|
+
PreparedStatementCache | undefined
|
|
303
|
+
>;
|
|
304
|
+
const cache = store[PREPARED_CACHE];
|
|
305
|
+
if (!cache) return;
|
|
306
|
+
for (const entry of cache.entries.values()) {
|
|
307
|
+
destroyPreparedStatement(entry);
|
|
308
|
+
}
|
|
309
|
+
cache.entries.clear();
|
|
310
|
+
}
|
|
311
|
+
|
|
114
312
|
function mapRowsToObjects(columns: string[], rows: unknown[][]): RowData[] {
|
|
115
313
|
return rows.map((vals) => {
|
|
116
314
|
const obj: Record<string, unknown> = {};
|
|
@@ -124,6 +322,8 @@ function mapRowsToObjects(columns: string[], rows: unknown[][]): RowData[] {
|
|
|
124
322
|
export async function closeClientConnection(
|
|
125
323
|
connection: DuckDBConnection
|
|
126
324
|
): Promise<void> {
|
|
325
|
+
clearPreparedCache(connection);
|
|
326
|
+
|
|
127
327
|
if ('close' in connection && typeof connection.close === 'function') {
|
|
128
328
|
await connection.close();
|
|
129
329
|
return;
|
|
@@ -145,35 +345,41 @@ export async function closeClientConnection(
|
|
|
145
345
|
export async function executeOnClient(
|
|
146
346
|
client: DuckDBClientLike,
|
|
147
347
|
query: string,
|
|
148
|
-
params: unknown[]
|
|
348
|
+
params: unknown[],
|
|
349
|
+
options: ExecuteClientOptions = {}
|
|
149
350
|
): Promise<RowData[]> {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
351
|
+
const { columns, rows } = await materializeRows(
|
|
352
|
+
client,
|
|
353
|
+
query,
|
|
354
|
+
params,
|
|
355
|
+
options
|
|
356
|
+
);
|
|
357
|
+
|
|
358
|
+
if (!rows || rows.length === 0) {
|
|
359
|
+
return [];
|
|
157
360
|
}
|
|
158
361
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
? (params.map((param) => toNodeApiValue(param)) as DuckDBValue[])
|
|
162
|
-
: undefined;
|
|
163
|
-
const result = await client.run(query, values);
|
|
164
|
-
const rows = await result.getRowsJS();
|
|
165
|
-
const columns =
|
|
166
|
-
// prefer deduplicated names when available (Node API >=1.4.2)
|
|
167
|
-
result.deduplicatedColumnNames?.() ?? result.columnNames();
|
|
168
|
-
const uniqueColumns = deduplicateColumns(columns);
|
|
362
|
+
return mapRowsToObjects(columns, rows);
|
|
363
|
+
}
|
|
169
364
|
|
|
170
|
-
|
|
365
|
+
export async function executeArraysOnClient(
|
|
366
|
+
client: DuckDBClientLike,
|
|
367
|
+
query: string,
|
|
368
|
+
params: unknown[],
|
|
369
|
+
options: ExecuteClientOptions = {}
|
|
370
|
+
): Promise<ExecuteArraysResult> {
|
|
371
|
+
return await materializeRows(client, query, params, options);
|
|
171
372
|
}
|
|
172
373
|
|
|
173
374
|
export interface ExecuteInBatchesOptions {
|
|
174
375
|
rowsPerChunk?: number;
|
|
175
376
|
}
|
|
176
377
|
|
|
378
|
+
export interface ExecuteBatchesRawChunk {
|
|
379
|
+
columns: string[];
|
|
380
|
+
rows: unknown[][];
|
|
381
|
+
}
|
|
382
|
+
|
|
177
383
|
/**
|
|
178
384
|
* Stream results from DuckDB in batches to avoid fully materializing rows in JS.
|
|
179
385
|
*/
|
|
@@ -203,15 +409,19 @@ export async function* executeInBatches(
|
|
|
203
409
|
: undefined;
|
|
204
410
|
|
|
205
411
|
const result = await client.stream(query, values);
|
|
412
|
+
const rawColumns =
|
|
413
|
+
typeof result.deduplicatedColumnNames === 'function'
|
|
414
|
+
? result.deduplicatedColumnNames()
|
|
415
|
+
: result.columnNames();
|
|
206
416
|
const columns =
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
417
|
+
typeof result.deduplicatedColumnNames === 'function'
|
|
418
|
+
? rawColumns
|
|
419
|
+
: deduplicateColumns(rawColumns);
|
|
210
420
|
|
|
211
421
|
let buffer: RowData[] = [];
|
|
212
422
|
|
|
213
423
|
for await (const chunk of result.yieldRowsJs()) {
|
|
214
|
-
const objects = mapRowsToObjects(
|
|
424
|
+
const objects = mapRowsToObjects(columns, chunk);
|
|
215
425
|
for (const row of objects) {
|
|
216
426
|
buffer.push(row);
|
|
217
427
|
if (buffer.length >= rowsPerChunk) {
|
|
@@ -226,6 +436,59 @@ export async function* executeInBatches(
|
|
|
226
436
|
}
|
|
227
437
|
}
|
|
228
438
|
|
|
439
|
+
export async function* executeInBatchesRaw(
|
|
440
|
+
client: DuckDBClientLike,
|
|
441
|
+
query: string,
|
|
442
|
+
params: unknown[],
|
|
443
|
+
options: ExecuteInBatchesOptions = {}
|
|
444
|
+
): AsyncGenerator<ExecuteBatchesRawChunk, void, void> {
|
|
445
|
+
if (isPool(client)) {
|
|
446
|
+
const connection = await client.acquire();
|
|
447
|
+
try {
|
|
448
|
+
yield* executeInBatchesRaw(connection, query, params, options);
|
|
449
|
+
return;
|
|
450
|
+
} finally {
|
|
451
|
+
await client.release(connection);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const rowsPerChunk =
|
|
456
|
+
options.rowsPerChunk && options.rowsPerChunk > 0
|
|
457
|
+
? options.rowsPerChunk
|
|
458
|
+
: 100_000;
|
|
459
|
+
|
|
460
|
+
const values =
|
|
461
|
+
params.length > 0
|
|
462
|
+
? (params.map((param) => toNodeApiValue(param)) as DuckDBValue[])
|
|
463
|
+
: undefined;
|
|
464
|
+
|
|
465
|
+
const result = await client.stream(query, values);
|
|
466
|
+
const rawColumns =
|
|
467
|
+
typeof result.deduplicatedColumnNames === 'function'
|
|
468
|
+
? result.deduplicatedColumnNames()
|
|
469
|
+
: result.columnNames();
|
|
470
|
+
const columns =
|
|
471
|
+
typeof result.deduplicatedColumnNames === 'function'
|
|
472
|
+
? rawColumns
|
|
473
|
+
: deduplicateColumns(rawColumns);
|
|
474
|
+
|
|
475
|
+
let buffer: unknown[][] = [];
|
|
476
|
+
|
|
477
|
+
for await (const chunk of result.yieldRowsJs()) {
|
|
478
|
+
for (const row of chunk) {
|
|
479
|
+
buffer.push(row as unknown[]);
|
|
480
|
+
if (buffer.length >= rowsPerChunk) {
|
|
481
|
+
yield { columns, rows: buffer };
|
|
482
|
+
buffer = [];
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
if (buffer.length > 0) {
|
|
488
|
+
yield { columns, rows: buffer };
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
|
|
229
492
|
/**
|
|
230
493
|
* Return columnar results when the underlying node-api exposes an Arrow/columnar API.
|
|
231
494
|
* Falls back to column-major JS arrays when Arrow is unavailable.
|