@loaders.gl/sql 5.0.0-alpha.9 → 5.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -30
- package/README.md +21 -12
- package/THIRD-PARTY-NOTICES.md +17 -0
- package/dist/arrow-table-source.d.ts +9 -9
- package/dist/arrow-table-source.d.ts.map +1 -1
- package/dist/arrow-table-source.js +11 -7
- package/dist/arrow-table-source.js.map +1 -1
- package/dist/compile-table-query.d.ts +3 -3
- package/dist/compile-table-query.d.ts.map +1 -1
- package/dist/compile-table-query.js +2 -2
- package/dist/compile-table-query.js.map +1 -1
- package/dist/duckdb-sql-source.d.ts +36 -32
- package/dist/duckdb-sql-source.d.ts.map +1 -1
- package/dist/duckdb-sql-source.js +61 -17
- package/dist/duckdb-sql-source.js.map +1 -1
- package/dist/duckdb-table-query.d.ts +9 -0
- package/dist/duckdb-table-query.d.ts.map +1 -0
- package/dist/duckdb-table-query.js +242 -0
- package/dist/duckdb-table-query.js.map +1 -0
- package/dist/index.cjs +1843 -1473
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +12 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -7
- package/dist/index.js.map +1 -1
- package/dist/snowflake-sql-source.d.ts +18 -31
- package/dist/snowflake-sql-source.d.ts.map +1 -1
- package/dist/snowflake-sql-source.js +2 -2
- package/dist/snowflake-sql-source.js.map +1 -1
- package/dist/sql-source.d.ts +24 -11
- package/dist/sql-source.d.ts.map +1 -1
- package/dist/sql-source.js +35 -3
- package/dist/sql-source.js.map +1 -1
- package/dist/sql-types.d.ts +15 -3
- package/dist/sql-types.d.ts.map +1 -1
- package/dist/sql-utils.d.ts +2 -3
- package/dist/sql-utils.d.ts.map +1 -1
- package/dist/sql-utils.js +1 -34
- package/dist/sql-utils.js.map +1 -1
- package/package.json +8 -28
- package/src/arrow-table-source.ts +17 -16
- package/src/compile-table-query.ts +4 -4
- package/src/duckdb-sql-source.ts +81 -20
- package/src/duckdb-table-query.ts +277 -0
- package/src/index.ts +19 -11
- package/src/snowflake-sql-source.ts +3 -3
- package/src/sql-source.ts +46 -7
- package/src/sql-types.ts +16 -3
- package/src/sql-utils.ts +2 -40
- package/dist/arrow-query.cjs +0 -1404
- package/dist/arrow-query.cjs.map +0 -6
- package/dist/arrow-query.d.ts +0 -5
- package/dist/arrow-query.d.ts.map +0 -1
- package/dist/arrow-query.js +0 -6
- package/dist/arrow-query.js.map +0 -1
- package/dist/bind-sql-predicate.d.ts +0 -4
- package/dist/bind-sql-predicate.d.ts.map +0 -1
- package/dist/bind-sql-predicate.js +0 -11
- package/dist/bind-sql-predicate.js.map +0 -1
- package/dist/parse-sql-predicate.d.ts +0 -10
- package/dist/parse-sql-predicate.d.ts.map +0 -1
- package/dist/parse-sql-predicate.js +0 -322
- package/dist/parse-sql-predicate.js.map +0 -1
- package/dist/query-arrow-table.d.ts +0 -83
- package/dist/query-arrow-table.d.ts.map +0 -1
- package/dist/query-arrow-table.js +0 -663
- package/dist/query-arrow-table.js.map +0 -1
- package/dist/sql-predicate-schema.d.ts +0 -156
- package/dist/sql-predicate-schema.d.ts.map +0 -1
- package/dist/sql-predicate-schema.js +0 -226
- package/dist/sql-predicate-schema.js.map +0 -1
- package/dist/sql-predicate-types.d.ts +0 -45
- package/dist/sql-predicate-types.d.ts.map +0 -1
- package/dist/sql-predicate-types.js +0 -9
- package/dist/sql-predicate-types.js.map +0 -1
- package/dist/sql-predicate-zod.cjs +0 -54
- package/dist/sql-predicate-zod.cjs.map +0 -6
- package/dist/sql-predicate-zod.d.ts +0 -11
- package/dist/sql-predicate-zod.d.ts.map +0 -1
- package/dist/sql-predicate-zod.js +0 -45
- package/dist/sql-predicate-zod.js.map +0 -1
- package/dist/sql-predicate.cjs +0 -607
- package/dist/sql-predicate.cjs.map +0 -6
- package/dist/sql-predicate.d.ts +0 -6
- package/dist/sql-predicate.d.ts.map +0 -1
- package/dist/sql-predicate.js +0 -8
- package/dist/sql-predicate.js.map +0 -1
- package/dist/table-query-capabilities.d.ts +0 -6
- package/dist/table-query-capabilities.d.ts.map +0 -1
- package/dist/table-query-capabilities.js +0 -26
- package/dist/table-query-capabilities.js.map +0 -1
- package/dist/table-query.cjs +0 -286
- package/dist/table-query.cjs.map +0 -6
- package/dist/table-query.d.ts +0 -29
- package/dist/table-query.d.ts.map +0 -1
- package/dist/table-query.js +0 -30
- package/dist/table-query.js.map +0 -1
- package/src/arrow-query.ts +0 -17
- package/src/bind-sql-predicate.ts +0 -21
- package/src/parse-sql-predicate.ts +0 -381
- package/src/query-arrow-table.ts +0 -846
- package/src/sql-predicate-schema.ts +0 -251
- package/src/sql-predicate-types.ts +0 -87
- package/src/sql-predicate-zod.ts +0 -52
- package/src/sql-predicate.ts +0 -28
- package/src/table-query-capabilities.ts +0 -29
- package/src/table-query.ts +0 -78
|
@@ -2,26 +2,26 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
+
import type {RecordBatch, Table} from 'apache-arrow';
|
|
6
|
+
import {makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
|
|
5
7
|
import type {
|
|
6
8
|
DataSourceOptions,
|
|
7
9
|
ScanQueryMetadata,
|
|
8
10
|
ScanQueryMetadataOptions
|
|
9
|
-
} from '@loaders.gl/loader-utils';
|
|
11
|
+
} from '@loaders.gl/core/loader-utils';
|
|
10
12
|
import {
|
|
11
13
|
createScanQueryMetadata,
|
|
12
14
|
DataSource,
|
|
13
|
-
makeTableScanBatch,
|
|
14
15
|
type ScanColumnRole
|
|
15
|
-
} from '@loaders.gl/loader-utils';
|
|
16
|
-
import {convertArrowToSchema} from '@loaders.gl/schema-utils';
|
|
17
|
-
import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
|
|
16
|
+
} from '@loaders.gl/core/loader-utils';
|
|
17
|
+
import {convertArrowToSchema} from '@loaders.gl/core/schema-utils';
|
|
18
18
|
import {
|
|
19
19
|
ARROW_TABLE_QUERY_CAPABILITIES,
|
|
20
20
|
explainArrowTableQuery,
|
|
21
21
|
queryArrowTable
|
|
22
|
-
} from '
|
|
23
|
-
import type {ArrowQueryOptions} from '
|
|
24
|
-
import type {TableQueryExplain} from '
|
|
22
|
+
} from '@loaders.gl/core/scan/query-arrow-table';
|
|
23
|
+
import type {ArrowQueryOptions} from '@loaders.gl/core/scan/query-arrow-table';
|
|
24
|
+
import type {TableQueryExplain} from '@loaders.gl/core/scan/table-query';
|
|
25
25
|
|
|
26
26
|
/** In-memory source options; the source owns the supplied Arrow table. */
|
|
27
27
|
export type ArrowTableSourceOptions = DataSourceOptions;
|
|
@@ -33,16 +33,16 @@ export type ArrowTableSourceOptions = DataSourceOptions;
|
|
|
33
33
|
* practice, and query execution delegates to the existing Arrow executor without ingesting data
|
|
34
34
|
* into DuckDB or another intermediate representation.
|
|
35
35
|
*/
|
|
36
|
-
export class ArrowTableSource extends DataSource<
|
|
36
|
+
export class ArrowTableSource extends DataSource<Table, ArrowTableSourceOptions> {
|
|
37
37
|
/** Creates a source over an existing Arrow table without copying its vectors. */
|
|
38
|
-
constructor(data:
|
|
38
|
+
constructor(data: Table, options: ArrowTableSourceOptions = {}) {
|
|
39
39
|
super(data, options);
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
/** Discovers query-visible fields and capabilities without evaluating any rows. */
|
|
43
43
|
async getQueryMetadata(options: ScanQueryMetadataOptions = {}): Promise<ScanQueryMetadata> {
|
|
44
44
|
throwIfAborted(options.signal);
|
|
45
|
-
const schema = convertArrowToSchema(this.data.
|
|
45
|
+
const schema = convertArrowToSchema(this.data.schema);
|
|
46
46
|
return createScanQueryMetadata({
|
|
47
47
|
sourceType: 'arrow-table',
|
|
48
48
|
queryType: 'table',
|
|
@@ -50,12 +50,12 @@ export class ArrowTableSource extends DataSource<ArrowTable, ArrowTableSourceOpt
|
|
|
50
50
|
schema,
|
|
51
51
|
capabilities: {table: ARROW_TABLE_QUERY_CAPABILITIES},
|
|
52
52
|
columnRoles: getArrowColumnRoles(schema.fields.map(field => field.name)),
|
|
53
|
-
statistics: {rowCount: this.data.
|
|
53
|
+
statistics: {rowCount: this.data.numRows}
|
|
54
54
|
});
|
|
55
55
|
}
|
|
56
56
|
|
|
57
57
|
/** Executes a portable query and returns an Arrow table result. */
|
|
58
|
-
query(options: ArrowQueryOptions = {}):
|
|
58
|
+
query(options: ArrowQueryOptions = {}): Table {
|
|
59
59
|
return queryArrowTable(this.data, options);
|
|
60
60
|
}
|
|
61
61
|
|
|
@@ -65,9 +65,10 @@ export class ArrowTableSource extends DataSource<ArrowTable, ArrowTableSourceOpt
|
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
/** Executes a query as one bounded Arrow batch. */
|
|
68
|
-
async *read(options: ArrowQueryOptions = {}): AsyncIterableIterator<
|
|
69
|
-
const
|
|
70
|
-
|
|
68
|
+
async *read(options: ArrowQueryOptions = {}): AsyncIterableIterator<RecordBatch> {
|
|
69
|
+
const data = this.query(options);
|
|
70
|
+
if (data.batches.length) yield* data.batches;
|
|
71
|
+
else yield makeArrowMetadataBatch(data.schema);
|
|
71
72
|
}
|
|
72
73
|
}
|
|
73
74
|
|
|
@@ -2,20 +2,20 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {validateSQLPredicate} from '
|
|
5
|
+
import {validateSQLPredicate} from '@loaders.gl/core/scan/sql-predicate-schema';
|
|
6
6
|
import {
|
|
7
7
|
isSQLPredicateParameter,
|
|
8
8
|
type SQLPredicate,
|
|
9
9
|
type SQLPredicateParameterValues,
|
|
10
10
|
type SQLPredicateScalar,
|
|
11
11
|
type SQLPredicateValue
|
|
12
|
-
} from '
|
|
13
|
-
import type {TableQueryOptions} from '
|
|
12
|
+
} from '@loaders.gl/core/scan/sql-predicate-types';
|
|
13
|
+
import type {TableQueryOptions} from '@loaders.gl/core/scan/table-query';
|
|
14
14
|
import type {
|
|
15
15
|
RelationalAggregate,
|
|
16
16
|
RelationalExpression,
|
|
17
17
|
RelationalOrderKey
|
|
18
|
-
} from '@loaders.gl/loader-utils';
|
|
18
|
+
} from '@loaders.gl/core/loader-utils';
|
|
19
19
|
|
|
20
20
|
/** A named child relation using the SQL module's property-oriented predicate AST. */
|
|
21
21
|
type SQLChildQuery = Readonly<{
|
package/src/duckdb-sql-source.ts
CHANGED
|
@@ -2,10 +2,18 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import type
|
|
6
|
-
import type {
|
|
7
|
-
import {
|
|
5
|
+
import {tableFromIPC, tableToIPC, type Table} from 'apache-arrow';
|
|
6
|
+
import type {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
|
|
7
|
+
import type {Schema} from '@loaders.gl/core/schema';
|
|
8
|
+
import type {
|
|
9
|
+
SourceLoader,
|
|
10
|
+
CoreAPI,
|
|
11
|
+
TableQuerySource,
|
|
12
|
+
TableMetadata
|
|
13
|
+
} from '@loaders.gl/core/loader-utils';
|
|
14
|
+
import {convertArrowToTable} from '@loaders.gl/core/schema-utils';
|
|
8
15
|
|
|
16
|
+
import {makeDuckDBTableQuerySource, isDuckDBQuerySchema} from './duckdb-table-query';
|
|
9
17
|
import {SQLDataSource, SQL_SOURCE_DEFAULT_OPTIONS} from './sql-source';
|
|
10
18
|
import type {
|
|
11
19
|
SQLAdapter,
|
|
@@ -14,7 +22,8 @@ import type {
|
|
|
14
22
|
SQLColumnInfo,
|
|
15
23
|
SQLQueryOptions,
|
|
16
24
|
SQLSourceOptions,
|
|
17
|
-
SQLTableInfo
|
|
25
|
+
SQLTableInfo,
|
|
26
|
+
SQLArrowInsertOptions
|
|
18
27
|
} from './sql-types';
|
|
19
28
|
import {
|
|
20
29
|
convertRowsToArrowTable,
|
|
@@ -66,7 +75,7 @@ export const DuckDBSQLSource = {
|
|
|
66
75
|
},
|
|
67
76
|
defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
|
|
68
77
|
testURL: (url: string): boolean => /^duckdb:\/\//i.test(url),
|
|
69
|
-
|
|
78
|
+
createDataSourceSync: (
|
|
70
79
|
data: string,
|
|
71
80
|
options: SQLSourceOptions,
|
|
72
81
|
coreApi?: CoreAPI
|
|
@@ -75,6 +84,47 @@ export const DuckDBSQLSource = {
|
|
|
75
84
|
|
|
76
85
|
/** SQLDataSource specialization for DuckDB. */
|
|
77
86
|
export class DuckDBSQLDataSource extends SQLDataSource {
|
|
87
|
+
/** Resolves a reusable common query handle for a named DuckDB table. */
|
|
88
|
+
async getTable(name: string, options: {signal?: AbortSignal} = {}): Promise<TableQuerySource> {
|
|
89
|
+
const table = await this.resolveTable(name, options.signal);
|
|
90
|
+
const schema = await this.getTableSchema(table);
|
|
91
|
+
options.signal?.throwIfAborted();
|
|
92
|
+
return makeDuckDBTableQuerySource(this, table, schema);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Reports whether the table's schema can be represented exactly by the common reader. */
|
|
96
|
+
override async getTableMetadata(
|
|
97
|
+
name: string,
|
|
98
|
+
options: {signal?: AbortSignal} = {}
|
|
99
|
+
): Promise<TableMetadata> {
|
|
100
|
+
const metadata = await super.getTableMetadata(name, options);
|
|
101
|
+
return {
|
|
102
|
+
...metadata,
|
|
103
|
+
queryable: Boolean(metadata.schema && isDuckDBQuerySchema(metadata.schema))
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Lists table names without claiming compatibility before schema discovery. */
|
|
108
|
+
override async getTablesMetadata(
|
|
109
|
+
options: {signal?: AbortSignal} = {}
|
|
110
|
+
): Promise<readonly TableMetadata[]> {
|
|
111
|
+
return (await super.getTablesMetadata(options)).map(({name}) => ({name}));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Uploads an Arrow table through IPC. Supported by the browser DuckDB-Wasm adapter. */
|
|
115
|
+
async insertArrowTable(table: Table, options: SQLArrowInsertOptions): Promise<void> {
|
|
116
|
+
await this.insertArrowIPC(tableToIPC(table, 'stream'), options);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Uploads a complete Arrow IPC stream to a browser DuckDB-Wasm table. */
|
|
120
|
+
async insertArrowIPC(bytes: Uint8Array, options: SQLArrowInsertOptions): Promise<void> {
|
|
121
|
+
const adapter = await this.getAdapter();
|
|
122
|
+
if (!adapter.insertArrowIPC) {
|
|
123
|
+
throw new Error('This DuckDB adapter does not support Arrow IPC uploads.');
|
|
124
|
+
}
|
|
125
|
+
await adapter.insertArrowIPC(bytes, options);
|
|
126
|
+
}
|
|
127
|
+
|
|
78
128
|
constructor(data: string, options: SQLSourceOptions, coreApi?: CoreAPI) {
|
|
79
129
|
super(
|
|
80
130
|
data,
|
|
@@ -253,7 +303,7 @@ async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Pr
|
|
|
253
303
|
options: SQLQueryOptions = {}
|
|
254
304
|
): Promise<Record<string, unknown>[]> => {
|
|
255
305
|
const result = await executeDuckDBBrowserQuery(connection, sqlText, options.parameters);
|
|
256
|
-
return convertArrowToTable(result, 'object-row-table').data;
|
|
306
|
+
return convertArrowToTable(tableFromIPC(result), 'object-row-table').data;
|
|
257
307
|
};
|
|
258
308
|
|
|
259
309
|
return {
|
|
@@ -263,6 +313,14 @@ async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Pr
|
|
|
263
313
|
runtime: 'browser',
|
|
264
314
|
isDynamic: true
|
|
265
315
|
},
|
|
316
|
+
async insertArrowIPC(bytes, options): Promise<void> {
|
|
317
|
+
// DuckDB transfers the buffer to its worker; retain the caller's IPC bytes.
|
|
318
|
+
await connection.insertArrowFromIPCStream(new Uint8Array(bytes), {
|
|
319
|
+
name: options.tableName,
|
|
320
|
+
schema: options.schemaName,
|
|
321
|
+
create: options.create ?? true
|
|
322
|
+
});
|
|
323
|
+
},
|
|
266
324
|
async connect(): Promise<void> {},
|
|
267
325
|
async close(): Promise<void> {
|
|
268
326
|
await connection.close();
|
|
@@ -355,27 +413,30 @@ async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Pr
|
|
|
355
413
|
},
|
|
356
414
|
async executeArrow(sqlText: string, options: SQLQueryOptions = {}) {
|
|
357
415
|
const result = await executeDuckDBBrowserQuery(connection, sqlText, options.parameters);
|
|
358
|
-
return
|
|
416
|
+
return tableFromIPC(result);
|
|
359
417
|
}
|
|
360
418
|
};
|
|
361
419
|
}
|
|
362
420
|
|
|
363
|
-
/**
|
|
421
|
+
/** Returns raw IPC so DuckDB's Arrow objects never cross into loaders.gl's Arrow runtime. */
|
|
364
422
|
async function executeDuckDBBrowserQuery(
|
|
365
|
-
connection:
|
|
423
|
+
connection: AsyncDuckDBConnection,
|
|
366
424
|
sqlText: string,
|
|
367
425
|
parameters?: SQLQueryOptions['parameters']
|
|
368
|
-
): Promise<
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
426
|
+
): Promise<Uint8Array> {
|
|
427
|
+
// Keep DuckDB's lower-level connection handle confined to this IPC boundary.
|
|
428
|
+
return await connection.useUnsafe(async (bindings, connectionId) => {
|
|
429
|
+
if (!parameters) {
|
|
430
|
+
return await bindings.runQuery(connectionId, sqlText);
|
|
431
|
+
}
|
|
432
|
+
const values = Array.isArray(parameters) ? parameters : Object.values(parameters);
|
|
433
|
+
const statementId = await bindings.createPrepared(connectionId, sqlText);
|
|
434
|
+
try {
|
|
435
|
+
return await bindings.runPrepared(connectionId, statementId, values);
|
|
436
|
+
} finally {
|
|
437
|
+
await bindings.closePrepared(connectionId, statementId);
|
|
438
|
+
}
|
|
439
|
+
});
|
|
379
440
|
}
|
|
380
441
|
|
|
381
442
|
function getDuckDBDatabasePath(url: string, options: SQLSourceOptions): string {
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {RecordBatch, Struct, Table, makeData, vectorFromArray} from 'apache-arrow';
|
|
6
|
+
import {
|
|
7
|
+
createQueryMetadata,
|
|
8
|
+
executeTableScanBatches,
|
|
9
|
+
makeTableScanBatch,
|
|
10
|
+
validateTableQueryLimit,
|
|
11
|
+
type TableQuerySource,
|
|
12
|
+
type TableQueryPushdownRequest,
|
|
13
|
+
type TableQueryReadOptions
|
|
14
|
+
} from '@loaders.gl/core/loader-utils';
|
|
15
|
+
import {filterArrowTable} from '@loaders.gl/core/query/arrow-execution';
|
|
16
|
+
import type {ArrowTableBatch, Field, Schema} from '@loaders.gl/core/schema';
|
|
17
|
+
import {convertSchemaToArrow, makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
|
|
18
|
+
import type {DuckDBSQLDataSource} from './duckdb-sql-source';
|
|
19
|
+
import type {SQLTableInfo} from './sql-types';
|
|
20
|
+
import {getQualifiedTableName, quoteSqlIdentifier} from './sql-utils';
|
|
21
|
+
|
|
22
|
+
/** Maps supported DuckDB scalars to their exact Arrow storage types. */
|
|
23
|
+
const SCALAR_TYPES: Readonly<Record<string, Field['type']>> = {
|
|
24
|
+
TINYINT: 'int8',
|
|
25
|
+
SMALLINT: 'int16',
|
|
26
|
+
INTEGER: 'int32',
|
|
27
|
+
BIGINT: 'int64',
|
|
28
|
+
FLOAT: 'float32',
|
|
29
|
+
REAL: 'float32',
|
|
30
|
+
DOUBLE: 'float64',
|
|
31
|
+
BOOLEAN: 'bool',
|
|
32
|
+
VARCHAR: 'utf8'
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/** Signed integer widths and corresponding DuckDB CAST targets. */
|
|
36
|
+
const INTEGER_TYPES = {int8: 'TINYINT', int16: 'SMALLINT', int32: 'INTEGER', int64: 'BIGINT'};
|
|
37
|
+
|
|
38
|
+
/** Returns the SQL type of a signed integer field when it is eligible for exact casts. */
|
|
39
|
+
function getIntegerType(type: Field['type']): string | undefined {
|
|
40
|
+
return typeof type === 'string' && Object.hasOwn(INTEGER_TYPES, type)
|
|
41
|
+
? INTEGER_TYPES[type as keyof typeof INTEGER_TYPES]
|
|
42
|
+
: undefined;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Reports whether all fields have exact Arrow storage representations. */
|
|
46
|
+
export function isDuckDBQuerySchema(schema: Schema): boolean {
|
|
47
|
+
return (
|
|
48
|
+
schema.fields.length > 0 &&
|
|
49
|
+
schema.fields.every(field => Object.hasOwn(SCALAR_TYPES, field.metadata?.sqlType || ''))
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Creates a reusable table handle with a schema snapshot and a lazy row reader. */
|
|
54
|
+
export function makeDuckDBTableQuerySource(
|
|
55
|
+
source: DuckDBSQLDataSource,
|
|
56
|
+
table: SQLTableInfo,
|
|
57
|
+
sourceSchema: Schema
|
|
58
|
+
): TableQuerySource {
|
|
59
|
+
if (!isDuckDBQuerySchema(sourceSchema))
|
|
60
|
+
throw new Error('Unsupported DuckDB table schema for the common query API');
|
|
61
|
+
const schema = {
|
|
62
|
+
...sourceSchema,
|
|
63
|
+
fields: sourceSchema.fields.map(field => {
|
|
64
|
+
const type = SCALAR_TYPES[field.metadata?.sqlType || ''];
|
|
65
|
+
if (!type) throw new Error(`Unsupported DuckDB query field type: ${field.metadata?.sqlType}`);
|
|
66
|
+
return {...field, type};
|
|
67
|
+
})
|
|
68
|
+
};
|
|
69
|
+
const tableName = `${getQualifiedTableName(table)} AS query_input`;
|
|
70
|
+
const ordered =
|
|
71
|
+
table.tableType === 'BASE TABLE' &&
|
|
72
|
+
!schema.fields.some(field => field.name.toLowerCase() === 'rowid');
|
|
73
|
+
const order = ordered ? ' ORDER BY query_input.rowid' : '';
|
|
74
|
+
const metadata = createQueryMetadata({
|
|
75
|
+
sourceType: 'duckdb-table',
|
|
76
|
+
queryType: 'table',
|
|
77
|
+
execution: {status: 'supported', method: 'read'},
|
|
78
|
+
schema,
|
|
79
|
+
queryCapabilities: {
|
|
80
|
+
table: {
|
|
81
|
+
projection: 'pushdown',
|
|
82
|
+
predicate: 'pushdown+residual',
|
|
83
|
+
cast: 'pushdown',
|
|
84
|
+
offset: ordered ? 'pushdown' : 'residual',
|
|
85
|
+
limit: ordered ? 'pushdown' : 'residual',
|
|
86
|
+
streaming: false,
|
|
87
|
+
cancellation: false
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
/** Reads typed rows without relying on inference for BIGINT, null-only, or empty columns. */
|
|
93
|
+
async function* readStatement(sql: string, outputSchema: Schema, signal?: AbortSignal) {
|
|
94
|
+
signal?.throwIfAborted();
|
|
95
|
+
const rows = await source.queryRows(sql, {signal});
|
|
96
|
+
signal?.throwIfAborted();
|
|
97
|
+
const arrowSchema = convertSchemaToArrow(outputSchema);
|
|
98
|
+
const vectors = arrowSchema.fields.map((field, index) =>
|
|
99
|
+
vectorFromArray(
|
|
100
|
+
rows.map(row => {
|
|
101
|
+
const value = row[field.name];
|
|
102
|
+
if (value == null) return null;
|
|
103
|
+
const type = outputSchema.fields[index].type;
|
|
104
|
+
if (type === 'int64') return BigInt(value as string | bigint);
|
|
105
|
+
if (type === 'float32' || type === 'float64') return Number(value);
|
|
106
|
+
return value;
|
|
107
|
+
}),
|
|
108
|
+
field.type
|
|
109
|
+
)
|
|
110
|
+
);
|
|
111
|
+
yield makeTableScanBatch({
|
|
112
|
+
shape: 'arrow-table',
|
|
113
|
+
schema: outputSchema,
|
|
114
|
+
data: new Table(arrowSchema, [
|
|
115
|
+
new RecordBatch(
|
|
116
|
+
arrowSchema,
|
|
117
|
+
makeData({
|
|
118
|
+
type: new Struct(arrowSchema.fields),
|
|
119
|
+
length: rows.length,
|
|
120
|
+
children: vectors.map(vector => vector.data[0])
|
|
121
|
+
})
|
|
122
|
+
)
|
|
123
|
+
])
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Compiles the exact subset without evaluating data or changing the connection. */
|
|
128
|
+
function plan(
|
|
129
|
+
query: Omit<TableQueryPushdownRequest, 'where'> & {
|
|
130
|
+
where?: TableQueryPushdownRequest['where'] | TableQueryReadOptions['predicate'];
|
|
131
|
+
}
|
|
132
|
+
) {
|
|
133
|
+
validateTableQueryLimit(query.limit);
|
|
134
|
+
if (query.offset !== undefined && (!Number.isSafeInteger(query.offset) || query.offset < 0)) {
|
|
135
|
+
throw new Error('DuckDB query offset must be a non-negative safe integer.');
|
|
136
|
+
}
|
|
137
|
+
const selections = query.select ?? schema.fields.map(field => field.name);
|
|
138
|
+
const columns: string[] = [];
|
|
139
|
+
const fields: Field[] = [];
|
|
140
|
+
for (const [index, selection] of selections.entries()) {
|
|
141
|
+
const expression =
|
|
142
|
+
typeof selection !== 'string' && 'expression' in selection
|
|
143
|
+
? selection.expression
|
|
144
|
+
: undefined;
|
|
145
|
+
if (typeof selection !== 'string' && 'array' in selection) return null;
|
|
146
|
+
const cast =
|
|
147
|
+
expression?.type === 'CallExpression' &&
|
|
148
|
+
expression.callee.type === 'Identifier' &&
|
|
149
|
+
expression.callee.name === 'cast'
|
|
150
|
+
? expression.arguments
|
|
151
|
+
: undefined;
|
|
152
|
+
const column =
|
|
153
|
+
typeof selection === 'string'
|
|
154
|
+
? selection
|
|
155
|
+
: 'column' in selection
|
|
156
|
+
? selection.column
|
|
157
|
+
: cast?.[0]?.type === 'Identifier'
|
|
158
|
+
? cast[0].name
|
|
159
|
+
: undefined;
|
|
160
|
+
const field = schema.fields.find(candidate => candidate.name === column);
|
|
161
|
+
if (!field) return null;
|
|
162
|
+
let sql = quoteSqlIdentifier(field.name);
|
|
163
|
+
const name =
|
|
164
|
+
typeof selection === 'string'
|
|
165
|
+
? selection
|
|
166
|
+
: (selection.as ?? (expression ? `column${index + 1}` : field.name));
|
|
167
|
+
if (expression) {
|
|
168
|
+
if (cast?.length !== 2 || cast[1].type !== 'Literal') return null;
|
|
169
|
+
const target = getExactCast(field, cast[1].value);
|
|
170
|
+
if (!target) return null;
|
|
171
|
+
sql = `CAST(${sql} AS ${target.sqlType})`;
|
|
172
|
+
fields.push({name, type: target.type, nullable: field.nullable});
|
|
173
|
+
} else {
|
|
174
|
+
fields.push({...field, name});
|
|
175
|
+
}
|
|
176
|
+
columns.push(`${sql} AS ${quoteSqlIdentifier(name)}`);
|
|
177
|
+
}
|
|
178
|
+
// DuckDB identifiers are case-insensitive, including quoted aliases.
|
|
179
|
+
if (new Set(fields.map(field => field.name.toLowerCase())).size !== fields.length) return null;
|
|
180
|
+
const predicate = query.where && compilePredicate(query.where, schema);
|
|
181
|
+
if (query.where && !predicate) return null;
|
|
182
|
+
if (!ordered && (query.offset !== undefined || query.limit !== undefined)) return null;
|
|
183
|
+
const sql =
|
|
184
|
+
`SELECT ${columns.join(', ')} FROM ${tableName}` +
|
|
185
|
+
(predicate ? ` WHERE ${predicate}` : '') +
|
|
186
|
+
order +
|
|
187
|
+
(query.limit === undefined ? '' : ` LIMIT ${query.limit}`) +
|
|
188
|
+
(query.offset === undefined ? '' : ` OFFSET ${query.offset}`);
|
|
189
|
+
const outputSchema = {...schema, fields};
|
|
190
|
+
return {
|
|
191
|
+
schema: outputSchema,
|
|
192
|
+
/** Executes the planned SQL when the consumer starts reading. */
|
|
193
|
+
async *read(options?: {signal?: AbortSignal}) {
|
|
194
|
+
for await (const {data} of readStatement(sql, outputSchema, options?.signal)) {
|
|
195
|
+
yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
return {
|
|
202
|
+
/** Returns metadata without issuing another catalog query. */
|
|
203
|
+
async getQueryMetadata() {
|
|
204
|
+
return metadata;
|
|
205
|
+
},
|
|
206
|
+
/** Accepts only exact whole queries. */
|
|
207
|
+
async planQueryPushdown(query, options) {
|
|
208
|
+
options?.signal?.throwIfAborted();
|
|
209
|
+
return plan(query);
|
|
210
|
+
},
|
|
211
|
+
/** Checks the exact CAST subset using the same compiler as whole-query execution. */
|
|
212
|
+
supportsQueryProjectionPushdown(selection) {
|
|
213
|
+
return Boolean(plan({select: [selection]}));
|
|
214
|
+
},
|
|
215
|
+
/** Executes simple reads natively when possible and applies other operators in Arrow. */
|
|
216
|
+
async *read(options = {}) {
|
|
217
|
+
const {columns, predicate, limit, offset, signal} = options;
|
|
218
|
+
const accepted = plan({select: columns, where: predicate, limit, offset});
|
|
219
|
+
if (accepted && limit !== 0) {
|
|
220
|
+
yield* accepted.read({signal});
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
for await (const batch of executeTableScanBatches(
|
|
224
|
+
readSignal => readStatement(`SELECT * FROM ${tableName}${order}`, schema, readSignal),
|
|
225
|
+
options,
|
|
226
|
+
{
|
|
227
|
+
filter: (batch, predicate, filterSignal) => {
|
|
228
|
+
if (!predicate) return batch;
|
|
229
|
+
const arrowBatch = batch as ArrowTableBatch;
|
|
230
|
+
return makeTableScanBatch({
|
|
231
|
+
...arrowBatch,
|
|
232
|
+
data: filterArrowTable(arrowBatch.data, predicate, {signal: filterSignal})
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
)) {
|
|
237
|
+
const {data} = batch as ArrowTableBatch;
|
|
238
|
+
yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Accepts signed integer identity and widening casts, which cannot round or overflow. */
|
|
245
|
+
function getExactCast(
|
|
246
|
+
field: Field,
|
|
247
|
+
target: unknown
|
|
248
|
+
): {type: Field['type']; sqlType: string} | null {
|
|
249
|
+
if (target === 'float64' && ['int8', 'int16', 'int32'].includes(String(field.type))) {
|
|
250
|
+
return {type: target, sqlType: 'DOUBLE'};
|
|
251
|
+
}
|
|
252
|
+
if (target !== 'int16' && target !== 'int32' && target !== 'int64') return null;
|
|
253
|
+
const sourceType = getIntegerType(field.type);
|
|
254
|
+
if (!sourceType || Number(String(field.type).slice(3)) > Number(target.slice(3))) return null;
|
|
255
|
+
return {type: target, sqlType: INTEGER_TYPES[target]};
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Compiles exact integer comparisons, including a single widening cast on the column. */
|
|
259
|
+
function compilePredicate(
|
|
260
|
+
predicate: NonNullable<TableQueryPushdownRequest['where'] | TableQueryReadOptions['predicate']>,
|
|
261
|
+
schema: Schema
|
|
262
|
+
): string | null {
|
|
263
|
+
if (!('op' in predicate) || !['=', '<>', '<', '<=', '>', '>='].includes(predicate.op))
|
|
264
|
+
return null;
|
|
265
|
+
const [property, value] = predicate.args;
|
|
266
|
+
if (!property || typeof property !== 'object' || !('property' in property)) return null;
|
|
267
|
+
const field = schema.fields.find(candidate => candidate.name === property.property);
|
|
268
|
+
if (!field || !getIntegerType(field.type)) return null;
|
|
269
|
+
if (typeof value !== 'bigint' && (typeof value !== 'number' || !Number.isSafeInteger(value)))
|
|
270
|
+
return null;
|
|
271
|
+
if (value < -(2n ** 63n) || value >= 2n ** 63n) return null;
|
|
272
|
+
const column = quoteSqlIdentifier(field.name);
|
|
273
|
+
if (!('casts' in property)) return `${column} ${predicate.op} ${value}`;
|
|
274
|
+
if (!Array.isArray(property.casts) || property.casts.length !== 1) return null;
|
|
275
|
+
const cast = getExactCast(field, property.casts[0]);
|
|
276
|
+
return cast ? `CAST(${column} AS ${cast.sqlType}) ${predicate.op} ${value}` : null;
|
|
277
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -10,18 +10,22 @@ export {
|
|
|
10
10
|
ARROW_TABLE_QUERY_CAPABILITIES,
|
|
11
11
|
explainArrowTableQuery,
|
|
12
12
|
queryArrowTable
|
|
13
|
-
} from '
|
|
14
|
-
export {parseSQLPredicate} from '
|
|
15
|
-
export {bindSQLPredicate} from '
|
|
13
|
+
} from '@loaders.gl/core/scan/query-arrow-table';
|
|
14
|
+
export {parseSQLPredicate} from '@loaders.gl/core/scan/parse-sql-predicate';
|
|
15
|
+
export {bindSQLPredicate} from '@loaders.gl/core/scan/bind-sql-predicate';
|
|
16
16
|
export {compileSQLTableQuery} from './compile-table-query';
|
|
17
|
-
export {
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
export {
|
|
18
|
+
explainTableQuery,
|
|
19
|
+
getSQLPredicateColumnNames,
|
|
20
|
+
planTableQuery
|
|
21
|
+
} from '@loaders.gl/core/scan/table-query';
|
|
22
|
+
export {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from '@loaders.gl/core/scan/table-query-capabilities';
|
|
23
|
+
export {isSQLPredicateParameter} from '@loaders.gl/core/scan/sql-predicate-types';
|
|
20
24
|
export {
|
|
21
25
|
isSQLPredicate,
|
|
22
26
|
SQL_PREDICATE_JSON_SCHEMA,
|
|
23
27
|
validateSQLPredicate
|
|
24
|
-
} from '
|
|
28
|
+
} from '@loaders.gl/core/scan/sql-predicate-schema';
|
|
25
29
|
|
|
26
30
|
export type {
|
|
27
31
|
BoundSQLPredicate,
|
|
@@ -37,7 +41,7 @@ export type {
|
|
|
37
41
|
SQLPredicateProperty,
|
|
38
42
|
SQLPredicateScalar,
|
|
39
43
|
SQLPredicateValue
|
|
40
|
-
} from '
|
|
44
|
+
} from '@loaders.gl/core/scan/sql-predicate-types';
|
|
41
45
|
|
|
42
46
|
export type {
|
|
43
47
|
SQLAdapter,
|
|
@@ -50,12 +54,13 @@ export type {
|
|
|
50
54
|
SQLParameterValues,
|
|
51
55
|
SQLQuery,
|
|
52
56
|
SQLQueryOptions,
|
|
57
|
+
SQLArrowInsertOptions,
|
|
53
58
|
SQLSchemaInfo,
|
|
54
59
|
SQLSourceOptions,
|
|
55
60
|
SQLTableInfo
|
|
56
61
|
} from './sql-types';
|
|
57
62
|
|
|
58
|
-
export type {ArrowQueryOptions} from '
|
|
63
|
+
export type {ArrowQueryOptions} from '@loaders.gl/core/scan/query-arrow-table';
|
|
59
64
|
export type {ArrowTableSourceOptions} from './arrow-table-source';
|
|
60
65
|
export type {
|
|
61
66
|
CompiledSQLTableQuery,
|
|
@@ -67,13 +72,16 @@ export type {
|
|
|
67
72
|
RelationalAggregate,
|
|
68
73
|
RelationalExpression,
|
|
69
74
|
RelationalOrderKey
|
|
70
|
-
} from '@loaders.gl/loader-utils';
|
|
75
|
+
} from '@loaders.gl/core/loader-utils';
|
|
71
76
|
export type {
|
|
72
77
|
TableQueryFilterStep,
|
|
73
78
|
TableQueryLimitStep,
|
|
79
|
+
TableQuery,
|
|
80
|
+
TableSelection,
|
|
81
|
+
TableQueryExecutionOptions,
|
|
74
82
|
TableQueryOptions,
|
|
75
83
|
TableQueryPlan,
|
|
76
84
|
TableQueryPlanStep,
|
|
77
85
|
TableQueryProjectStep,
|
|
78
86
|
TableQueryScanStep
|
|
79
|
-
} from '
|
|
87
|
+
} from '@loaders.gl/core/scan/table-query';
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import type {Schema} from '@loaders.gl/schema';
|
|
6
|
-
import type {SourceLoader, CoreAPI} from '@loaders.gl/loader-utils';
|
|
5
|
+
import type {Schema} from '@loaders.gl/core/schema';
|
|
6
|
+
import type {SourceLoader, CoreAPI} from '@loaders.gl/core/loader-utils';
|
|
7
7
|
|
|
8
8
|
import {SQLDataSource, SQL_SOURCE_DEFAULT_OPTIONS} from './sql-source';
|
|
9
9
|
import type {
|
|
@@ -62,7 +62,7 @@ export const SnowflakeSQLSource = {
|
|
|
62
62
|
},
|
|
63
63
|
defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
|
|
64
64
|
testURL: (url: string): boolean => /^(sql\+snowflake|snowflake):\/\//i.test(url),
|
|
65
|
-
|
|
65
|
+
createDataSourceSync: (
|
|
66
66
|
data: string,
|
|
67
67
|
options: SQLSourceOptions,
|
|
68
68
|
coreApi?: CoreAPI
|