@loaders.gl/sql 5.0.0-alpha.9 → 5.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/LICENSE +2 -30
  2. package/README.md +21 -12
  3. package/THIRD-PARTY-NOTICES.md +17 -0
  4. package/dist/arrow-table-source.d.ts +9 -9
  5. package/dist/arrow-table-source.d.ts.map +1 -1
  6. package/dist/arrow-table-source.js +11 -7
  7. package/dist/arrow-table-source.js.map +1 -1
  8. package/dist/compile-table-query.d.ts +3 -3
  9. package/dist/compile-table-query.d.ts.map +1 -1
  10. package/dist/compile-table-query.js +2 -2
  11. package/dist/compile-table-query.js.map +1 -1
  12. package/dist/duckdb-sql-source.d.ts +36 -32
  13. package/dist/duckdb-sql-source.d.ts.map +1 -1
  14. package/dist/duckdb-sql-source.js +61 -17
  15. package/dist/duckdb-sql-source.js.map +1 -1
  16. package/dist/duckdb-table-query.d.ts +9 -0
  17. package/dist/duckdb-table-query.d.ts.map +1 -0
  18. package/dist/duckdb-table-query.js +242 -0
  19. package/dist/duckdb-table-query.js.map +1 -0
  20. package/dist/index.cjs +1843 -1473
  21. package/dist/index.cjs.map +3 -3
  22. package/dist/index.d.ts +12 -12
  23. package/dist/index.d.ts.map +1 -1
  24. package/dist/index.js +7 -7
  25. package/dist/index.js.map +1 -1
  26. package/dist/snowflake-sql-source.d.ts +18 -31
  27. package/dist/snowflake-sql-source.d.ts.map +1 -1
  28. package/dist/snowflake-sql-source.js +2 -2
  29. package/dist/snowflake-sql-source.js.map +1 -1
  30. package/dist/sql-source.d.ts +24 -11
  31. package/dist/sql-source.d.ts.map +1 -1
  32. package/dist/sql-source.js +35 -3
  33. package/dist/sql-source.js.map +1 -1
  34. package/dist/sql-types.d.ts +15 -3
  35. package/dist/sql-types.d.ts.map +1 -1
  36. package/dist/sql-utils.d.ts +2 -3
  37. package/dist/sql-utils.d.ts.map +1 -1
  38. package/dist/sql-utils.js +1 -34
  39. package/dist/sql-utils.js.map +1 -1
  40. package/package.json +8 -28
  41. package/src/arrow-table-source.ts +17 -16
  42. package/src/compile-table-query.ts +4 -4
  43. package/src/duckdb-sql-source.ts +81 -20
  44. package/src/duckdb-table-query.ts +277 -0
  45. package/src/index.ts +19 -11
  46. package/src/snowflake-sql-source.ts +3 -3
  47. package/src/sql-source.ts +46 -7
  48. package/src/sql-types.ts +16 -3
  49. package/src/sql-utils.ts +2 -40
  50. package/dist/arrow-query.cjs +0 -1404
  51. package/dist/arrow-query.cjs.map +0 -6
  52. package/dist/arrow-query.d.ts +0 -5
  53. package/dist/arrow-query.d.ts.map +0 -1
  54. package/dist/arrow-query.js +0 -6
  55. package/dist/arrow-query.js.map +0 -1
  56. package/dist/bind-sql-predicate.d.ts +0 -4
  57. package/dist/bind-sql-predicate.d.ts.map +0 -1
  58. package/dist/bind-sql-predicate.js +0 -11
  59. package/dist/bind-sql-predicate.js.map +0 -1
  60. package/dist/parse-sql-predicate.d.ts +0 -10
  61. package/dist/parse-sql-predicate.d.ts.map +0 -1
  62. package/dist/parse-sql-predicate.js +0 -322
  63. package/dist/parse-sql-predicate.js.map +0 -1
  64. package/dist/query-arrow-table.d.ts +0 -83
  65. package/dist/query-arrow-table.d.ts.map +0 -1
  66. package/dist/query-arrow-table.js +0 -663
  67. package/dist/query-arrow-table.js.map +0 -1
  68. package/dist/sql-predicate-schema.d.ts +0 -156
  69. package/dist/sql-predicate-schema.d.ts.map +0 -1
  70. package/dist/sql-predicate-schema.js +0 -226
  71. package/dist/sql-predicate-schema.js.map +0 -1
  72. package/dist/sql-predicate-types.d.ts +0 -45
  73. package/dist/sql-predicate-types.d.ts.map +0 -1
  74. package/dist/sql-predicate-types.js +0 -9
  75. package/dist/sql-predicate-types.js.map +0 -1
  76. package/dist/sql-predicate-zod.cjs +0 -54
  77. package/dist/sql-predicate-zod.cjs.map +0 -6
  78. package/dist/sql-predicate-zod.d.ts +0 -11
  79. package/dist/sql-predicate-zod.d.ts.map +0 -1
  80. package/dist/sql-predicate-zod.js +0 -45
  81. package/dist/sql-predicate-zod.js.map +0 -1
  82. package/dist/sql-predicate.cjs +0 -607
  83. package/dist/sql-predicate.cjs.map +0 -6
  84. package/dist/sql-predicate.d.ts +0 -6
  85. package/dist/sql-predicate.d.ts.map +0 -1
  86. package/dist/sql-predicate.js +0 -8
  87. package/dist/sql-predicate.js.map +0 -1
  88. package/dist/table-query-capabilities.d.ts +0 -6
  89. package/dist/table-query-capabilities.d.ts.map +0 -1
  90. package/dist/table-query-capabilities.js +0 -26
  91. package/dist/table-query-capabilities.js.map +0 -1
  92. package/dist/table-query.cjs +0 -286
  93. package/dist/table-query.cjs.map +0 -6
  94. package/dist/table-query.d.ts +0 -29
  95. package/dist/table-query.d.ts.map +0 -1
  96. package/dist/table-query.js +0 -30
  97. package/dist/table-query.js.map +0 -1
  98. package/src/arrow-query.ts +0 -17
  99. package/src/bind-sql-predicate.ts +0 -21
  100. package/src/parse-sql-predicate.ts +0 -381
  101. package/src/query-arrow-table.ts +0 -846
  102. package/src/sql-predicate-schema.ts +0 -251
  103. package/src/sql-predicate-types.ts +0 -87
  104. package/src/sql-predicate-zod.ts +0 -52
  105. package/src/sql-predicate.ts +0 -28
  106. package/src/table-query-capabilities.ts +0 -29
  107. package/src/table-query.ts +0 -78
@@ -2,26 +2,26 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
+ import type {RecordBatch, Table} from 'apache-arrow';
6
+ import {makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
5
7
  import type {
6
8
  DataSourceOptions,
7
9
  ScanQueryMetadata,
8
10
  ScanQueryMetadataOptions
9
- } from '@loaders.gl/loader-utils';
11
+ } from '@loaders.gl/core/loader-utils';
10
12
  import {
11
13
  createScanQueryMetadata,
12
14
  DataSource,
13
- makeTableScanBatch,
14
15
  type ScanColumnRole
15
- } from '@loaders.gl/loader-utils';
16
- import {convertArrowToSchema} from '@loaders.gl/schema-utils';
17
- import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
16
+ } from '@loaders.gl/core/loader-utils';
17
+ import {convertArrowToSchema} from '@loaders.gl/core/schema-utils';
18
18
  import {
19
19
  ARROW_TABLE_QUERY_CAPABILITIES,
20
20
  explainArrowTableQuery,
21
21
  queryArrowTable
22
- } from './query-arrow-table';
23
- import type {ArrowQueryOptions} from './query-arrow-table';
24
- import type {TableQueryExplain} from './table-query';
22
+ } from '@loaders.gl/core/scan/query-arrow-table';
23
+ import type {ArrowQueryOptions} from '@loaders.gl/core/scan/query-arrow-table';
24
+ import type {TableQueryExplain} from '@loaders.gl/core/scan/table-query';
25
25
 
26
26
  /** In-memory source options; the source owns the supplied Arrow table. */
27
27
  export type ArrowTableSourceOptions = DataSourceOptions;
@@ -33,16 +33,16 @@ export type ArrowTableSourceOptions = DataSourceOptions;
33
33
  * practice, and query execution delegates to the existing Arrow executor without ingesting data
34
34
  * into DuckDB or another intermediate representation.
35
35
  */
36
- export class ArrowTableSource extends DataSource<ArrowTable, ArrowTableSourceOptions> {
36
+ export class ArrowTableSource extends DataSource<Table, ArrowTableSourceOptions> {
37
37
  /** Creates a source over an existing Arrow table without copying its vectors. */
38
- constructor(data: ArrowTable, options: ArrowTableSourceOptions = {}) {
38
+ constructor(data: Table, options: ArrowTableSourceOptions = {}) {
39
39
  super(data, options);
40
40
  }
41
41
 
42
42
  /** Discovers query-visible fields and capabilities without evaluating any rows. */
43
43
  async getQueryMetadata(options: ScanQueryMetadataOptions = {}): Promise<ScanQueryMetadata> {
44
44
  throwIfAborted(options.signal);
45
- const schema = convertArrowToSchema(this.data.data.schema);
45
+ const schema = convertArrowToSchema(this.data.schema);
46
46
  return createScanQueryMetadata({
47
47
  sourceType: 'arrow-table',
48
48
  queryType: 'table',
@@ -50,12 +50,12 @@ export class ArrowTableSource extends DataSource<ArrowTable, ArrowTableSourceOpt
50
50
  schema,
51
51
  capabilities: {table: ARROW_TABLE_QUERY_CAPABILITIES},
52
52
  columnRoles: getArrowColumnRoles(schema.fields.map(field => field.name)),
53
- statistics: {rowCount: this.data.data.numRows}
53
+ statistics: {rowCount: this.data.numRows}
54
54
  });
55
55
  }
56
56
 
57
57
  /** Executes a portable query and returns an Arrow table result. */
58
- query(options: ArrowQueryOptions = {}): ArrowTable {
58
+ query(options: ArrowQueryOptions = {}): Table {
59
59
  return queryArrowTable(this.data, options);
60
60
  }
61
61
 
@@ -65,9 +65,10 @@ export class ArrowTableSource extends DataSource<ArrowTable, ArrowTableSourceOpt
65
65
  }
66
66
 
67
67
  /** Executes a query as one bounded Arrow batch. */
68
- async *read(options: ArrowQueryOptions = {}): AsyncIterableIterator<ArrowTableBatch> {
69
- const result = this.query(options);
70
- yield makeTableScanBatch(result);
68
+ async *read(options: ArrowQueryOptions = {}): AsyncIterableIterator<RecordBatch> {
69
+ const data = this.query(options);
70
+ if (data.batches.length) yield* data.batches;
71
+ else yield makeArrowMetadataBatch(data.schema);
71
72
  }
72
73
  }
73
74
 
@@ -2,20 +2,20 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import {validateSQLPredicate} from './sql-predicate-schema';
5
+ import {validateSQLPredicate} from '@loaders.gl/core/scan/sql-predicate-schema';
6
6
  import {
7
7
  isSQLPredicateParameter,
8
8
  type SQLPredicate,
9
9
  type SQLPredicateParameterValues,
10
10
  type SQLPredicateScalar,
11
11
  type SQLPredicateValue
12
- } from './sql-predicate-types';
13
- import type {TableQueryOptions} from './table-query';
12
+ } from '@loaders.gl/core/scan/sql-predicate-types';
13
+ import type {TableQueryOptions} from '@loaders.gl/core/scan/table-query';
14
14
  import type {
15
15
  RelationalAggregate,
16
16
  RelationalExpression,
17
17
  RelationalOrderKey
18
- } from '@loaders.gl/loader-utils';
18
+ } from '@loaders.gl/core/loader-utils';
19
19
 
20
20
  /** A named child relation using the SQL module's property-oriented predicate AST. */
21
21
  type SQLChildQuery = Readonly<{
@@ -2,10 +2,18 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {Schema} from '@loaders.gl/schema';
6
- import type {SourceLoader, CoreAPI} from '@loaders.gl/loader-utils';
7
- import {convertArrowToTable} from '@loaders.gl/schema-utils';
5
+ import {tableFromIPC, tableToIPC, type Table} from 'apache-arrow';
6
+ import type {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
7
+ import type {Schema} from '@loaders.gl/core/schema';
8
+ import type {
9
+ SourceLoader,
10
+ CoreAPI,
11
+ TableQuerySource,
12
+ TableMetadata
13
+ } from '@loaders.gl/core/loader-utils';
14
+ import {convertArrowToTable} from '@loaders.gl/core/schema-utils';
8
15
 
16
+ import {makeDuckDBTableQuerySource, isDuckDBQuerySchema} from './duckdb-table-query';
9
17
  import {SQLDataSource, SQL_SOURCE_DEFAULT_OPTIONS} from './sql-source';
10
18
  import type {
11
19
  SQLAdapter,
@@ -14,7 +22,8 @@ import type {
14
22
  SQLColumnInfo,
15
23
  SQLQueryOptions,
16
24
  SQLSourceOptions,
17
- SQLTableInfo
25
+ SQLTableInfo,
26
+ SQLArrowInsertOptions
18
27
  } from './sql-types';
19
28
  import {
20
29
  convertRowsToArrowTable,
@@ -66,7 +75,7 @@ export const DuckDBSQLSource = {
66
75
  },
67
76
  defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
68
77
  testURL: (url: string): boolean => /^duckdb:\/\//i.test(url),
69
- createDataSource: (
78
+ createDataSourceSync: (
70
79
  data: string,
71
80
  options: SQLSourceOptions,
72
81
  coreApi?: CoreAPI
@@ -75,6 +84,47 @@ export const DuckDBSQLSource = {
75
84
 
76
85
  /** SQLDataSource specialization for DuckDB. */
77
86
  export class DuckDBSQLDataSource extends SQLDataSource {
87
+ /** Resolves a reusable common query handle for a named DuckDB table. */
88
+ async getTable(name: string, options: {signal?: AbortSignal} = {}): Promise<TableQuerySource> {
89
+ const table = await this.resolveTable(name, options.signal);
90
+ const schema = await this.getTableSchema(table);
91
+ options.signal?.throwIfAborted();
92
+ return makeDuckDBTableQuerySource(this, table, schema);
93
+ }
94
+
95
+ /** Reports whether the table's schema can be represented exactly by the common reader. */
96
+ override async getTableMetadata(
97
+ name: string,
98
+ options: {signal?: AbortSignal} = {}
99
+ ): Promise<TableMetadata> {
100
+ const metadata = await super.getTableMetadata(name, options);
101
+ return {
102
+ ...metadata,
103
+ queryable: Boolean(metadata.schema && isDuckDBQuerySchema(metadata.schema))
104
+ };
105
+ }
106
+
107
+ /** Lists table names without claiming compatibility before schema discovery. */
108
+ override async getTablesMetadata(
109
+ options: {signal?: AbortSignal} = {}
110
+ ): Promise<readonly TableMetadata[]> {
111
+ return (await super.getTablesMetadata(options)).map(({name}) => ({name}));
112
+ }
113
+
114
+ /** Uploads an Arrow table through IPC. Supported by the browser DuckDB-Wasm adapter. */
115
+ async insertArrowTable(table: Table, options: SQLArrowInsertOptions): Promise<void> {
116
+ await this.insertArrowIPC(tableToIPC(table, 'stream'), options);
117
+ }
118
+
119
+ /** Uploads a complete Arrow IPC stream to a browser DuckDB-Wasm table. */
120
+ async insertArrowIPC(bytes: Uint8Array, options: SQLArrowInsertOptions): Promise<void> {
121
+ const adapter = await this.getAdapter();
122
+ if (!adapter.insertArrowIPC) {
123
+ throw new Error('This DuckDB adapter does not support Arrow IPC uploads.');
124
+ }
125
+ await adapter.insertArrowIPC(bytes, options);
126
+ }
127
+
78
128
  constructor(data: string, options: SQLSourceOptions, coreApi?: CoreAPI) {
79
129
  super(
80
130
  data,
@@ -253,7 +303,7 @@ async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Pr
253
303
  options: SQLQueryOptions = {}
254
304
  ): Promise<Record<string, unknown>[]> => {
255
305
  const result = await executeDuckDBBrowserQuery(connection, sqlText, options.parameters);
256
- return convertArrowToTable(result, 'object-row-table').data;
306
+ return convertArrowToTable(tableFromIPC(result), 'object-row-table').data;
257
307
  };
258
308
 
259
309
  return {
@@ -263,6 +313,14 @@ async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Pr
263
313
  runtime: 'browser',
264
314
  isDynamic: true
265
315
  },
316
+ async insertArrowIPC(bytes, options): Promise<void> {
317
+ // DuckDB transfers the buffer to its worker; retain the caller's IPC bytes.
318
+ await connection.insertArrowFromIPCStream(new Uint8Array(bytes), {
319
+ name: options.tableName,
320
+ schema: options.schemaName,
321
+ create: options.create ?? true
322
+ });
323
+ },
266
324
  async connect(): Promise<void> {},
267
325
  async close(): Promise<void> {
268
326
  await connection.close();
@@ -355,27 +413,30 @@ async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Pr
355
413
  },
356
414
  async executeArrow(sqlText: string, options: SQLQueryOptions = {}) {
357
415
  const result = await executeDuckDBBrowserQuery(connection, sqlText, options.parameters);
358
- return convertArrowToTable(result, 'arrow-table');
416
+ return tableFromIPC(result);
359
417
  }
360
418
  };
361
419
  }
362
420
 
363
- /** Executes a DuckDB-Wasm query directly or through a prepared statement when values are bound. */
421
+ /** Returns raw IPC so DuckDB's Arrow objects never cross into loaders.gl's Arrow runtime. */
364
422
  async function executeDuckDBBrowserQuery(
365
- connection: any,
423
+ connection: AsyncDuckDBConnection,
366
424
  sqlText: string,
367
425
  parameters?: SQLQueryOptions['parameters']
368
- ): Promise<any> {
369
- if (!parameters) {
370
- return await connection.query(sqlText);
371
- }
372
- const values = Array.isArray(parameters) ? parameters : Object.values(parameters);
373
- const statement = await connection.prepare(sqlText);
374
- try {
375
- return await statement.query(...values);
376
- } finally {
377
- await statement.close();
378
- }
426
+ ): Promise<Uint8Array> {
427
+ // Keep DuckDB's lower-level connection handle confined to this IPC boundary.
428
+ return await connection.useUnsafe(async (bindings, connectionId) => {
429
+ if (!parameters) {
430
+ return await bindings.runQuery(connectionId, sqlText);
431
+ }
432
+ const values = Array.isArray(parameters) ? parameters : Object.values(parameters);
433
+ const statementId = await bindings.createPrepared(connectionId, sqlText);
434
+ try {
435
+ return await bindings.runPrepared(connectionId, statementId, values);
436
+ } finally {
437
+ await bindings.closePrepared(connectionId, statementId);
438
+ }
439
+ });
379
440
  }
380
441
 
381
442
  function getDuckDBDatabasePath(url: string, options: SQLSourceOptions): string {
@@ -0,0 +1,277 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import {RecordBatch, Struct, Table, makeData, vectorFromArray} from 'apache-arrow';
6
+ import {
7
+ createQueryMetadata,
8
+ executeTableScanBatches,
9
+ makeTableScanBatch,
10
+ validateTableQueryLimit,
11
+ type TableQuerySource,
12
+ type TableQueryPushdownRequest,
13
+ type TableQueryReadOptions
14
+ } from '@loaders.gl/core/loader-utils';
15
+ import {filterArrowTable} from '@loaders.gl/core/query/arrow-execution';
16
+ import type {ArrowTableBatch, Field, Schema} from '@loaders.gl/core/schema';
17
+ import {convertSchemaToArrow, makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
18
+ import type {DuckDBSQLDataSource} from './duckdb-sql-source';
19
+ import type {SQLTableInfo} from './sql-types';
20
+ import {getQualifiedTableName, quoteSqlIdentifier} from './sql-utils';
21
+
22
+ /** Maps supported DuckDB scalars to their exact Arrow storage types. */
23
+ const SCALAR_TYPES: Readonly<Record<string, Field['type']>> = {
24
+ TINYINT: 'int8',
25
+ SMALLINT: 'int16',
26
+ INTEGER: 'int32',
27
+ BIGINT: 'int64',
28
+ FLOAT: 'float32',
29
+ REAL: 'float32',
30
+ DOUBLE: 'float64',
31
+ BOOLEAN: 'bool',
32
+ VARCHAR: 'utf8'
33
+ };
34
+
35
+ /** Signed integer widths and corresponding DuckDB CAST targets. */
36
+ const INTEGER_TYPES = {int8: 'TINYINT', int16: 'SMALLINT', int32: 'INTEGER', int64: 'BIGINT'};
37
+
38
+ /** Returns the SQL type of a signed integer field when it is eligible for exact casts. */
39
+ function getIntegerType(type: Field['type']): string | undefined {
40
+ return typeof type === 'string' && Object.hasOwn(INTEGER_TYPES, type)
41
+ ? INTEGER_TYPES[type as keyof typeof INTEGER_TYPES]
42
+ : undefined;
43
+ }
44
+
45
+ /** Reports whether all fields have exact Arrow storage representations. */
46
+ export function isDuckDBQuerySchema(schema: Schema): boolean {
47
+ return (
48
+ schema.fields.length > 0 &&
49
+ schema.fields.every(field => Object.hasOwn(SCALAR_TYPES, field.metadata?.sqlType || ''))
50
+ );
51
+ }
52
+
53
+ /** Creates a reusable table handle with a schema snapshot and a lazy row reader. */
54
+ export function makeDuckDBTableQuerySource(
55
+ source: DuckDBSQLDataSource,
56
+ table: SQLTableInfo,
57
+ sourceSchema: Schema
58
+ ): TableQuerySource {
59
+ if (!isDuckDBQuerySchema(sourceSchema))
60
+ throw new Error('Unsupported DuckDB table schema for the common query API');
61
+ const schema = {
62
+ ...sourceSchema,
63
+ fields: sourceSchema.fields.map(field => {
64
+ const type = SCALAR_TYPES[field.metadata?.sqlType || ''];
65
+ if (!type) throw new Error(`Unsupported DuckDB query field type: ${field.metadata?.sqlType}`);
66
+ return {...field, type};
67
+ })
68
+ };
69
+ const tableName = `${getQualifiedTableName(table)} AS query_input`;
70
+ const ordered =
71
+ table.tableType === 'BASE TABLE' &&
72
+ !schema.fields.some(field => field.name.toLowerCase() === 'rowid');
73
+ const order = ordered ? ' ORDER BY query_input.rowid' : '';
74
+ const metadata = createQueryMetadata({
75
+ sourceType: 'duckdb-table',
76
+ queryType: 'table',
77
+ execution: {status: 'supported', method: 'read'},
78
+ schema,
79
+ queryCapabilities: {
80
+ table: {
81
+ projection: 'pushdown',
82
+ predicate: 'pushdown+residual',
83
+ cast: 'pushdown',
84
+ offset: ordered ? 'pushdown' : 'residual',
85
+ limit: ordered ? 'pushdown' : 'residual',
86
+ streaming: false,
87
+ cancellation: false
88
+ }
89
+ }
90
+ });
91
+
92
+ /** Reads typed rows without relying on inference for BIGINT, null-only, or empty columns. */
93
+ async function* readStatement(sql: string, outputSchema: Schema, signal?: AbortSignal) {
94
+ signal?.throwIfAborted();
95
+ const rows = await source.queryRows(sql, {signal});
96
+ signal?.throwIfAborted();
97
+ const arrowSchema = convertSchemaToArrow(outputSchema);
98
+ const vectors = arrowSchema.fields.map((field, index) =>
99
+ vectorFromArray(
100
+ rows.map(row => {
101
+ const value = row[field.name];
102
+ if (value == null) return null;
103
+ const type = outputSchema.fields[index].type;
104
+ if (type === 'int64') return BigInt(value as string | bigint);
105
+ if (type === 'float32' || type === 'float64') return Number(value);
106
+ return value;
107
+ }),
108
+ field.type
109
+ )
110
+ );
111
+ yield makeTableScanBatch({
112
+ shape: 'arrow-table',
113
+ schema: outputSchema,
114
+ data: new Table(arrowSchema, [
115
+ new RecordBatch(
116
+ arrowSchema,
117
+ makeData({
118
+ type: new Struct(arrowSchema.fields),
119
+ length: rows.length,
120
+ children: vectors.map(vector => vector.data[0])
121
+ })
122
+ )
123
+ ])
124
+ });
125
+ }
126
+
127
+ /** Compiles the exact subset without evaluating data or changing the connection. */
128
+ function plan(
129
+ query: Omit<TableQueryPushdownRequest, 'where'> & {
130
+ where?: TableQueryPushdownRequest['where'] | TableQueryReadOptions['predicate'];
131
+ }
132
+ ) {
133
+ validateTableQueryLimit(query.limit);
134
+ if (query.offset !== undefined && (!Number.isSafeInteger(query.offset) || query.offset < 0)) {
135
+ throw new Error('DuckDB query offset must be a non-negative safe integer.');
136
+ }
137
+ const selections = query.select ?? schema.fields.map(field => field.name);
138
+ const columns: string[] = [];
139
+ const fields: Field[] = [];
140
+ for (const [index, selection] of selections.entries()) {
141
+ const expression =
142
+ typeof selection !== 'string' && 'expression' in selection
143
+ ? selection.expression
144
+ : undefined;
145
+ if (typeof selection !== 'string' && 'array' in selection) return null;
146
+ const cast =
147
+ expression?.type === 'CallExpression' &&
148
+ expression.callee.type === 'Identifier' &&
149
+ expression.callee.name === 'cast'
150
+ ? expression.arguments
151
+ : undefined;
152
+ const column =
153
+ typeof selection === 'string'
154
+ ? selection
155
+ : 'column' in selection
156
+ ? selection.column
157
+ : cast?.[0]?.type === 'Identifier'
158
+ ? cast[0].name
159
+ : undefined;
160
+ const field = schema.fields.find(candidate => candidate.name === column);
161
+ if (!field) return null;
162
+ let sql = quoteSqlIdentifier(field.name);
163
+ const name =
164
+ typeof selection === 'string'
165
+ ? selection
166
+ : (selection.as ?? (expression ? `column${index + 1}` : field.name));
167
+ if (expression) {
168
+ if (cast?.length !== 2 || cast[1].type !== 'Literal') return null;
169
+ const target = getExactCast(field, cast[1].value);
170
+ if (!target) return null;
171
+ sql = `CAST(${sql} AS ${target.sqlType})`;
172
+ fields.push({name, type: target.type, nullable: field.nullable});
173
+ } else {
174
+ fields.push({...field, name});
175
+ }
176
+ columns.push(`${sql} AS ${quoteSqlIdentifier(name)}`);
177
+ }
178
+ // DuckDB identifiers are case-insensitive, including quoted aliases.
179
+ if (new Set(fields.map(field => field.name.toLowerCase())).size !== fields.length) return null;
180
+ const predicate = query.where && compilePredicate(query.where, schema);
181
+ if (query.where && !predicate) return null;
182
+ if (!ordered && (query.offset !== undefined || query.limit !== undefined)) return null;
183
+ const sql =
184
+ `SELECT ${columns.join(', ')} FROM ${tableName}` +
185
+ (predicate ? ` WHERE ${predicate}` : '') +
186
+ order +
187
+ (query.limit === undefined ? '' : ` LIMIT ${query.limit}`) +
188
+ (query.offset === undefined ? '' : ` OFFSET ${query.offset}`);
189
+ const outputSchema = {...schema, fields};
190
+ return {
191
+ schema: outputSchema,
192
+ /** Executes the planned SQL when the consumer starts reading. */
193
+ async *read(options?: {signal?: AbortSignal}) {
194
+ for await (const {data} of readStatement(sql, outputSchema, options?.signal)) {
195
+ yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
196
+ }
197
+ }
198
+ };
199
+ }
200
+
201
+ return {
202
+ /** Returns metadata without issuing another catalog query. */
203
+ async getQueryMetadata() {
204
+ return metadata;
205
+ },
206
+ /** Accepts only exact whole queries. */
207
+ async planQueryPushdown(query, options) {
208
+ options?.signal?.throwIfAborted();
209
+ return plan(query);
210
+ },
211
+ /** Checks the exact CAST subset using the same compiler as whole-query execution. */
212
+ supportsQueryProjectionPushdown(selection) {
213
+ return Boolean(plan({select: [selection]}));
214
+ },
215
+ /** Executes simple reads natively when possible and applies other operators in Arrow. */
216
+ async *read(options = {}) {
217
+ const {columns, predicate, limit, offset, signal} = options;
218
+ const accepted = plan({select: columns, where: predicate, limit, offset});
219
+ if (accepted && limit !== 0) {
220
+ yield* accepted.read({signal});
221
+ return;
222
+ }
223
+ for await (const batch of executeTableScanBatches(
224
+ readSignal => readStatement(`SELECT * FROM ${tableName}${order}`, schema, readSignal),
225
+ options,
226
+ {
227
+ filter: (batch, predicate, filterSignal) => {
228
+ if (!predicate) return batch;
229
+ const arrowBatch = batch as ArrowTableBatch;
230
+ return makeTableScanBatch({
231
+ ...arrowBatch,
232
+ data: filterArrowTable(arrowBatch.data, predicate, {signal: filterSignal})
233
+ });
234
+ }
235
+ }
236
+ )) {
237
+ const {data} = batch as ArrowTableBatch;
238
+ yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
239
+ }
240
+ }
241
+ };
242
+ }
243
+
244
+ /** Accepts signed integer identity and widening casts, which cannot round or overflow. */
245
+ function getExactCast(
246
+ field: Field,
247
+ target: unknown
248
+ ): {type: Field['type']; sqlType: string} | null {
249
+ if (target === 'float64' && ['int8', 'int16', 'int32'].includes(String(field.type))) {
250
+ return {type: target, sqlType: 'DOUBLE'};
251
+ }
252
+ if (target !== 'int16' && target !== 'int32' && target !== 'int64') return null;
253
+ const sourceType = getIntegerType(field.type);
254
+ if (!sourceType || Number(String(field.type).slice(3)) > Number(target.slice(3))) return null;
255
+ return {type: target, sqlType: INTEGER_TYPES[target]};
256
+ }
257
+
258
+ /** Compiles exact integer comparisons, including a single widening cast on the column. */
259
+ function compilePredicate(
260
+ predicate: NonNullable<TableQueryPushdownRequest['where'] | TableQueryReadOptions['predicate']>,
261
+ schema: Schema
262
+ ): string | null {
263
+ if (!('op' in predicate) || !['=', '<>', '<', '<=', '>', '>='].includes(predicate.op))
264
+ return null;
265
+ const [property, value] = predicate.args;
266
+ if (!property || typeof property !== 'object' || !('property' in property)) return null;
267
+ const field = schema.fields.find(candidate => candidate.name === property.property);
268
+ if (!field || !getIntegerType(field.type)) return null;
269
+ if (typeof value !== 'bigint' && (typeof value !== 'number' || !Number.isSafeInteger(value)))
270
+ return null;
271
+ if (value < -(2n ** 63n) || value >= 2n ** 63n) return null;
272
+ const column = quoteSqlIdentifier(field.name);
273
+ if (!('casts' in property)) return `${column} ${predicate.op} ${value}`;
274
+ if (!Array.isArray(property.casts) || property.casts.length !== 1) return null;
275
+ const cast = getExactCast(field, property.casts[0]);
276
+ return cast ? `CAST(${column} AS ${cast.sqlType}) ${predicate.op} ${value}` : null;
277
+ }
package/src/index.ts CHANGED
@@ -10,18 +10,22 @@ export {
10
10
  ARROW_TABLE_QUERY_CAPABILITIES,
11
11
  explainArrowTableQuery,
12
12
  queryArrowTable
13
- } from './query-arrow-table';
14
- export {parseSQLPredicate} from './parse-sql-predicate';
15
- export {bindSQLPredicate} from './bind-sql-predicate';
13
+ } from '@loaders.gl/core/scan/query-arrow-table';
14
+ export {parseSQLPredicate} from '@loaders.gl/core/scan/parse-sql-predicate';
15
+ export {bindSQLPredicate} from '@loaders.gl/core/scan/bind-sql-predicate';
16
16
  export {compileSQLTableQuery} from './compile-table-query';
17
- export {explainTableQuery, getSQLPredicateColumnNames, planTableQuery} from './table-query';
18
- export {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from './table-query-capabilities';
19
- export {isSQLPredicateParameter} from './sql-predicate-types';
17
+ export {
18
+ explainTableQuery,
19
+ getSQLPredicateColumnNames,
20
+ planTableQuery
21
+ } from '@loaders.gl/core/scan/table-query';
22
+ export {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from '@loaders.gl/core/scan/table-query-capabilities';
23
+ export {isSQLPredicateParameter} from '@loaders.gl/core/scan/sql-predicate-types';
20
24
  export {
21
25
  isSQLPredicate,
22
26
  SQL_PREDICATE_JSON_SCHEMA,
23
27
  validateSQLPredicate
24
- } from './sql-predicate-schema';
28
+ } from '@loaders.gl/core/scan/sql-predicate-schema';
25
29
 
26
30
  export type {
27
31
  BoundSQLPredicate,
@@ -37,7 +41,7 @@ export type {
37
41
  SQLPredicateProperty,
38
42
  SQLPredicateScalar,
39
43
  SQLPredicateValue
40
- } from './sql-predicate-types';
44
+ } from '@loaders.gl/core/scan/sql-predicate-types';
41
45
 
42
46
  export type {
43
47
  SQLAdapter,
@@ -50,12 +54,13 @@ export type {
50
54
  SQLParameterValues,
51
55
  SQLQuery,
52
56
  SQLQueryOptions,
57
+ SQLArrowInsertOptions,
53
58
  SQLSchemaInfo,
54
59
  SQLSourceOptions,
55
60
  SQLTableInfo
56
61
  } from './sql-types';
57
62
 
58
- export type {ArrowQueryOptions} from './query-arrow-table';
63
+ export type {ArrowQueryOptions} from '@loaders.gl/core/scan/query-arrow-table';
59
64
  export type {ArrowTableSourceOptions} from './arrow-table-source';
60
65
  export type {
61
66
  CompiledSQLTableQuery,
@@ -67,13 +72,16 @@ export type {
67
72
  RelationalAggregate,
68
73
  RelationalExpression,
69
74
  RelationalOrderKey
70
- } from '@loaders.gl/loader-utils';
75
+ } from '@loaders.gl/core/loader-utils';
71
76
  export type {
72
77
  TableQueryFilterStep,
73
78
  TableQueryLimitStep,
79
+ TableQuery,
80
+ TableSelection,
81
+ TableQueryExecutionOptions,
74
82
  TableQueryOptions,
75
83
  TableQueryPlan,
76
84
  TableQueryPlanStep,
77
85
  TableQueryProjectStep,
78
86
  TableQueryScanStep
79
- } from './table-query';
87
+ } from '@loaders.gl/core/scan/table-query';
@@ -2,8 +2,8 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {Schema} from '@loaders.gl/schema';
6
- import type {SourceLoader, CoreAPI} from '@loaders.gl/loader-utils';
5
+ import type {Schema} from '@loaders.gl/core/schema';
6
+ import type {SourceLoader, CoreAPI} from '@loaders.gl/core/loader-utils';
7
7
 
8
8
  import {SQLDataSource, SQL_SOURCE_DEFAULT_OPTIONS} from './sql-source';
9
9
  import type {
@@ -62,7 +62,7 @@ export const SnowflakeSQLSource = {
62
62
  },
63
63
  defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
64
64
  testURL: (url: string): boolean => /^(sql\+snowflake|snowflake):\/\//i.test(url),
65
- createDataSource: (
65
+ createDataSourceSync: (
66
66
  data: string,
67
67
  options: SQLSourceOptions,
68
68
  coreApi?: CoreAPI