@loaders.gl/sql 5.0.0-beta.1 → 5.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,277 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import {RecordBatch, Struct, Table, makeData, vectorFromArray} from 'apache-arrow';
6
+ import {
7
+ createQueryMetadata,
8
+ executeTableScanBatches,
9
+ makeTableScanBatch,
10
+ validateTableQueryLimit,
11
+ type TableQuerySource,
12
+ type TableQueryPushdownRequest,
13
+ type TableQueryReadOptions
14
+ } from '@loaders.gl/core/loader-utils';
15
+ import {filterArrowTable} from '@loaders.gl/core/query/arrow-execution';
16
+ import type {ArrowTableBatch, Field, Schema} from '@loaders.gl/core/schema';
17
+ import {convertSchemaToArrow, makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
18
+ import type {DuckDBSQLDataSource} from './duckdb-sql-source';
19
+ import type {SQLTableInfo} from './sql-types';
20
+ import {getQualifiedTableName, quoteSqlIdentifier} from './sql-utils';
21
+
22
+ /** Maps supported DuckDB scalars to their exact Arrow storage types. */
23
+ const SCALAR_TYPES: Readonly<Record<string, Field['type']>> = {
24
+ TINYINT: 'int8',
25
+ SMALLINT: 'int16',
26
+ INTEGER: 'int32',
27
+ BIGINT: 'int64',
28
+ FLOAT: 'float32',
29
+ REAL: 'float32',
30
+ DOUBLE: 'float64',
31
+ BOOLEAN: 'bool',
32
+ VARCHAR: 'utf8'
33
+ };
34
+
35
+ /** Signed integer widths and corresponding DuckDB CAST targets. */
36
+ const INTEGER_TYPES = {int8: 'TINYINT', int16: 'SMALLINT', int32: 'INTEGER', int64: 'BIGINT'};
37
+
38
+ /** Returns the SQL type of a signed integer field when it is eligible for exact casts. */
39
+ function getIntegerType(type: Field['type']): string | undefined {
40
+ return typeof type === 'string' && Object.hasOwn(INTEGER_TYPES, type)
41
+ ? INTEGER_TYPES[type as keyof typeof INTEGER_TYPES]
42
+ : undefined;
43
+ }
44
+
45
+ /** Reports whether all fields have exact Arrow storage representations. */
46
+ export function isDuckDBQuerySchema(schema: Schema): boolean {
47
+ return (
48
+ schema.fields.length > 0 &&
49
+ schema.fields.every(field => Object.hasOwn(SCALAR_TYPES, field.metadata?.sqlType || ''))
50
+ );
51
+ }
52
+
53
+ /** Creates a reusable table handle with a schema snapshot and a lazy row reader. */
54
+ export function makeDuckDBTableQuerySource(
55
+ source: DuckDBSQLDataSource,
56
+ table: SQLTableInfo,
57
+ sourceSchema: Schema
58
+ ): TableQuerySource {
59
+ if (!isDuckDBQuerySchema(sourceSchema))
60
+ throw new Error('Unsupported DuckDB table schema for the common query API');
61
+ const schema = {
62
+ ...sourceSchema,
63
+ fields: sourceSchema.fields.map(field => {
64
+ const type = SCALAR_TYPES[field.metadata?.sqlType || ''];
65
+ if (!type) throw new Error(`Unsupported DuckDB query field type: ${field.metadata?.sqlType}`);
66
+ return {...field, type};
67
+ })
68
+ };
69
+ const tableName = `${getQualifiedTableName(table)} AS query_input`;
70
+ const ordered =
71
+ table.tableType === 'BASE TABLE' &&
72
+ !schema.fields.some(field => field.name.toLowerCase() === 'rowid');
73
+ const order = ordered ? ' ORDER BY query_input.rowid' : '';
74
+ const metadata = createQueryMetadata({
75
+ sourceType: 'duckdb-table',
76
+ queryType: 'table',
77
+ execution: {status: 'supported', method: 'read'},
78
+ schema,
79
+ queryCapabilities: {
80
+ table: {
81
+ projection: 'pushdown',
82
+ predicate: 'pushdown+residual',
83
+ cast: 'pushdown',
84
+ offset: ordered ? 'pushdown' : 'residual',
85
+ limit: ordered ? 'pushdown' : 'residual',
86
+ streaming: false,
87
+ cancellation: false
88
+ }
89
+ }
90
+ });
91
+
92
+ /** Reads typed rows without relying on inference for BIGINT, null-only, or empty columns. */
93
+ async function* readStatement(sql: string, outputSchema: Schema, signal?: AbortSignal) {
94
+ signal?.throwIfAborted();
95
+ const rows = await source.queryRows(sql, {signal});
96
+ signal?.throwIfAborted();
97
+ const arrowSchema = convertSchemaToArrow(outputSchema);
98
+ const vectors = arrowSchema.fields.map((field, index) =>
99
+ vectorFromArray(
100
+ rows.map(row => {
101
+ const value = row[field.name];
102
+ if (value == null) return null;
103
+ const type = outputSchema.fields[index].type;
104
+ if (type === 'int64') return BigInt(value as string | bigint);
105
+ if (type === 'float32' || type === 'float64') return Number(value);
106
+ return value;
107
+ }),
108
+ field.type
109
+ )
110
+ );
111
+ yield makeTableScanBatch({
112
+ shape: 'arrow-table',
113
+ schema: outputSchema,
114
+ data: new Table(arrowSchema, [
115
+ new RecordBatch(
116
+ arrowSchema,
117
+ makeData({
118
+ type: new Struct(arrowSchema.fields),
119
+ length: rows.length,
120
+ children: vectors.map(vector => vector.data[0])
121
+ })
122
+ )
123
+ ])
124
+ });
125
+ }
126
+
127
+ /** Compiles the exact subset without evaluating data or changing the connection. */
128
+ function plan(
129
+ query: Omit<TableQueryPushdownRequest, 'where'> & {
130
+ where?: TableQueryPushdownRequest['where'] | TableQueryReadOptions['predicate'];
131
+ }
132
+ ) {
133
+ validateTableQueryLimit(query.limit);
134
+ if (query.offset !== undefined && (!Number.isSafeInteger(query.offset) || query.offset < 0)) {
135
+ throw new Error('DuckDB query offset must be a non-negative safe integer.');
136
+ }
137
+ const selections = query.select ?? schema.fields.map(field => field.name);
138
+ const columns: string[] = [];
139
+ const fields: Field[] = [];
140
+ for (const [index, selection] of selections.entries()) {
141
+ const expression =
142
+ typeof selection !== 'string' && 'expression' in selection
143
+ ? selection.expression
144
+ : undefined;
145
+ if (typeof selection !== 'string' && 'array' in selection) return null;
146
+ const cast =
147
+ expression?.type === 'CallExpression' &&
148
+ expression.callee.type === 'Identifier' &&
149
+ expression.callee.name === 'cast'
150
+ ? expression.arguments
151
+ : undefined;
152
+ const column =
153
+ typeof selection === 'string'
154
+ ? selection
155
+ : 'column' in selection
156
+ ? selection.column
157
+ : cast?.[0]?.type === 'Identifier'
158
+ ? cast[0].name
159
+ : undefined;
160
+ const field = schema.fields.find(candidate => candidate.name === column);
161
+ if (!field) return null;
162
+ let sql = quoteSqlIdentifier(field.name);
163
+ const name =
164
+ typeof selection === 'string'
165
+ ? selection
166
+ : (selection.as ?? (expression ? `column${index + 1}` : field.name));
167
+ if (expression) {
168
+ if (cast?.length !== 2 || cast[1].type !== 'Literal') return null;
169
+ const target = getExactCast(field, cast[1].value);
170
+ if (!target) return null;
171
+ sql = `CAST(${sql} AS ${target.sqlType})`;
172
+ fields.push({name, type: target.type, nullable: field.nullable});
173
+ } else {
174
+ fields.push({...field, name});
175
+ }
176
+ columns.push(`${sql} AS ${quoteSqlIdentifier(name)}`);
177
+ }
178
+ // DuckDB identifiers are case-insensitive, including quoted aliases.
179
+ if (new Set(fields.map(field => field.name.toLowerCase())).size !== fields.length) return null;
180
+ const predicate = query.where && compilePredicate(query.where, schema);
181
+ if (query.where && !predicate) return null;
182
+ if (!ordered && (query.offset !== undefined || query.limit !== undefined)) return null;
183
+ const sql =
184
+ `SELECT ${columns.join(', ')} FROM ${tableName}` +
185
+ (predicate ? ` WHERE ${predicate}` : '') +
186
+ order +
187
+ (query.limit === undefined ? '' : ` LIMIT ${query.limit}`) +
188
+ (query.offset === undefined ? '' : ` OFFSET ${query.offset}`);
189
+ const outputSchema = {...schema, fields};
190
+ return {
191
+ schema: outputSchema,
192
+ /** Executes the planned SQL when the consumer starts reading. */
193
+ async *read(options?: {signal?: AbortSignal}) {
194
+ for await (const {data} of readStatement(sql, outputSchema, options?.signal)) {
195
+ yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
196
+ }
197
+ }
198
+ };
199
+ }
200
+
201
+ return {
202
+ /** Returns metadata without issuing another catalog query. */
203
+ async getQueryMetadata() {
204
+ return metadata;
205
+ },
206
+ /** Accepts only exact whole queries. */
207
+ async planQueryPushdown(query, options) {
208
+ options?.signal?.throwIfAborted();
209
+ return plan(query);
210
+ },
211
+ /** Checks the exact CAST subset using the same compiler as whole-query execution. */
212
+ supportsQueryProjectionPushdown(selection) {
213
+ return Boolean(plan({select: [selection]}));
214
+ },
215
+ /** Executes simple reads natively when possible and applies other operators in Arrow. */
216
+ async *read(options = {}) {
217
+ const {columns, predicate, limit, offset, signal} = options;
218
+ const accepted = plan({select: columns, where: predicate, limit, offset});
219
+ if (accepted && limit !== 0) {
220
+ yield* accepted.read({signal});
221
+ return;
222
+ }
223
+ for await (const batch of executeTableScanBatches(
224
+ readSignal => readStatement(`SELECT * FROM ${tableName}${order}`, schema, readSignal),
225
+ options,
226
+ {
227
+ filter: (batch, predicate, filterSignal) => {
228
+ if (!predicate) return batch;
229
+ const arrowBatch = batch as ArrowTableBatch;
230
+ return makeTableScanBatch({
231
+ ...arrowBatch,
232
+ data: filterArrowTable(arrowBatch.data, predicate, {signal: filterSignal})
233
+ });
234
+ }
235
+ }
236
+ )) {
237
+ const {data} = batch as ArrowTableBatch;
238
+ yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
239
+ }
240
+ }
241
+ };
242
+ }
243
+
244
+ /** Accepts signed integer identity and widening casts, which cannot round or overflow. */
245
+ function getExactCast(
246
+ field: Field,
247
+ target: unknown
248
+ ): {type: Field['type']; sqlType: string} | null {
249
+ if (target === 'float64' && ['int8', 'int16', 'int32'].includes(String(field.type))) {
250
+ return {type: target, sqlType: 'DOUBLE'};
251
+ }
252
+ if (target !== 'int16' && target !== 'int32' && target !== 'int64') return null;
253
+ const sourceType = getIntegerType(field.type);
254
+ if (!sourceType || Number(String(field.type).slice(3)) > Number(target.slice(3))) return null;
255
+ return {type: target, sqlType: INTEGER_TYPES[target]};
256
+ }
257
+
258
+ /** Compiles exact integer comparisons, including a single widening cast on the column. */
259
+ function compilePredicate(
260
+ predicate: NonNullable<TableQueryPushdownRequest['where'] | TableQueryReadOptions['predicate']>,
261
+ schema: Schema
262
+ ): string | null {
263
+ if (!('op' in predicate) || !['=', '<>', '<', '<=', '>', '>='].includes(predicate.op))
264
+ return null;
265
+ const [property, value] = predicate.args;
266
+ if (!property || typeof property !== 'object' || !('property' in property)) return null;
267
+ const field = schema.fields.find(candidate => candidate.name === property.property);
268
+ if (!field || !getIntegerType(field.type)) return null;
269
+ if (typeof value !== 'bigint' && (typeof value !== 'number' || !Number.isSafeInteger(value)))
270
+ return null;
271
+ if (value < -(2n ** 63n) || value >= 2n ** 63n) return null;
272
+ const column = quoteSqlIdentifier(field.name);
273
+ if (!('casts' in property)) return `${column} ${predicate.op} ${value}`;
274
+ if (!Array.isArray(property.casts) || property.casts.length !== 1) return null;
275
+ const cast = getExactCast(field, property.casts[0]);
276
+ return cast ? `CAST(${column} AS ${cast.sqlType}) ${predicate.op} ${value}` : null;
277
+ }
package/src/index.ts CHANGED
@@ -76,6 +76,9 @@ export type {
76
76
  export type {
77
77
  TableQueryFilterStep,
78
78
  TableQueryLimitStep,
79
+ TableQuery,
80
+ TableSelection,
81
+ TableQueryExecutionOptions,
79
82
  TableQueryOptions,
80
83
  TableQueryPlan,
81
84
  TableQueryPlanStep,
@@ -62,7 +62,7 @@ export const SnowflakeSQLSource = {
62
62
  },
63
63
  defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
64
64
  testURL: (url: string): boolean => /^(sql\+snowflake|snowflake):\/\//i.test(url),
65
- createDataSource: (
65
+ createDataSourceSync: (
66
66
  data: string,
67
67
  options: SQLSourceOptions,
68
68
  coreApi?: CoreAPI
package/src/sql-source.ts CHANGED
@@ -2,9 +2,10 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {ArrowTable, Schema} from '@loaders.gl/core/schema';
5
+ import type {Table} from 'apache-arrow';
6
+ import type {Schema} from '@loaders.gl/core/schema';
6
7
  import {DataSource} from '@loaders.gl/core/loader-utils';
7
- import type {CoreAPI, RequiredOptions} from '@loaders.gl/core/loader-utils';
8
+ import type {CoreAPI, RequiredOptions, TableMetadata} from '@loaders.gl/core/loader-utils';
8
9
 
9
10
  import type {
10
11
  SQLAdapter,
@@ -108,6 +109,36 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
108
109
  return await adapter.listTables(options);
109
110
  }
110
111
 
112
+ /** Lists stable native table identifiers without fetching any column schemas or data rows. */
113
+ async getTablesMetadata(options: {signal?: AbortSignal} = {}): Promise<readonly TableMetadata[]> {
114
+ options.signal?.throwIfAborted();
115
+ const tables = await this.listTables();
116
+ options.signal?.throwIfAborted();
117
+ return tables.map(table => ({name: getTableMetadataName(table), queryable: false}));
118
+ }
119
+
120
+ /** Discovers one native table schema; ambiguous unqualified names are rejected. */
121
+ async getTableMetadata(
122
+ name: string,
123
+ options: {signal?: AbortSignal} = {}
124
+ ): Promise<TableMetadata> {
125
+ const table = await this.resolveTable(name, options.signal);
126
+ const schema = await this.getTableSchema(table);
127
+ options.signal?.throwIfAborted();
128
+ return {name: getTableMetadataName(table), schema, queryable: false};
129
+ }
130
+
131
+ /** Resolves a qualified or unique unqualified table name. */
132
+ protected async resolveTable(name: string, signal?: AbortSignal): Promise<SQLTableInfo> {
133
+ signal?.throwIfAborted();
134
+ const tables = await this.listTables();
135
+ signal?.throwIfAborted();
136
+ const exact = tables.filter(table => getTableMetadataName(table) === name);
137
+ const matches = exact.length ? exact : tables.filter(table => table.tableName === name);
138
+ if (matches.length !== 1) throw new Error(`Unknown or ambiguous SQL table: ${name}`);
139
+ return matches[0];
140
+ }
141
+
111
142
  /** Returns the schema for a specific SQL table. */
112
143
  async getTableSchema(options: {
113
144
  catalogName?: string;
@@ -128,8 +159,8 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
128
159
  return await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
129
160
  }
130
161
 
131
- /** Executes a SQL query and returns a loaders.gl Arrow table. */
132
- async queryArrow(query: SQLQuery, options: SQLQueryOptions = {}): Promise<ArrowTable> {
162
+ /** Executes a SQL query and returns a native Arrow table. */
163
+ async queryArrow(query: SQLQuery, options: SQLQueryOptions = {}): Promise<Table> {
133
164
  const adapter = await this.getAdapter();
134
165
  const compiledQuery = this.compileQuery(query, options);
135
166
  if (adapter.executeArrow) {
@@ -226,3 +257,11 @@ function getSQLTableQueryDialect(sourceType: string): SQLTableQueryDialect {
226
257
  if (sourceType === 'snowflake-sql') return 'snowflake';
227
258
  throw new Error(`Portable table queries are not supported for SQL source type "${sourceType}".`);
228
259
  }
260
+
261
+ /** Preserves native namespace components as one unambiguous provider-defined loaded name. */
262
+ function getTableMetadataName(table: SQLTableInfo): string {
263
+ return [table.catalogName, table.schemaName, table.tableName]
264
+ .filter((name): name is string => typeof name === 'string' && name.length > 0)
265
+ .map(name => `"${name.replaceAll('"', '""')}"`)
266
+ .join('.');
267
+ }
package/src/sql-types.ts CHANGED
@@ -2,7 +2,8 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {ArrowTable, Schema} from '@loaders.gl/core/schema';
5
+ import type {Table} from 'apache-arrow';
6
+ import type {Schema} from '@loaders.gl/core/schema';
6
7
  import type {CoreAPI, DataSourceOptions} from '@loaders.gl/core/loader-utils';
7
8
  import type {SQLTableQuery} from './compile-table-query';
8
9
 
@@ -90,7 +91,7 @@ export interface SQLAdapter {
90
91
  executeRows(sql: string, options?: SQLQueryOptions): Promise<Record<string, unknown>[]>;
91
92
  /** Uploads an Arrow IPC stream without passing Arrow runtime objects. */
92
93
  insertArrowIPC?(bytes: Uint8Array, options: SQLArrowInsertOptions): Promise<void>;
93
- executeArrow?(sql: string, options?: SQLQueryOptions): Promise<ArrowTable>;
94
+ executeArrow?(sql: string, options?: SQLQueryOptions): Promise<Table>;
94
95
  }
95
96
 
96
97
  /** Context passed to SQL adapter factories during source initialization. */