@loaders.gl/sql 5.0.0-alpha.2 → 5.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +60 -2
- package/dist/arrow-query.cjs +1373 -0
- package/dist/arrow-query.cjs.map +6 -0
- package/dist/arrow-query.d.ts +5 -0
- package/dist/arrow-query.d.ts.map +1 -0
- package/dist/arrow-query.js +6 -0
- package/dist/arrow-query.js.map +1 -0
- package/dist/arrow-table-source.d.ts +27 -0
- package/dist/arrow-table-source.d.ts.map +1 -0
- package/dist/arrow-table-source.js +79 -0
- package/dist/arrow-table-source.js.map +1 -0
- package/dist/bind-sql-predicate.d.ts +4 -0
- package/dist/bind-sql-predicate.d.ts.map +1 -0
- package/dist/bind-sql-predicate.js +11 -0
- package/dist/bind-sql-predicate.js.map +1 -0
- package/dist/compile-table-query.d.ts +55 -0
- package/dist/compile-table-query.d.ts.map +1 -0
- package/dist/compile-table-query.js +255 -0
- package/dist/compile-table-query.js.map +1 -0
- package/dist/duckdb-sql-source.d.ts.map +1 -1
- package/dist/duckdb-sql-source.js +21 -7
- package/dist/duckdb-sql-source.js.map +1 -1
- package/dist/index.cjs +1746 -307
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +14 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/parse-sql-predicate.d.ts.map +1 -1
- package/dist/parse-sql-predicate.js +10 -4
- package/dist/parse-sql-predicate.js.map +1 -1
- package/dist/query-arrow-table.d.ts +83 -0
- package/dist/query-arrow-table.d.ts.map +1 -0
- package/dist/query-arrow-table.js +660 -0
- package/dist/query-arrow-table.js.map +1 -0
- package/dist/snowflake-sql-source.d.ts.map +1 -1
- package/dist/snowflake-sql-source.js +8 -5
- package/dist/snowflake-sql-source.js.map +1 -1
- package/dist/sql-predicate-schema.d.ts +17 -2
- package/dist/sql-predicate-schema.d.ts.map +1 -1
- package/dist/sql-predicate-schema.js +25 -4
- package/dist/sql-predicate-schema.js.map +1 -1
- package/dist/sql-predicate-types.d.ts +22 -32
- package/dist/sql-predicate-types.d.ts.map +1 -1
- package/dist/sql-predicate-types.js +5 -1
- package/dist/sql-predicate-types.js.map +1 -1
- package/dist/sql-predicate-zod.cjs +3 -2
- package/dist/sql-predicate-zod.cjs.map +1 -1
- package/dist/sql-predicate-zod.d.ts.map +1 -1
- package/dist/sql-predicate-zod.js +5 -2
- package/dist/sql-predicate-zod.js.map +1 -1
- package/dist/sql-predicate.cjs +114 -7
- package/dist/sql-predicate.cjs.map +2 -2
- package/dist/sql-predicate.d.ts +3 -1
- package/dist/sql-predicate.d.ts.map +1 -1
- package/dist/sql-predicate.js +2 -0
- package/dist/sql-predicate.js.map +1 -1
- package/dist/sql-source.d.ts +16 -3
- package/dist/sql-source.d.ts.map +1 -1
- package/dist/sql-source.js +34 -5
- package/dist/sql-source.js.map +1 -1
- package/dist/sql-types.d.ts +3 -0
- package/dist/sql-types.d.ts.map +1 -1
- package/dist/table-query-capabilities.d.ts +6 -0
- package/dist/table-query-capabilities.d.ts.map +1 -0
- package/dist/table-query-capabilities.js +26 -0
- package/dist/table-query-capabilities.js.map +1 -0
- package/dist/table-query.cjs +286 -0
- package/dist/table-query.cjs.map +6 -0
- package/dist/table-query.d.ts +29 -0
- package/dist/table-query.d.ts.map +1 -0
- package/dist/table-query.js +30 -0
- package/dist/table-query.js.map +1 -0
- package/package.json +15 -5
- package/src/arrow-query.ts +17 -0
- package/src/arrow-table-source.ts +111 -0
- package/src/bind-sql-predicate.ts +21 -0
- package/src/compile-table-query.ts +354 -0
- package/src/duckdb-sql-source.ts +30 -6
- package/src/index.ts +39 -0
- package/src/parse-sql-predicate.ts +21 -5
- package/src/query-arrow-table.ts +843 -0
- package/src/snowflake-sql-source.ts +7 -4
- package/src/sql-predicate-schema.ts +30 -5
- package/src/sql-predicate-types.ts +48 -32
- package/src/sql-predicate-zod.ts +5 -2
- package/src/sql-predicate.ts +6 -0
- package/src/sql-source.ts +40 -5
- package/src/sql-types.ts +4 -0
- package/src/table-query-capabilities.ts +29 -0
- package/src/table-query.ts +78 -0
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {validateSQLPredicate} from './sql-predicate-schema';
|
|
6
|
+
import {
|
|
7
|
+
isSQLPredicateParameter,
|
|
8
|
+
type SQLPredicate,
|
|
9
|
+
type SQLPredicateParameterValues,
|
|
10
|
+
type SQLPredicateScalar,
|
|
11
|
+
type SQLPredicateValue
|
|
12
|
+
} from './sql-predicate-types';
|
|
13
|
+
import type {TableQueryOptions} from './table-query';
|
|
14
|
+
import type {
|
|
15
|
+
RelationalAggregate,
|
|
16
|
+
RelationalExpression,
|
|
17
|
+
RelationalOrderKey
|
|
18
|
+
} from '@loaders.gl/loader-utils';
|
|
19
|
+
|
|
20
|
+
/** A named child relation using the SQL module's property-oriented predicate AST. */
|
|
21
|
+
type SQLChildQuery = Readonly<{
|
|
22
|
+
/** Logical source name resolved by the SQL data source. */
|
|
23
|
+
source: string;
|
|
24
|
+
/** Optional query applied to the child relation before combining it. */
|
|
25
|
+
query?: TableQueryOptions;
|
|
26
|
+
}>;
|
|
27
|
+
|
|
28
|
+
/** SQL dialects supported by the portable table-query compiler. */
|
|
29
|
+
export type SQLTableQueryDialect = 'duckdb' | 'snowflake';
|
|
30
|
+
|
|
31
|
+
/** A table-bound portable query accepted by SQL-backed data sources. */
|
|
32
|
+
export type SQLTableQuery = TableQueryOptions &
|
|
33
|
+
Readonly<{
|
|
34
|
+
/** Unqualified table name. */
|
|
35
|
+
tableName: string;
|
|
36
|
+
/** Optional schema containing the table. */
|
|
37
|
+
schemaName?: string;
|
|
38
|
+
/** Optional catalog containing the schema. */
|
|
39
|
+
catalogName?: string;
|
|
40
|
+
/** Computed columns evaluated before projection. */
|
|
41
|
+
expressions?: readonly RelationalExpression[];
|
|
42
|
+
/** Stable ordering applied before the global limit. */
|
|
43
|
+
orderBy?: readonly RelationalOrderKey[];
|
|
44
|
+
/** Grouping keys for aggregate output. */
|
|
45
|
+
groupBy?: readonly string[];
|
|
46
|
+
/** Aggregate output definitions. */
|
|
47
|
+
aggregates?: readonly RelationalAggregate[];
|
|
48
|
+
/** Additional tables concatenated with `UNION ALL`. */
|
|
49
|
+
union?: readonly SQLChildQuery[];
|
|
50
|
+
/** Optional equi-join child table and key mapping. */
|
|
51
|
+
join?: Readonly<{child: SQLChildQuery; left: string; right: string}>;
|
|
52
|
+
}>;
|
|
53
|
+
|
|
54
|
+
/** Options controlling portable table-query compilation. */
|
|
55
|
+
export type SQLTableQueryCompilerOptions = Readonly<{
|
|
56
|
+
/** Target SQL dialect. */
|
|
57
|
+
dialect: SQLTableQueryDialect;
|
|
58
|
+
/** Values used to resolve named parameter nodes retained in the predicate AST. */
|
|
59
|
+
parameters?: SQLPredicateParameterValues;
|
|
60
|
+
}>;
|
|
61
|
+
|
|
62
|
+
/** Parameterized SQL emitted from a portable table query. */
|
|
63
|
+
export type CompiledSQLTableQuery = Readonly<{
|
|
64
|
+
/** SQL statement containing positional placeholders. */
|
|
65
|
+
sql: string;
|
|
66
|
+
/** Values bound to placeholders in statement order. */
|
|
67
|
+
parameters: SQLPredicateScalar[];
|
|
68
|
+
}>;
|
|
69
|
+
|
|
70
|
+
type SQLCompilationContext = {
|
|
71
|
+
dialect: SQLTableQueryDialect;
|
|
72
|
+
namedParameters?: SQLPredicateParameterValues;
|
|
73
|
+
parameters: SQLPredicateScalar[];
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
/** Compiles a portable table query into parameterized DuckDB or Snowflake SQL. */
|
|
77
|
+
export function compileSQLTableQuery(
|
|
78
|
+
query: SQLTableQuery,
|
|
79
|
+
options: SQLTableQueryCompilerOptions
|
|
80
|
+
): CompiledSQLTableQuery {
|
|
81
|
+
validateSQLTableQuery(query);
|
|
82
|
+
const context: SQLCompilationContext = {
|
|
83
|
+
dialect: options.dialect,
|
|
84
|
+
namedParameters: options.parameters,
|
|
85
|
+
parameters: []
|
|
86
|
+
};
|
|
87
|
+
return {
|
|
88
|
+
sql: compileTableStatement(query, context),
|
|
89
|
+
parameters: context.parameters
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Compiles one table statement while sharing parameter bindings with child relations. */
|
|
94
|
+
function compileTableStatement(query: SQLTableQuery, context: SQLCompilationContext): string {
|
|
95
|
+
const projection = compileProjection(query);
|
|
96
|
+
const table = [query.catalogName, query.schemaName, query.tableName]
|
|
97
|
+
.filter((identifier): identifier is string => identifier !== undefined)
|
|
98
|
+
.map(quoteSQLIdentifier)
|
|
99
|
+
.join('.');
|
|
100
|
+
const fromClause = query.join ? compileJoinFromClause(query, table, context) : `FROM ${table}`;
|
|
101
|
+
const clauses = [`SELECT ${projection}`, fromClause];
|
|
102
|
+
if (query.predicate) {
|
|
103
|
+
clauses.push(`WHERE ${compileSQLPredicate(query.predicate, context)}`);
|
|
104
|
+
}
|
|
105
|
+
if (query.groupBy?.length) {
|
|
106
|
+
clauses.push(`GROUP BY ${query.groupBy.map(quoteSQLIdentifier).join(', ')}`);
|
|
107
|
+
}
|
|
108
|
+
let sql = clauses.join('\n');
|
|
109
|
+
for (const child of query.union || []) {
|
|
110
|
+
const childQuery: SQLTableQuery = {
|
|
111
|
+
tableName: child.source,
|
|
112
|
+
...(child.query || {})
|
|
113
|
+
};
|
|
114
|
+
validateSQLTableQuery(childQuery);
|
|
115
|
+
const childSQL = compileTableStatement(childQuery, context);
|
|
116
|
+
const branchSQL = hasBranchLocalModifiers(childQuery) ? `(${childSQL})` : childSQL;
|
|
117
|
+
sql = `${sql}\nUNION ALL\n${branchSQL}`;
|
|
118
|
+
}
|
|
119
|
+
if (query.orderBy?.length) sql += `\nORDER BY ${query.orderBy.map(compileOrderKey).join(', ')}`;
|
|
120
|
+
if (query.limit !== undefined) sql += `\nLIMIT ${query.limit}`;
|
|
121
|
+
return sql;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Compiles a base table and optional child relation for an equi-join. */
|
|
125
|
+
function compileJoinFromClause(
|
|
126
|
+
query: SQLTableQuery,
|
|
127
|
+
table: string,
|
|
128
|
+
context: SQLCompilationContext
|
|
129
|
+
): string {
|
|
130
|
+
const join = query.join;
|
|
131
|
+
if (!join) return `FROM ${table}`;
|
|
132
|
+
validateSQLIdentifier(join.child.source);
|
|
133
|
+
validateSQLIdentifier(join.left);
|
|
134
|
+
validateSQLIdentifier(join.right);
|
|
135
|
+
const childTable = quoteSQLIdentifier(join.child.source);
|
|
136
|
+
const rightRelation = join.child.query
|
|
137
|
+
? `(${compileTableStatement({tableName: join.child.source, ...join.child.query}, context)})`
|
|
138
|
+
: childTable;
|
|
139
|
+
return `FROM ${table} JOIN ${rightRelation} AS ${childTable} ON ${table}.${quoteSQLIdentifier(join.left)} = ${childTable}.${quoteSQLIdentifier(join.right)}`;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Compiles the SELECT list, including computed and aggregate output columns. */
|
|
143
|
+
function compileProjection(query: SQLTableQuery): string {
|
|
144
|
+
const expressions = query.expressions || [];
|
|
145
|
+
const expressionByName = new Map(expressions.map(expression => [expression.name, expression]));
|
|
146
|
+
const aggregates = query.aggregates || [];
|
|
147
|
+
const aggregateByName = new Map(aggregates.map(aggregate => [aggregate.name, aggregate]));
|
|
148
|
+
const columns = query.columns === undefined ? undefined : [...query.columns];
|
|
149
|
+
const projectionColumns =
|
|
150
|
+
columns ||
|
|
151
|
+
(query.groupBy?.length || query.aggregates?.length ? [...(query.groupBy || [])] : []);
|
|
152
|
+
const selections = projectionColumns.map(column => {
|
|
153
|
+
const expression = expressionByName.get(column);
|
|
154
|
+
if (expression) {
|
|
155
|
+
return `${compileRelationalExpression(expression.expression)} AS ${quoteSQLIdentifier(expression.name)}`;
|
|
156
|
+
}
|
|
157
|
+
const aggregate = aggregateByName.get(column);
|
|
158
|
+
return aggregate
|
|
159
|
+
? `${compileAggregate(aggregate)} AS ${quoteSQLIdentifier(aggregate.name)}`
|
|
160
|
+
: compileProjectionColumn(column, query);
|
|
161
|
+
});
|
|
162
|
+
if (!selections.length)
|
|
163
|
+
return expressions.length
|
|
164
|
+
? expressions
|
|
165
|
+
.map(
|
|
166
|
+
expression =>
|
|
167
|
+
`${compileRelationalExpression(expression.expression)} AS ${quoteSQLIdentifier(expression.name)}`
|
|
168
|
+
)
|
|
169
|
+
.join(', ')
|
|
170
|
+
: aggregates.length
|
|
171
|
+
? aggregates
|
|
172
|
+
.map(
|
|
173
|
+
aggregate => `${compileAggregate(aggregate)} AS ${quoteSQLIdentifier(aggregate.name)}`
|
|
174
|
+
)
|
|
175
|
+
.join(', ')
|
|
176
|
+
: '*';
|
|
177
|
+
return selections.join(', ');
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Quotes a projection column, qualifying joined child fields component-wise. */
|
|
181
|
+
function compileProjectionColumn(column: string, query: SQLTableQuery): string {
|
|
182
|
+
const childSource = query.join?.child.source;
|
|
183
|
+
return childSource && column.startsWith(`${childSource}.`)
|
|
184
|
+
? column.split('.').map(quoteSQLIdentifier).join('.')
|
|
185
|
+
: quoteSQLIdentifier(column);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Returns whether a UNION child contains modifiers that must remain branch-local. */
|
|
189
|
+
function hasBranchLocalModifiers(query: SQLTableQuery): boolean {
|
|
190
|
+
return Boolean(query.limit !== undefined || query.orderBy?.length || query.union?.length);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Compiles one portable scalar expression into quoted SQL. */
|
|
194
|
+
function compileRelationalExpression(expression: RelationalExpression['expression']): string {
|
|
195
|
+
switch (expression.op) {
|
|
196
|
+
case 'column':
|
|
197
|
+
return quoteSQLIdentifier(expression.column);
|
|
198
|
+
case 'literal':
|
|
199
|
+
return expression.value === null
|
|
200
|
+
? 'NULL'
|
|
201
|
+
: typeof expression.value === 'string'
|
|
202
|
+
? `'${expression.value.replace(/'/g, "''")}'`
|
|
203
|
+
: String(expression.value);
|
|
204
|
+
case 'add':
|
|
205
|
+
return `(${quoteSQLIdentifier(expression.left)} + ${quoteSQLIdentifier(expression.right)})`;
|
|
206
|
+
case 'subtract':
|
|
207
|
+
return `(${quoteSQLIdentifier(expression.left)} - ${quoteSQLIdentifier(expression.right)})`;
|
|
208
|
+
case 'multiply':
|
|
209
|
+
return `(${quoteSQLIdentifier(expression.left)} * ${quoteSQLIdentifier(expression.right)})`;
|
|
210
|
+
case 'divide':
|
|
211
|
+
return `(${quoteSQLIdentifier(expression.left)} / NULLIF(${quoteSQLIdentifier(expression.right)}, 0))`;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Compiles one portable aggregate function call. */
|
|
216
|
+
function compileAggregate(aggregate: RelationalAggregate): string {
|
|
217
|
+
const column = aggregate.column ? quoteSQLIdentifier(aggregate.column) : '*';
|
|
218
|
+
return `${aggregate.function.toUpperCase()}(${column})`;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** Compiles one ordering key with explicit direction and null placement. */
|
|
222
|
+
function compileOrderKey(orderKey: RelationalOrderKey): string {
|
|
223
|
+
const direction = orderKey.direction?.toUpperCase() || 'ASC';
|
|
224
|
+
const nulls = orderKey.nulls ? ` NULLS ${orderKey.nulls.toUpperCase()}` : '';
|
|
225
|
+
return `${quoteSQLIdentifier(orderKey.column)} ${direction}${nulls}`;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Compiles one portable predicate node while collecting bound scalar values. */
|
|
229
|
+
function compileSQLPredicate(predicate: SQLPredicate, context: SQLCompilationContext): string {
|
|
230
|
+
switch (predicate.op) {
|
|
231
|
+
case 'and':
|
|
232
|
+
case 'or':
|
|
233
|
+
return `(${predicate.args
|
|
234
|
+
.map(child => compileSQLPredicate(child, context))
|
|
235
|
+
.join(` ${predicate.op.toUpperCase()} `)})`;
|
|
236
|
+
case 'not':
|
|
237
|
+
return `(NOT ${compileSQLPredicate(predicate.args[0], context)})`;
|
|
238
|
+
case 'isNull':
|
|
239
|
+
return `(${quoteSQLProperty(predicate.args[0])} IS NULL)`;
|
|
240
|
+
case 'in':
|
|
241
|
+
return `(${quoteSQLProperty(predicate.args[0])} IN (${predicate.args[1]
|
|
242
|
+
.map(value => compileSQLPredicateValue(value, context))
|
|
243
|
+
.join(', ')}))`;
|
|
244
|
+
default:
|
|
245
|
+
return `(${quoteSQLProperty(predicate.args[0])} ${predicate.op} ${compileSQLPredicateValue(
|
|
246
|
+
predicate.args[1],
|
|
247
|
+
context
|
|
248
|
+
)})`;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Adds a predicate value to the positional bindings and returns its placeholder. */
|
|
253
|
+
function compileSQLPredicateValue(
|
|
254
|
+
value: SQLPredicateValue,
|
|
255
|
+
context: SQLCompilationContext
|
|
256
|
+
): string {
|
|
257
|
+
const scalar = isSQLPredicateParameter(value)
|
|
258
|
+
? resolveSQLPredicateParameter(value.parameter, context.namedParameters)
|
|
259
|
+
: value;
|
|
260
|
+
context.parameters.push(scalar);
|
|
261
|
+
return getSQLPlaceholder(context.dialect, context.parameters.length);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** Resolves one retained named parameter reference. */
|
|
265
|
+
function resolveSQLPredicateParameter(
|
|
266
|
+
parameterName: string,
|
|
267
|
+
parameters?: SQLPredicateParameterValues
|
|
268
|
+
): SQLPredicateScalar {
|
|
269
|
+
if (!parameters || !Object.hasOwn(parameters, parameterName)) {
|
|
270
|
+
throw new Error(`SQL predicate parameter ":${parameterName}" requires a value`);
|
|
271
|
+
}
|
|
272
|
+
const value = parameters[parameterName];
|
|
273
|
+
validateSQLPredicate({op: '=', args: [{property: '_parameter'}, value]});
|
|
274
|
+
return value;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Returns the positional placeholder used by the selected SQL API. */
|
|
278
|
+
function getSQLPlaceholder(_dialect: SQLTableQueryDialect, _position: number): string {
|
|
279
|
+
return '?';
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** Quotes a possibly qualified predicate property one identifier component at a time. */
|
|
283
|
+
function quoteSQLProperty(property: {property: string; quoted?: boolean}): string {
|
|
284
|
+
return property.quoted
|
|
285
|
+
? quoteSQLIdentifier(property.property)
|
|
286
|
+
: property.property.split('.').map(quoteSQLIdentifier).join('.');
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** Quotes one SQL identifier using standard double-quote escaping. */
|
|
290
|
+
function quoteSQLIdentifier(identifier: string): string {
|
|
291
|
+
validateSQLIdentifier(identifier);
|
|
292
|
+
return `"${identifier.replace(/"/g, '""')}"`;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** Validates the structural fields that do not require a source schema. */
|
|
296
|
+
function validateSQLTableQuery(query: SQLTableQuery): void {
|
|
297
|
+
validateSQLIdentifier(query.tableName);
|
|
298
|
+
if (query.schemaName !== undefined) validateSQLIdentifier(query.schemaName);
|
|
299
|
+
if (query.catalogName !== undefined) validateSQLIdentifier(query.catalogName);
|
|
300
|
+
const seenColumns = new Set<string>();
|
|
301
|
+
if (query.columns?.length === 0) {
|
|
302
|
+
throw new Error('SQL table-query projections must select at least one column.');
|
|
303
|
+
}
|
|
304
|
+
for (const column of query.columns ?? []) {
|
|
305
|
+
validateSQLIdentifier(column);
|
|
306
|
+
if (seenColumns.has(column)) {
|
|
307
|
+
throw new Error(`Table query column was selected more than once: ${column}`);
|
|
308
|
+
}
|
|
309
|
+
seenColumns.add(column);
|
|
310
|
+
}
|
|
311
|
+
if (query.limit !== undefined && (!Number.isSafeInteger(query.limit) || query.limit < 0)) {
|
|
312
|
+
throw new Error('Table query limit must be a non-negative safe integer.');
|
|
313
|
+
}
|
|
314
|
+
if (query.predicate) validateSQLPredicate(query.predicate);
|
|
315
|
+
validateRelationalQuery(query);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** Validates relational identifiers and enum values before SQL generation. */
|
|
319
|
+
function validateRelationalQuery(query: SQLTableQuery): void {
|
|
320
|
+
const expressionNames = new Set<string>();
|
|
321
|
+
for (const expression of query.expressions || []) {
|
|
322
|
+
validateSQLIdentifier(expression.name);
|
|
323
|
+
if (expressionNames.has(expression.name))
|
|
324
|
+
throw new Error(`Relational expression duplicates column: ${expression.name}`);
|
|
325
|
+
expressionNames.add(expression.name);
|
|
326
|
+
const definition = expression.expression;
|
|
327
|
+
if (definition.op === 'column') validateSQLIdentifier(definition.column);
|
|
328
|
+
if ('left' in definition) {
|
|
329
|
+
validateSQLIdentifier(definition.left);
|
|
330
|
+
validateSQLIdentifier(definition.right);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
for (const column of query.groupBy || []) validateSQLIdentifier(column);
|
|
334
|
+
for (const key of query.orderBy || []) {
|
|
335
|
+
validateSQLIdentifier(key.column);
|
|
336
|
+
if (key.direction && key.direction !== 'asc' && key.direction !== 'desc')
|
|
337
|
+
throw new Error(`Invalid order direction: ${key.direction}`);
|
|
338
|
+
if (key.nulls && key.nulls !== 'first' && key.nulls !== 'last')
|
|
339
|
+
throw new Error(`Invalid null placement: ${key.nulls}`);
|
|
340
|
+
}
|
|
341
|
+
for (const aggregate of query.aggregates || []) {
|
|
342
|
+
validateSQLIdentifier(aggregate.name);
|
|
343
|
+
if (aggregate.function !== 'count' && !aggregate.column)
|
|
344
|
+
throw new Error(`Relational aggregate ${aggregate.function} requires a column.`);
|
|
345
|
+
if (aggregate.column) validateSQLIdentifier(aggregate.column);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Rejects empty or NUL-containing identifiers before quoting them. */
|
|
350
|
+
function validateSQLIdentifier(identifier: string): void {
|
|
351
|
+
if (!identifier || identifier.includes('\0')) {
|
|
352
|
+
throw new Error('SQL table-query identifiers must be non-empty and cannot contain NUL bytes.');
|
|
353
|
+
}
|
|
354
|
+
}
|
package/src/duckdb-sql-source.ts
CHANGED
|
@@ -248,8 +248,11 @@ async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Pr
|
|
|
248
248
|
await database.instantiate(bundles.mainModule, bundles.pthreadWorker);
|
|
249
249
|
const connection = await database.connect();
|
|
250
250
|
|
|
251
|
-
const executeRows = async (
|
|
252
|
-
|
|
251
|
+
const executeRows = async (
|
|
252
|
+
sqlText: string,
|
|
253
|
+
options: SQLQueryOptions = {}
|
|
254
|
+
): Promise<Record<string, unknown>[]> => {
|
|
255
|
+
const result = await executeDuckDBBrowserQuery(connection, sqlText, options.parameters);
|
|
253
256
|
return convertArrowToTable(result, 'object-row-table').data;
|
|
254
257
|
};
|
|
255
258
|
|
|
@@ -344,16 +347,37 @@ async function createDuckDBBrowserAdapter(context: SQLAdapterFactoryContext): Pr
|
|
|
344
347
|
);
|
|
345
348
|
return convertSQLColumnsToSchema(columns);
|
|
346
349
|
},
|
|
347
|
-
async executeRows(
|
|
348
|
-
|
|
350
|
+
async executeRows(
|
|
351
|
+
sqlText: string,
|
|
352
|
+
options: SQLQueryOptions = {}
|
|
353
|
+
): Promise<Record<string, unknown>[]> {
|
|
354
|
+
return await executeRows(sqlText, options);
|
|
349
355
|
},
|
|
350
|
-
async executeArrow(sqlText: string) {
|
|
351
|
-
const result = await connection.
|
|
356
|
+
async executeArrow(sqlText: string, options: SQLQueryOptions = {}) {
|
|
357
|
+
const result = await executeDuckDBBrowserQuery(connection, sqlText, options.parameters);
|
|
352
358
|
return convertArrowToTable(result, 'arrow-table');
|
|
353
359
|
}
|
|
354
360
|
};
|
|
355
361
|
}
|
|
356
362
|
|
|
363
|
+
/** Executes a DuckDB-Wasm query directly or through a prepared statement when values are bound. */
|
|
364
|
+
async function executeDuckDBBrowserQuery(
|
|
365
|
+
connection: any,
|
|
366
|
+
sqlText: string,
|
|
367
|
+
parameters?: SQLQueryOptions['parameters']
|
|
368
|
+
): Promise<any> {
|
|
369
|
+
if (!parameters) {
|
|
370
|
+
return await connection.query(sqlText);
|
|
371
|
+
}
|
|
372
|
+
const values = Array.isArray(parameters) ? parameters : Object.values(parameters);
|
|
373
|
+
const statement = await connection.prepare(sqlText);
|
|
374
|
+
try {
|
|
375
|
+
return await statement.query(...values);
|
|
376
|
+
} finally {
|
|
377
|
+
await statement.close();
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
357
381
|
function getDuckDBDatabasePath(url: string, options: SQLSourceOptions): string {
|
|
358
382
|
if (options.duckdb?.databasePath) {
|
|
359
383
|
return options.duckdb.databasePath;
|
package/src/index.ts
CHANGED
|
@@ -3,9 +3,20 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
export {SQLDataSource, registerSQLAdapter, getSQLAdapterFactory} from './sql-source';
|
|
6
|
+
export {ArrowTableSource} from './arrow-table-source';
|
|
6
7
|
export {DuckDBSQLSource, DuckDBSQLDataSource} from './duckdb-sql-source';
|
|
7
8
|
export {SnowflakeSQLSource, SnowflakeSQLDataSource} from './snowflake-sql-source';
|
|
9
|
+
export {
|
|
10
|
+
ARROW_TABLE_QUERY_CAPABILITIES,
|
|
11
|
+
explainArrowTableQuery,
|
|
12
|
+
queryArrowTable
|
|
13
|
+
} from './query-arrow-table';
|
|
8
14
|
export {parseSQLPredicate} from './parse-sql-predicate';
|
|
15
|
+
export {bindSQLPredicate} from './bind-sql-predicate';
|
|
16
|
+
export {compileSQLTableQuery} from './compile-table-query';
|
|
17
|
+
export {explainTableQuery, getSQLPredicateColumnNames, planTableQuery} from './table-query';
|
|
18
|
+
export {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from './table-query-capabilities';
|
|
19
|
+
export {isSQLPredicateParameter} from './sql-predicate-types';
|
|
9
20
|
export {
|
|
10
21
|
isSQLPredicate,
|
|
11
22
|
SQL_PREDICATE_JSON_SCHEMA,
|
|
@@ -13,14 +24,18 @@ export {
|
|
|
13
24
|
} from './sql-predicate-schema';
|
|
14
25
|
|
|
15
26
|
export type {
|
|
27
|
+
BoundSQLPredicate,
|
|
16
28
|
SQLComparisonPredicate,
|
|
17
29
|
SQLInPredicate,
|
|
18
30
|
SQLLogicalPredicate,
|
|
19
31
|
SQLNotPredicate,
|
|
20
32
|
SQLNullPredicate,
|
|
21
33
|
SQLPredicate,
|
|
34
|
+
SQLPredicateParameter,
|
|
35
|
+
SQLPredicateParameterValues,
|
|
22
36
|
SQLPredicateParserOptions,
|
|
23
37
|
SQLPredicateProperty,
|
|
38
|
+
SQLPredicateScalar,
|
|
24
39
|
SQLPredicateValue
|
|
25
40
|
} from './sql-predicate-types';
|
|
26
41
|
|
|
@@ -33,8 +48,32 @@ export type {
|
|
|
33
48
|
SQLColumnInfo,
|
|
34
49
|
SQLMetadata,
|
|
35
50
|
SQLParameterValues,
|
|
51
|
+
SQLQuery,
|
|
36
52
|
SQLQueryOptions,
|
|
37
53
|
SQLSchemaInfo,
|
|
38
54
|
SQLSourceOptions,
|
|
39
55
|
SQLTableInfo
|
|
40
56
|
} from './sql-types';
|
|
57
|
+
|
|
58
|
+
export type {ArrowQueryOptions} from './query-arrow-table';
|
|
59
|
+
export type {ArrowTableSourceOptions} from './arrow-table-source';
|
|
60
|
+
export type {
|
|
61
|
+
CompiledSQLTableQuery,
|
|
62
|
+
SQLTableQuery,
|
|
63
|
+
SQLTableQueryCompilerOptions,
|
|
64
|
+
SQLTableQueryDialect
|
|
65
|
+
} from './compile-table-query';
|
|
66
|
+
export type {
|
|
67
|
+
RelationalAggregate,
|
|
68
|
+
RelationalExpression,
|
|
69
|
+
RelationalOrderKey
|
|
70
|
+
} from '@loaders.gl/loader-utils';
|
|
71
|
+
export type {
|
|
72
|
+
TableQueryFilterStep,
|
|
73
|
+
TableQueryLimitStep,
|
|
74
|
+
TableQueryOptions,
|
|
75
|
+
TableQueryPlan,
|
|
76
|
+
TableQueryPlanStep,
|
|
77
|
+
TableQueryProjectStep,
|
|
78
|
+
TableQueryScanStep
|
|
79
|
+
} from './table-query';
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import type {
|
|
6
6
|
SQLLogicalPredicate,
|
|
7
7
|
SQLPredicate,
|
|
8
|
+
SQLPredicateParameterValues,
|
|
8
9
|
SQLPredicateParserOptions,
|
|
9
10
|
SQLPredicateProperty,
|
|
10
11
|
SQLPredicateValue
|
|
@@ -18,6 +19,7 @@ const MAXIMUM_EXPRESSION_DEPTH = 64;
|
|
|
18
19
|
type SQLPredicateToken = Readonly<{
|
|
19
20
|
kind: 'identifier' | 'number' | 'parameter' | 'string' | 'symbol';
|
|
20
21
|
value: string;
|
|
22
|
+
quoted?: boolean;
|
|
21
23
|
}>;
|
|
22
24
|
|
|
23
25
|
/**
|
|
@@ -31,7 +33,11 @@ export function parseSQLPredicate(
|
|
|
31
33
|
source: string,
|
|
32
34
|
options: SQLPredicateParserOptions = {}
|
|
33
35
|
): SQLPredicate {
|
|
34
|
-
const parser = new SQLPredicateParser(
|
|
36
|
+
const parser = new SQLPredicateParser(
|
|
37
|
+
source,
|
|
38
|
+
options.parameters ?? {},
|
|
39
|
+
options.preserveParameters === true
|
|
40
|
+
);
|
|
35
41
|
const predicate = parser.parse();
|
|
36
42
|
validateSQLPredicate(predicate);
|
|
37
43
|
return predicate;
|
|
@@ -42,13 +48,20 @@ class SQLPredicateParser {
|
|
|
42
48
|
/** Token stream parsed by this parser instance. */
|
|
43
49
|
private readonly tokens: readonly SQLPredicateToken[];
|
|
44
50
|
/** Named scalar parameters available to the expression. */
|
|
45
|
-
private readonly parameters:
|
|
51
|
+
private readonly parameters: SQLPredicateParameterValues;
|
|
52
|
+
/** Whether named parameters remain references in the parsed AST. */
|
|
53
|
+
private readonly preserveParameters: boolean;
|
|
46
54
|
/** Current index in the token stream. */
|
|
47
55
|
private position = 0;
|
|
48
56
|
|
|
49
|
-
constructor(
|
|
57
|
+
constructor(
|
|
58
|
+
source: string,
|
|
59
|
+
parameters: SQLPredicateParameterValues,
|
|
60
|
+
preserveParameters: boolean
|
|
61
|
+
) {
|
|
50
62
|
this.tokens = tokenizeSQLPredicate(source);
|
|
51
63
|
this.parameters = parameters;
|
|
64
|
+
this.preserveParameters = preserveParameters;
|
|
52
65
|
}
|
|
53
66
|
|
|
54
67
|
/** Parses the complete token stream. */
|
|
@@ -140,7 +153,7 @@ class SQLPredicateParser {
|
|
|
140
153
|
if (token?.kind !== 'identifier') {
|
|
141
154
|
throw new Error('SQL predicate expected a column identifier');
|
|
142
155
|
}
|
|
143
|
-
return {property: token.value};
|
|
156
|
+
return token.quoted ? {property: token.value, quoted: true} : {property: token.value};
|
|
144
157
|
}
|
|
145
158
|
|
|
146
159
|
/** Parses one scalar literal or named parameter. */
|
|
@@ -163,6 +176,9 @@ class SQLPredicateParser {
|
|
|
163
176
|
return token.value;
|
|
164
177
|
}
|
|
165
178
|
if (token.kind === 'parameter') {
|
|
179
|
+
if (this.preserveParameters) {
|
|
180
|
+
return {parameter: token.value};
|
|
181
|
+
}
|
|
166
182
|
if (!Object.hasOwn(this.parameters, token.value)) {
|
|
167
183
|
throw new Error(`SQL predicate parameter ":${token.value}" requires a value`);
|
|
168
184
|
}
|
|
@@ -300,7 +316,7 @@ function tokenizeSQLPredicate(source: string): SQLPredicateToken[] {
|
|
|
300
316
|
position = result.position;
|
|
301
317
|
} else if (character === '"') {
|
|
302
318
|
const result = readQuotedToken(source, position, '"', '"');
|
|
303
|
-
tokens.push({kind: 'identifier', value: result.value});
|
|
319
|
+
tokens.push({kind: 'identifier', value: result.value, quoted: true});
|
|
304
320
|
position = result.position;
|
|
305
321
|
} else if (character === ':') {
|
|
306
322
|
const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(source.slice(position + 1));
|