@loaders.gl/sql 5.0.0-alpha.1 → 5.0.0-alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/LICENSE +14 -3
  2. package/README.md +107 -0
  3. package/dist/arrow-query.cjs +1404 -0
  4. package/dist/arrow-query.cjs.map +6 -0
  5. package/dist/arrow-query.d.ts +5 -0
  6. package/dist/arrow-query.d.ts.map +1 -0
  7. package/dist/arrow-query.js +6 -0
  8. package/dist/arrow-query.js.map +1 -0
  9. package/dist/arrow-table-source.d.ts +27 -0
  10. package/dist/arrow-table-source.d.ts.map +1 -0
  11. package/dist/arrow-table-source.js +79 -0
  12. package/dist/arrow-table-source.js.map +1 -0
  13. package/dist/bind-sql-predicate.d.ts +4 -0
  14. package/dist/bind-sql-predicate.d.ts.map +1 -0
  15. package/dist/bind-sql-predicate.js +11 -0
  16. package/dist/bind-sql-predicate.js.map +1 -0
  17. package/dist/compile-table-query.d.ts +55 -0
  18. package/dist/compile-table-query.d.ts.map +1 -0
  19. package/dist/compile-table-query.js +255 -0
  20. package/dist/compile-table-query.js.map +1 -0
  21. package/dist/duckdb-sql-source.d.ts +51 -3
  22. package/dist/duckdb-sql-source.d.ts.map +1 -1
  23. package/dist/duckdb-sql-source.js +21 -7
  24. package/dist/duckdb-sql-source.js.map +1 -1
  25. package/dist/index.cjs +3352 -37
  26. package/dist/index.cjs.map +3 -4
  27. package/dist/index.d.ts +16 -1
  28. package/dist/index.d.ts.map +1 -1
  29. package/dist/index.js +9 -0
  30. package/dist/index.js.map +1 -1
  31. package/dist/parse-sql-predicate.d.ts +10 -0
  32. package/dist/parse-sql-predicate.d.ts.map +1 -0
  33. package/dist/parse-sql-predicate.js +322 -0
  34. package/dist/parse-sql-predicate.js.map +1 -0
  35. package/dist/query-arrow-table.d.ts +83 -0
  36. package/dist/query-arrow-table.d.ts.map +1 -0
  37. package/dist/query-arrow-table.js +663 -0
  38. package/dist/query-arrow-table.js.map +1 -0
  39. package/dist/snowflake-sql-source.d.ts +51 -3
  40. package/dist/snowflake-sql-source.d.ts.map +1 -1
  41. package/dist/snowflake-sql-source.js +9 -6
  42. package/dist/snowflake-sql-source.js.map +1 -1
  43. package/dist/sql-predicate-schema.d.ts +156 -0
  44. package/dist/sql-predicate-schema.d.ts.map +1 -0
  45. package/dist/sql-predicate-schema.js +226 -0
  46. package/dist/sql-predicate-schema.js.map +1 -0
  47. package/dist/sql-predicate-types.d.ts +45 -0
  48. package/dist/sql-predicate-types.d.ts.map +1 -0
  49. package/dist/sql-predicate-types.js +9 -0
  50. package/dist/sql-predicate-types.js.map +1 -0
  51. package/dist/sql-predicate-zod.cjs +54 -0
  52. package/dist/sql-predicate-zod.cjs.map +6 -0
  53. package/dist/sql-predicate-zod.d.ts +11 -0
  54. package/dist/sql-predicate-zod.d.ts.map +1 -0
  55. package/dist/sql-predicate-zod.js +45 -0
  56. package/dist/sql-predicate-zod.js.map +1 -0
  57. package/dist/sql-predicate.cjs +607 -0
  58. package/dist/sql-predicate.cjs.map +6 -0
  59. package/dist/sql-predicate.d.ts +6 -0
  60. package/dist/sql-predicate.d.ts.map +1 -0
  61. package/dist/sql-predicate.js +8 -0
  62. package/dist/sql-predicate.js.map +1 -0
  63. package/dist/sql-source.d.ts +16 -3
  64. package/dist/sql-source.d.ts.map +1 -1
  65. package/dist/sql-source.js +34 -5
  66. package/dist/sql-source.js.map +1 -1
  67. package/dist/sql-types.d.ts +3 -0
  68. package/dist/sql-types.d.ts.map +1 -1
  69. package/dist/table-query-capabilities.d.ts +6 -0
  70. package/dist/table-query-capabilities.d.ts.map +1 -0
  71. package/dist/table-query-capabilities.js +26 -0
  72. package/dist/table-query-capabilities.js.map +1 -0
  73. package/dist/table-query.cjs +286 -0
  74. package/dist/table-query.cjs.map +6 -0
  75. package/dist/table-query.d.ts +29 -0
  76. package/dist/table-query.d.ts.map +1 -0
  77. package/dist/table-query.js +30 -0
  78. package/dist/table-query.js.map +1 -0
  79. package/package.json +45 -8
  80. package/src/arrow-query.ts +17 -0
  81. package/src/arrow-table-source.ts +111 -0
  82. package/src/bind-sql-predicate.ts +21 -0
  83. package/src/compile-table-query.ts +354 -0
  84. package/src/duckdb-sql-source.ts +30 -6
  85. package/src/index.ts +57 -0
  86. package/src/parse-sql-predicate.ts +381 -0
  87. package/src/query-arrow-table.ts +846 -0
  88. package/src/snowflake-sql-source.ts +8 -5
  89. package/src/sql-predicate-schema.ts +251 -0
  90. package/src/sql-predicate-types.ts +87 -0
  91. package/src/sql-predicate-zod.ts +52 -0
  92. package/src/sql-predicate.ts +28 -0
  93. package/src/sql-source.ts +40 -5
  94. package/src/sql-types.ts +4 -0
  95. package/src/table-query-capabilities.ts +29 -0
  96. package/src/table-query.ts +78 -0
@@ -237,7 +237,7 @@ async function fetchSnowflakeJson(
237
237
  authorization: string,
238
238
  requestInit: RequestInit
239
239
  ): Promise<any> {
240
- const customFetch = context.options.core?.loadOptions?.core?.fetch;
240
+ const customFetch = context.options.core?.fetch;
241
241
  const headers = new Headers(requestInit.headers);
242
242
  headers.set(
243
243
  'Authorization',
@@ -265,14 +265,17 @@ async function fetchSnowflakeJson(
265
265
  function normalizeSnowflakeBindings(
266
266
  parameters: SQLQueryOptions['parameters']
267
267
  ): Record<string, {type: string; value: unknown}> | undefined {
268
- if (!parameters || Array.isArray(parameters)) {
268
+ if (!parameters) {
269
269
  return undefined;
270
270
  }
271
271
  const bindings: Record<string, {type: string; value: unknown}> = {};
272
- for (const [key, value] of Object.entries(parameters)) {
272
+ const entries = Array.isArray(parameters)
273
+ ? parameters.map((value, index) => [String(index + 1), value] as const)
274
+ : Object.entries(parameters);
275
+ for (const [key, value] of entries) {
273
276
  bindings[key] = {
274
- type: typeof value === 'number' ? 'FIXED' : 'TEXT',
275
- value
277
+ type: typeof value === 'number' || typeof value === 'bigint' ? 'FIXED' : 'TEXT',
278
+ value: typeof value === 'bigint' ? value.toString() : value
276
279
  };
277
280
  }
278
281
  return bindings;
@@ -0,0 +1,251 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import {
6
+ isSQLPredicateParameter,
7
+ type SQLPredicate,
8
+ type SQLPredicateValue
9
+ } from './sql-predicate-types';
10
+
11
+ const MAXIMUM_PREDICATE_DEPTH = 64;
12
+ const MAXIMUM_PREDICATE_NODES = 4096;
13
+
14
+ /**
15
+ * JSON Schema for the JSON-serializable portion of the loaders.gl SQL predicate AST.
16
+ *
17
+ * In-process predicates additionally accept `bigint`, `Date`, and `Uint8Array` values, which JSON
18
+ * Schema cannot represent directly.
19
+ */
20
+ export const SQL_PREDICATE_JSON_SCHEMA = {
21
+ $schema: 'https://json-schema.org/draft/2020-12/schema',
22
+ $id: 'https://loaders.gl/schemas/sql-predicate.schema.json',
23
+ title: 'loaders.gl SQL predicate',
24
+ oneOf: [
25
+ {$ref: '#/$defs/comparison'},
26
+ {$ref: '#/$defs/in'},
27
+ {$ref: '#/$defs/isNull'},
28
+ {$ref: '#/$defs/logical'},
29
+ {$ref: '#/$defs/not'}
30
+ ],
31
+ $defs: {
32
+ property: {
33
+ type: 'object',
34
+ required: ['property'],
35
+ properties: {
36
+ property: {type: 'string', minLength: 1},
37
+ quoted: {type: 'boolean'}
38
+ },
39
+ additionalProperties: false
40
+ },
41
+ value: {
42
+ oneOf: [
43
+ {type: ['boolean', 'number', 'string']},
44
+ {
45
+ type: 'object',
46
+ required: ['parameter'],
47
+ properties: {parameter: {type: 'string', minLength: 1}},
48
+ additionalProperties: false
49
+ }
50
+ ]
51
+ },
52
+ comparison: {
53
+ type: 'object',
54
+ required: ['op', 'args'],
55
+ properties: {
56
+ op: {enum: ['=', '<>', '<', '<=', '>', '>=']},
57
+ args: {
58
+ type: 'array',
59
+ prefixItems: [{$ref: '#/$defs/property'}, {$ref: '#/$defs/value'}],
60
+ minItems: 2,
61
+ maxItems: 2
62
+ }
63
+ },
64
+ additionalProperties: false
65
+ },
66
+ in: {
67
+ type: 'object',
68
+ required: ['op', 'args'],
69
+ properties: {
70
+ op: {const: 'in'},
71
+ args: {
72
+ type: 'array',
73
+ prefixItems: [
74
+ {$ref: '#/$defs/property'},
75
+ {type: 'array', items: {$ref: '#/$defs/value'}, minItems: 1}
76
+ ],
77
+ minItems: 2,
78
+ maxItems: 2
79
+ }
80
+ },
81
+ additionalProperties: false
82
+ },
83
+ isNull: {
84
+ type: 'object',
85
+ required: ['op', 'args'],
86
+ properties: {
87
+ op: {const: 'isNull'},
88
+ args: {
89
+ type: 'array',
90
+ prefixItems: [{$ref: '#/$defs/property'}],
91
+ minItems: 1,
92
+ maxItems: 1
93
+ }
94
+ },
95
+ additionalProperties: false
96
+ },
97
+ logical: {
98
+ type: 'object',
99
+ required: ['op', 'args'],
100
+ properties: {
101
+ op: {enum: ['and', 'or']},
102
+ args: {type: 'array', items: {$ref: '#'}, minItems: 2}
103
+ },
104
+ additionalProperties: false
105
+ },
106
+ not: {
107
+ type: 'object',
108
+ required: ['op', 'args'],
109
+ properties: {
110
+ op: {const: 'not'},
111
+ args: {type: 'array', prefixItems: [{$ref: '#'}], minItems: 1, maxItems: 1}
112
+ },
113
+ additionalProperties: false
114
+ }
115
+ }
116
+ } as const;
117
+
118
+ /** Returns whether an unknown value is a valid portable SQL predicate. */
119
+ export function isSQLPredicate(value: unknown): value is SQLPredicate {
120
+ try {
121
+ validateSQLPredicate(value);
122
+ return true;
123
+ } catch {
124
+ return false;
125
+ }
126
+ }
127
+
128
+ /** Validates an unknown portable SQL predicate or throws a path-specific error. */
129
+ export function validateSQLPredicate(value: unknown): asserts value is SQLPredicate {
130
+ const state = {nodes: 0};
131
+ validatePredicateNode(value, '$', 0, state);
132
+ }
133
+
134
+ /** Validates one recursive predicate node. */
135
+ function validatePredicateNode(
136
+ value: unknown,
137
+ path: string,
138
+ depth: number,
139
+ state: {nodes: number}
140
+ ): asserts value is SQLPredicate {
141
+ if (depth > MAXIMUM_PREDICATE_DEPTH) {
142
+ throw new Error(`SQL predicate ${path} exceeds the maximum nesting depth`);
143
+ }
144
+ state.nodes++;
145
+ if (state.nodes > MAXIMUM_PREDICATE_NODES) {
146
+ throw new Error('SQL predicate exceeds the maximum node count');
147
+ }
148
+ if (!isRecord(value) || !hasExactKeys(value, ['op', 'args'])) {
149
+ throw new Error(`SQL predicate ${path} must contain only op and args`);
150
+ }
151
+ if (typeof value.op !== 'string' || !Array.isArray(value.args)) {
152
+ throw new Error(`SQL predicate ${path} requires string op and array args`);
153
+ }
154
+
155
+ switch (value.op) {
156
+ case 'and':
157
+ case 'or':
158
+ if (value.args.length < 2) {
159
+ throw new Error(`SQL predicate ${path}.${value.op} requires at least two arguments`);
160
+ }
161
+ for (let index = 0; index < value.args.length; index++) {
162
+ validatePredicateNode(value.args[index], `${path}.args[${index}]`, depth + 1, state);
163
+ }
164
+ return;
165
+ case 'not':
166
+ if (value.args.length !== 1) {
167
+ throw new Error(`SQL predicate ${path}.not requires exactly one argument`);
168
+ }
169
+ validatePredicateNode(value.args[0], `${path}.args[0]`, depth + 1, state);
170
+ return;
171
+ case 'isNull':
172
+ if (value.args.length !== 1) {
173
+ throw new Error(`SQL predicate ${path}.isNull requires exactly one argument`);
174
+ }
175
+ validatePredicateProperty(value.args[0], `${path}.args[0]`);
176
+ return;
177
+ case 'in':
178
+ if (value.args.length !== 2 || !Array.isArray(value.args[1]) || value.args[1].length === 0) {
179
+ throw new Error(`SQL predicate ${path}.in requires a property and non-empty value list`);
180
+ }
181
+ validatePredicateProperty(value.args[0], `${path}.args[0]`);
182
+ value.args[1].forEach((item, index) =>
183
+ validatePredicateValue(item, `${path}.args[1][${index}]`)
184
+ );
185
+ return;
186
+ case '=':
187
+ case '<>':
188
+ case '<':
189
+ case '<=':
190
+ case '>':
191
+ case '>=':
192
+ if (value.args.length !== 2) {
193
+ throw new Error(`SQL predicate ${path}.${value.op} requires exactly two arguments`);
194
+ }
195
+ validatePredicateProperty(value.args[0], `${path}.args[0]`);
196
+ validatePredicateValue(value.args[1], `${path}.args[1]`);
197
+ return;
198
+ default:
199
+ throw new Error(`SQL predicate ${path} has unsupported operator ${JSON.stringify(value.op)}`);
200
+ }
201
+ }
202
+
203
+ /** Validates one property reference. */
204
+ function validatePredicateProperty(value: unknown, path: string): void {
205
+ if (
206
+ !isRecord(value) ||
207
+ !hasAllowedKeys(value, ['property', 'quoted']) ||
208
+ typeof value.property !== 'string' ||
209
+ value.property.length === 0 ||
210
+ (value.quoted !== undefined && typeof value.quoted !== 'boolean')
211
+ ) {
212
+ throw new Error(`SQL predicate ${path} requires a non-empty property reference`);
213
+ }
214
+ }
215
+
216
+ /** Validates one structured-cloneable predicate scalar. */
217
+ function validatePredicateValue(value: unknown, path: string): asserts value is SQLPredicateValue {
218
+ if (isSQLPredicateParameter(value)) {
219
+ return;
220
+ }
221
+ if (
222
+ typeof value === 'boolean' ||
223
+ typeof value === 'bigint' ||
224
+ typeof value === 'string' ||
225
+ value instanceof Uint8Array
226
+ ) {
227
+ return;
228
+ }
229
+ if (typeof value === 'number' && Number.isFinite(value)) {
230
+ return;
231
+ }
232
+ if (value instanceof Date && Number.isFinite(value.getTime())) {
233
+ return;
234
+ }
235
+ throw new Error(`SQL predicate ${path} contains an unsupported scalar value`);
236
+ }
237
+
238
+ /** Returns whether an unknown value is a non-null object. */
239
+ function isRecord(value: unknown): value is Record<string, unknown> {
240
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
241
+ }
242
+
243
+ /** Returns whether an object contains exactly the expected enumerable keys. */
244
+ function hasExactKeys(value: Record<string, unknown>, expectedKeys: readonly string[]): boolean {
245
+ const keys = Object.keys(value);
246
+ return keys.length === expectedKeys.length && expectedKeys.every(key => keys.includes(key));
247
+ }
248
+
249
+ function hasAllowedKeys(value: Record<string, unknown>, allowedKeys: readonly string[]): boolean {
250
+ return Object.keys(value).every(key => allowedKeys.includes(key));
251
+ }
@@ -0,0 +1,87 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import {
6
+ isColumnarPredicateParameter,
7
+ type ColumnarComparisonPredicate,
8
+ type ColumnarInPredicate,
9
+ type ColumnarLogicalPredicate,
10
+ type ColumnarNotPredicate,
11
+ type ColumnarNullPredicate,
12
+ type ColumnarPredicate,
13
+ type ColumnarPredicateInputValue,
14
+ type ColumnarPredicateParameter,
15
+ type ColumnarPredicateParameterValues,
16
+ type ColumnarPredicateValue
17
+ } from '@loaders.gl/loader-utils';
18
+
19
+ /** Scalar values supported by portable SQL predicate expressions. */
20
+ export type SQLPredicateScalar = ColumnarPredicateValue;
21
+
22
+ /** Named parameter reference retained in a portable SQL predicate until execution time. */
23
+ export type SQLPredicateParameter = ColumnarPredicateParameter;
24
+
25
+ /** Scalar value or unresolved named parameter accepted by a portable predicate expression. */
26
+ export type SQLPredicateValue = ColumnarPredicateInputValue;
27
+
28
+ /** Runtime values used to bind named predicate parameters before a backend executes them. */
29
+ export type SQLPredicateParameterValues = ColumnarPredicateParameterValues;
30
+
31
+ /** Reference to one column in a portable SQL predicate expression. */
32
+ export type SQLPredicateProperty = Readonly<{
33
+ /** Column or qualified column name. */
34
+ property: string;
35
+ /** Whether the complete property was a quoted SQL identifier. */
36
+ quoted?: boolean;
37
+ }>;
38
+
39
+ /** Binary comparison between a column and a scalar value. */
40
+ export type SQLComparisonPredicate = Readonly<
41
+ ColumnarComparisonPredicate<SQLPredicateValue, SQLPredicateProperty>
42
+ >;
43
+
44
+ /** Membership test between a column and scalar values. */
45
+ export type SQLInPredicate = Readonly<ColumnarInPredicate<SQLPredicateValue, SQLPredicateProperty>>;
46
+
47
+ /** Null test for one column. */
48
+ export type SQLNullPredicate = Readonly<ColumnarNullPredicate<SQLPredicateProperty>>;
49
+
50
+ /** Conjunction or disjunction of two or more predicates. */
51
+ export type SQLLogicalPredicate = Readonly<
52
+ ColumnarLogicalPredicate<SQLPredicateValue, SQLPredicateProperty>
53
+ >;
54
+
55
+ /** Negation of one predicate. */
56
+ export type SQLNotPredicate = Readonly<
57
+ ColumnarNotPredicate<SQLPredicateValue, SQLPredicateProperty>
58
+ >;
59
+
60
+ /**
61
+ * Portable predicate AST emitted by the loaders.gl SQL expression parser.
62
+ *
63
+ * The expression shape is directionally aligned with CQL2 JSON, but this deliberately small
64
+ * subset does not claim CQL2 conformance.
65
+ */
66
+ export type SQLPredicate =
67
+ | SQLComparisonPredicate
68
+ | SQLInPredicate
69
+ | SQLNullPredicate
70
+ | SQLLogicalPredicate
71
+ | SQLNotPredicate;
72
+
73
+ /** Portable SQL predicate whose named parameters have all been replaced by concrete scalars. */
74
+ export type BoundSQLPredicate = ColumnarPredicate<SQLPredicateScalar, SQLPredicateProperty>;
75
+
76
+ /** Options for parsing one SQL predicate expression. */
77
+ export type SQLPredicateParserOptions = Readonly<{
78
+ /** Named values referenced using SQL parameters such as `:minimumFare`. */
79
+ parameters?: SQLPredicateParameterValues;
80
+ /** Retains named references in the emitted AST instead of resolving them while parsing. */
81
+ preserveParameters?: boolean;
82
+ }>;
83
+
84
+ /** Returns whether a value is a named parameter reference. */
85
+ export function isSQLPredicateParameter(value: unknown): value is SQLPredicateParameter {
86
+ return isColumnarPredicateParameter(value);
87
+ }
@@ -0,0 +1,52 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import {z} from 'zod';
6
+
7
+ import type {SQLPredicate, SQLPredicateValue} from './sql-predicate-types';
8
+
9
+ /** Zod schema for scalar values accepted by in-process SQL predicates. */
10
+ export const SQLPredicateValueSchema: z.ZodType<SQLPredicateValue> = z.union([
11
+ z.boolean(),
12
+ z.number().finite(),
13
+ z.bigint(),
14
+ z.string(),
15
+ z.date(),
16
+ z.instanceof(Uint8Array),
17
+ z.object({parameter: z.string().min(1)}).strict()
18
+ ]);
19
+
20
+ const SQLPredicatePropertySchema = z
21
+ .object({property: z.string().min(1), quoted: z.boolean().optional()})
22
+ .strict();
23
+
24
+ /**
25
+ * Optional Zod schema for validating the portable loaders.gl SQL predicate AST.
26
+ *
27
+ * Import from `@loaders.gl/sql/sql-predicate-zod`; the package root does not load Zod.
28
+ */
29
+ export const SQLPredicateSchema: z.ZodType<SQLPredicate> = z.lazy(() =>
30
+ z.union([
31
+ z
32
+ .object({
33
+ op: z.enum(['=', '<>', '<', '<=', '>', '>=']),
34
+ args: z.tuple([SQLPredicatePropertySchema, SQLPredicateValueSchema])
35
+ })
36
+ .strict(),
37
+ z
38
+ .object({
39
+ op: z.literal('in'),
40
+ args: z.tuple([SQLPredicatePropertySchema, z.array(SQLPredicateValueSchema).min(1)])
41
+ })
42
+ .strict(),
43
+ z.object({op: z.literal('isNull'), args: z.tuple([SQLPredicatePropertySchema])}).strict(),
44
+ z
45
+ .object({
46
+ op: z.enum(['and', 'or']),
47
+ args: z.array(SQLPredicateSchema).min(2)
48
+ })
49
+ .strict(),
50
+ z.object({op: z.literal('not'), args: z.tuple([SQLPredicateSchema])}).strict()
51
+ ])
52
+ );
@@ -0,0 +1,28 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ export {parseSQLPredicate} from './parse-sql-predicate';
6
+ export {
7
+ isSQLPredicate,
8
+ SQL_PREDICATE_JSON_SCHEMA,
9
+ validateSQLPredicate
10
+ } from './sql-predicate-schema';
11
+ export {bindSQLPredicate} from './bind-sql-predicate';
12
+ export {isSQLPredicateParameter} from './sql-predicate-types';
13
+
14
+ export type {
15
+ BoundSQLPredicate,
16
+ SQLComparisonPredicate,
17
+ SQLInPredicate,
18
+ SQLLogicalPredicate,
19
+ SQLNotPredicate,
20
+ SQLNullPredicate,
21
+ SQLPredicate,
22
+ SQLPredicateParameter,
23
+ SQLPredicateParameterValues,
24
+ SQLPredicateParserOptions,
25
+ SQLPredicateProperty,
26
+ SQLPredicateScalar,
27
+ SQLPredicateValue
28
+ } from './sql-predicate-types';
package/src/sql-source.ts CHANGED
@@ -11,12 +11,16 @@ import type {
11
11
  SQLAdapterFactory,
12
12
  SQLCatalogInfo,
13
13
  SQLMetadata,
14
+ SQLQuery,
14
15
  SQLQueryOptions,
15
16
  SQLSchemaInfo,
16
17
  SQLSourceOptions,
17
18
  SQLTableInfo
18
19
  } from './sql-types';
20
+ import {compileSQLTableQuery, type SQLTableQueryDialect} from './compile-table-query';
21
+ import type {SQLPredicateParameterValues} from './sql-predicate-types';
19
22
  import {convertRowsToArrowTable, isNodeRuntime} from './sql-utils';
23
+ import {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from './table-query-capabilities';
20
24
 
21
25
  const SQL_ADAPTER_FACTORIES = new Map<string, SQLAdapterFactory>();
22
26
 
@@ -58,6 +62,8 @@ export const SQL_SOURCE_DEFAULT_OPTIONS: Omit<RequiredOptions<SQLSourceOptions>,
58
62
 
59
63
  /** Shared SQL-backed data source implementation. */
60
64
  export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
65
+ /** Optimization and execution capabilities of portable table-bound queries. */
66
+ readonly tableQueryCapabilities = SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES;
61
67
  private readonly sourceType: string;
62
68
  private readonly adapterFactory?: SQLAdapterFactory;
63
69
  private adapterPromise: Promise<SQLAdapter> | null = null;
@@ -114,20 +120,22 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
114
120
 
115
121
  /** Executes a SQL query and returns object rows. */
116
122
  async queryRows(
117
- sqlText: string,
123
+ query: SQLQuery,
118
124
  options: SQLQueryOptions = {}
119
125
  ): Promise<Record<string, unknown>[]> {
120
126
  const adapter = await this.getAdapter();
121
- return await adapter.executeRows(sqlText, options);
127
+ const compiledQuery = this.compileQuery(query, options);
128
+ return await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
122
129
  }
123
130
 
124
131
  /** Executes a SQL query and returns a loaders.gl Arrow table. */
125
- async queryArrow(sqlText: string, options: SQLQueryOptions = {}): Promise<ArrowTable> {
132
+ async queryArrow(query: SQLQuery, options: SQLQueryOptions = {}): Promise<ArrowTable> {
126
133
  const adapter = await this.getAdapter();
134
+ const compiledQuery = this.compileQuery(query, options);
127
135
  if (adapter.executeArrow) {
128
- return await adapter.executeArrow(sqlText, options);
136
+ return await adapter.executeArrow(compiledQuery.sql, compiledQuery.options);
129
137
  }
130
- const rows = await adapter.executeRows(sqlText, options);
138
+ const rows = await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
131
139
  return convertRowsToArrowTable(rows);
132
140
  }
133
141
 
@@ -149,6 +157,26 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
149
157
  return await this.adapterPromise;
150
158
  }
151
159
 
160
+ /** Converts a portable query object to SQL while leaving raw SQL text unchanged. */
161
+ private compileQuery(
162
+ query: SQLQuery,
163
+ options: SQLQueryOptions
164
+ ): {sql: string; options: SQLQueryOptions} {
165
+ if (typeof query === 'string') {
166
+ return {sql: query, options};
167
+ }
168
+ const namedParameters =
169
+ options.parameters && !Array.isArray(options.parameters) ? options.parameters : undefined;
170
+ const compiledQuery = compileSQLTableQuery(query, {
171
+ dialect: getSQLTableQueryDialect(this.sourceType),
172
+ parameters: namedParameters as SQLPredicateParameterValues | undefined
173
+ });
174
+ return {
175
+ sql: compiledQuery.sql,
176
+ options: {signal: options.signal, parameters: compiledQuery.parameters}
177
+ };
178
+ }
179
+
152
180
  private async loadAdapter(): Promise<SQLAdapter> {
153
181
  try {
154
182
  const runtimeFactory = isNodeRuntime()
@@ -191,3 +219,10 @@ export class SQLDataSource extends DataSource<string, SQLSourceOptions> {
191
219
  };
192
220
  }
193
221
  }
222
+
223
+ /** Maps a SQL source adapter to the dialect used by the portable query compiler. */
224
+ function getSQLTableQueryDialect(sourceType: string): SQLTableQueryDialect {
225
+ if (sourceType === 'duckdb-sql') return 'duckdb';
226
+ if (sourceType === 'snowflake-sql') return 'snowflake';
227
+ throw new Error(`Portable table queries are not supported for SQL source type "${sourceType}".`);
228
+ }
package/src/sql-types.ts CHANGED
@@ -4,9 +4,13 @@
4
4
 
5
5
  import type {ArrowTable, Schema} from '@loaders.gl/schema';
6
6
  import type {CoreAPI, DataSourceOptions} from '@loaders.gl/loader-utils';
7
+ import type {SQLTableQuery} from './compile-table-query';
7
8
 
8
9
  export type SQLParameterValues = unknown[] | Record<string, unknown>;
9
10
 
11
+ /** Raw SQL text or a portable query bound to one database table. */
12
+ export type SQLQuery = string | SQLTableQuery;
13
+
10
14
  /** Query-level options for SQL data sources. */
11
15
  export type SQLQueryOptions = {
12
16
  parameters?: SQLParameterValues;
@@ -0,0 +1,29 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {TableQueryCapabilities} from '@loaders.gl/loader-utils';
6
+
7
+ /** Portable query capabilities of the in-memory Arrow executor. */
8
+ export const ARROW_TABLE_QUERY_CAPABILITIES: TableQueryCapabilities = Object.freeze({
9
+ projection: 'residual',
10
+ predicate: 'residual',
11
+ limit: 'residual',
12
+ streaming: false,
13
+ cancellation: true,
14
+ expressions: 'residual',
15
+ orderBy: 'residual',
16
+ aggregates: 'residual'
17
+ });
18
+
19
+ /** Portable query capabilities shared by the current SQL data-source adapters. */
20
+ export const SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES: TableQueryCapabilities = Object.freeze({
21
+ projection: 'pushdown',
22
+ predicate: 'pushdown',
23
+ limit: 'pushdown',
24
+ streaming: false,
25
+ cancellation: false,
26
+ expressions: 'pushdown',
27
+ orderBy: 'pushdown',
28
+ aggregates: 'pushdown'
29
+ });
@@ -0,0 +1,78 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import {
6
+ getColumnarPredicateColumns,
7
+ explainTableQuery as explainColumnarTableQuery,
8
+ planTableQuery as planColumnarTableQuery,
9
+ type TableQueryFilterStep as ColumnarTableQueryFilterStep,
10
+ type TableQueryLimitStep as ColumnarTableQueryLimitStep,
11
+ type TableQueryOptions as ColumnarTableQueryOptions,
12
+ type TableQueryPlan as ColumnarTableQueryPlan,
13
+ type TableQueryPlanStep as ColumnarTableQueryPlanStep,
14
+ type TableQueryProjectStep as ColumnarTableQueryProjectStep,
15
+ type TableQueryScanStep as ColumnarTableQueryScanStep
16
+ } from '@loaders.gl/loader-utils';
17
+
18
+ import {validateSQLPredicate} from './sql-predicate-schema';
19
+ import type {SQLPredicate} from './sql-predicate-types';
20
+
21
+ /** Immutable, backend-neutral request for a table query. */
22
+ export type TableQueryOptions = ColumnarTableQueryOptions<SQLPredicate>;
23
+
24
+ /** Reads the minimum source columns required by the remaining table-query plan. */
25
+ export type TableQueryScanStep = ColumnarTableQueryScanStep;
26
+
27
+ /** Applies a portable SQL predicate while all predicate columns remain available. */
28
+ export type TableQueryFilterStep = ColumnarTableQueryFilterStep<SQLPredicate>;
29
+
30
+ /** Narrows the result to requested output columns. */
31
+ export type TableQueryProjectStep = ColumnarTableQueryProjectStep;
32
+
33
+ /** Retains the first rows in source order after filtering and projection. */
34
+ export type TableQueryLimitStep = ColumnarTableQueryLimitStep;
35
+
36
+ /** One immutable operation in a normalized table-query plan. */
37
+ export type TableQueryPlanStep = ColumnarTableQueryPlanStep<SQLPredicate>;
38
+
39
+ /** Immutable logical query plan shared by Arrow and GPU dataframe backends. */
40
+ export type TableQueryPlan = ColumnarTableQueryPlan<SQLPredicate>;
41
+ export type TableQueryExplain = import('@loaders.gl/loader-utils').TableQueryExplain<SQLPredicate>;
42
+ import {SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES} from './table-query-capabilities';
43
+
44
+ /**
45
+ * Produces the canonical scan, filter, project, and limit sequence for a table query.
46
+ *
47
+ * Predicate columns are retained by the scan even if callers omit them from the output projection.
48
+ * This makes SQL `WHERE` semantics independent of a backend's internal API call order.
49
+ */
50
+ export function planTableQuery(
51
+ sourceColumnNames: readonly string[],
52
+ options: TableQueryOptions = {}
53
+ ): TableQueryPlan {
54
+ if (options.predicate) {
55
+ validateSQLPredicate(options.predicate);
56
+ }
57
+ return planColumnarTableQuery(sourceColumnNames, options);
58
+ }
59
+
60
+ /** Explains a portable SQL table query without executing it. */
61
+ export function explainTableQuery(
62
+ sourceColumnNames: readonly string[],
63
+ options: TableQueryOptions = {}
64
+ ): TableQueryExplain {
65
+ if (options.predicate) {
66
+ validateSQLPredicate(options.predicate);
67
+ }
68
+ return explainColumnarTableQuery(
69
+ sourceColumnNames,
70
+ options,
71
+ SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES
72
+ );
73
+ }
74
+
75
+ /** Returns every source column referenced by one portable predicate, in first-use order. */
76
+ export function getSQLPredicateColumnNames(predicate: SQLPredicate): string[] {
77
+ return getColumnarPredicateColumns(predicate);
78
+ }