@loaders.gl/sql 5.0.0-alpha.8 → 5.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -30
- package/README.md +15 -10
- package/THIRD-PARTY-NOTICES.md +17 -0
- package/dist/arrow-table-source.d.ts +5 -5
- package/dist/arrow-table-source.d.ts.map +1 -1
- package/dist/arrow-table-source.js +3 -3
- package/dist/arrow-table-source.js.map +1 -1
- package/dist/compile-table-query.d.ts +3 -3
- package/dist/compile-table-query.d.ts.map +1 -1
- package/dist/compile-table-query.js +2 -2
- package/dist/compile-table-query.js.map +1 -1
- package/dist/duckdb-sql-source.d.ts +22 -13
- package/dist/duckdb-sql-source.d.ts.map +1 -1
- package/dist/duckdb-sql-source.js +40 -16
- package/dist/duckdb-sql-source.js.map +1 -1
- package/dist/index.cjs +218 -169
- package/dist/index.cjs.map +2 -2
- package/dist/index.d.ts +12 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -7
- package/dist/index.js.map +1 -1
- package/dist/snowflake-sql-source.d.ts +16 -12
- package/dist/snowflake-sql-source.d.ts.map +1 -1
- package/dist/snowflake-sql-source.js +1 -1
- package/dist/snowflake-sql-source.js.map +1 -1
- package/dist/sql-source.d.ts +9 -9
- package/dist/sql-source.d.ts.map +1 -1
- package/dist/sql-source.js +2 -2
- package/dist/sql-source.js.map +1 -1
- package/dist/sql-types.d.ts +13 -2
- package/dist/sql-types.d.ts.map +1 -1
- package/dist/sql-utils.d.ts +2 -3
- package/dist/sql-utils.d.ts.map +1 -1
- package/dist/sql-utils.js +1 -34
- package/dist/sql-utils.js.map +1 -1
- package/package.json +5 -26
- package/src/arrow-table-source.ts +7 -7
- package/src/compile-table-query.ts +4 -4
- package/src/duckdb-sql-source.ts +47 -19
- package/src/index.ts +16 -11
- package/src/snowflake-sql-source.ts +2 -2
- package/src/sql-source.ts +5 -5
- package/src/sql-types.ts +14 -2
- package/src/sql-utils.ts +2 -40
- package/dist/arrow-query.cjs +0 -1404
- package/dist/arrow-query.cjs.map +0 -6
- package/dist/arrow-query.d.ts +0 -5
- package/dist/arrow-query.d.ts.map +0 -1
- package/dist/arrow-query.js +0 -6
- package/dist/arrow-query.js.map +0 -1
- package/dist/bind-sql-predicate.d.ts +0 -4
- package/dist/bind-sql-predicate.d.ts.map +0 -1
- package/dist/bind-sql-predicate.js +0 -11
- package/dist/bind-sql-predicate.js.map +0 -1
- package/dist/parse-sql-predicate.d.ts +0 -10
- package/dist/parse-sql-predicate.d.ts.map +0 -1
- package/dist/parse-sql-predicate.js +0 -322
- package/dist/parse-sql-predicate.js.map +0 -1
- package/dist/query-arrow-table.d.ts +0 -83
- package/dist/query-arrow-table.d.ts.map +0 -1
- package/dist/query-arrow-table.js +0 -663
- package/dist/query-arrow-table.js.map +0 -1
- package/dist/sql-predicate-schema.d.ts +0 -156
- package/dist/sql-predicate-schema.d.ts.map +0 -1
- package/dist/sql-predicate-schema.js +0 -226
- package/dist/sql-predicate-schema.js.map +0 -1
- package/dist/sql-predicate-types.d.ts +0 -45
- package/dist/sql-predicate-types.d.ts.map +0 -1
- package/dist/sql-predicate-types.js +0 -9
- package/dist/sql-predicate-types.js.map +0 -1
- package/dist/sql-predicate-zod.cjs +0 -54
- package/dist/sql-predicate-zod.cjs.map +0 -6
- package/dist/sql-predicate-zod.d.ts +0 -11
- package/dist/sql-predicate-zod.d.ts.map +0 -1
- package/dist/sql-predicate-zod.js +0 -45
- package/dist/sql-predicate-zod.js.map +0 -1
- package/dist/sql-predicate.cjs +0 -607
- package/dist/sql-predicate.cjs.map +0 -6
- package/dist/sql-predicate.d.ts +0 -6
- package/dist/sql-predicate.d.ts.map +0 -1
- package/dist/sql-predicate.js +0 -8
- package/dist/sql-predicate.js.map +0 -1
- package/dist/table-query-capabilities.d.ts +0 -6
- package/dist/table-query-capabilities.d.ts.map +0 -1
- package/dist/table-query-capabilities.js +0 -26
- package/dist/table-query-capabilities.js.map +0 -1
- package/dist/table-query.cjs +0 -286
- package/dist/table-query.cjs.map +0 -6
- package/dist/table-query.d.ts +0 -29
- package/dist/table-query.d.ts.map +0 -1
- package/dist/table-query.js +0 -30
- package/dist/table-query.js.map +0 -1
- package/src/arrow-query.ts +0 -17
- package/src/bind-sql-predicate.ts +0 -21
- package/src/parse-sql-predicate.ts +0 -381
- package/src/query-arrow-table.ts +0 -846
- package/src/sql-predicate-schema.ts +0 -251
- package/src/sql-predicate-types.ts +0 -87
- package/src/sql-predicate-zod.ts +0 -52
- package/src/sql-predicate.ts +0 -28
- package/src/table-query-capabilities.ts +0 -29
- package/src/table-query.ts +0 -78
package/src/query-arrow-table.ts
DELETED
|
@@ -1,846 +0,0 @@
|
|
|
1
|
-
// loaders.gl
|
|
2
|
-
// SPDX-License-Identifier: MIT
|
|
3
|
-
// Copyright (c) vis.gl contributors
|
|
4
|
-
|
|
5
|
-
import * as arrow from 'apache-arrow';
|
|
6
|
-
import type {ArrowTable} from '@loaders.gl/schema';
|
|
7
|
-
import {convertArrowToSchema} from '@loaders.gl/schema-utils';
|
|
8
|
-
import {convertRowsToArrowTable} from './sql-utils';
|
|
9
|
-
|
|
10
|
-
import {
|
|
11
|
-
isSQLPredicateParameter,
|
|
12
|
-
type SQLComparisonPredicate,
|
|
13
|
-
type SQLInPredicate,
|
|
14
|
-
type SQLLogicalPredicate,
|
|
15
|
-
type SQLNotPredicate,
|
|
16
|
-
type SQLNullPredicate,
|
|
17
|
-
type SQLPredicate,
|
|
18
|
-
type SQLPredicateValue
|
|
19
|
-
} from './sql-predicate-types';
|
|
20
|
-
import {
|
|
21
|
-
getSQLPredicateColumnNames,
|
|
22
|
-
planTableQuery,
|
|
23
|
-
type TableQueryFilterStep,
|
|
24
|
-
type TableQueryLimitStep,
|
|
25
|
-
type TableQueryOptions,
|
|
26
|
-
type TableQueryPlanStep,
|
|
27
|
-
type TableQueryProjectStep,
|
|
28
|
-
type TableQueryScanStep
|
|
29
|
-
} from './table-query';
|
|
30
|
-
import type {
|
|
31
|
-
RelationalAggregate,
|
|
32
|
-
RelationalExpression,
|
|
33
|
-
RelationalOrderKey
|
|
34
|
-
} from '@loaders.gl/loader-utils';
|
|
35
|
-
import {
|
|
36
|
-
explainTableQuery as explainColumnarTableQuery,
|
|
37
|
-
planRelationalQuery
|
|
38
|
-
} from '@loaders.gl/loader-utils';
|
|
39
|
-
import {ARROW_TABLE_QUERY_CAPABILITIES} from './table-query-capabilities';
|
|
40
|
-
import {validateSQLPredicate} from './sql-predicate-schema';
|
|
41
|
-
export {ARROW_TABLE_QUERY_CAPABILITIES};
|
|
42
|
-
|
|
43
|
-
/** A named child relation using the SQL module's property-oriented predicate AST. */
|
|
44
|
-
type ArrowChildQuery = Readonly<{
|
|
45
|
-
/** Logical source name resolved through the query's table map. */
|
|
46
|
-
source: string;
|
|
47
|
-
/** Optional query applied to the child relation before combining it. */
|
|
48
|
-
query?: TableQueryOptions;
|
|
49
|
-
}>;
|
|
50
|
-
|
|
51
|
-
/** Options for the lightweight in-memory Arrow query executor. */
|
|
52
|
-
export type ArrowQueryOptions = TableQueryOptions &
|
|
53
|
-
Readonly<{
|
|
54
|
-
/** Cancels the query before or during predicate evaluation. */
|
|
55
|
-
signal?: AbortSignal;
|
|
56
|
-
/** Computed columns evaluated before ordering and projection. */
|
|
57
|
-
expressions?: readonly RelationalExpression[];
|
|
58
|
-
/** Stable ordering applied before the global limit. */
|
|
59
|
-
orderBy?: readonly RelationalOrderKey[];
|
|
60
|
-
/** Grouping keys and aggregate definitions for relational output. */
|
|
61
|
-
groupBy?: readonly string[];
|
|
62
|
-
/** Aggregate definitions evaluated after filtering and expressions. */
|
|
63
|
-
aggregates?: readonly RelationalAggregate[];
|
|
64
|
-
/** Additional Arrow tables concatenated in source order before relational operators. */
|
|
65
|
-
union?: readonly ArrowChildQuery[];
|
|
66
|
-
/** Optional equi-join against a table supplied in `tables`. */
|
|
67
|
-
join?: Readonly<{child: ArrowChildQuery; left: string; right: string}>;
|
|
68
|
-
/** Named in-memory child tables used by union and join operators. */
|
|
69
|
-
tables?: Readonly<Record<string, ArrowTable>>;
|
|
70
|
-
}>;
|
|
71
|
-
|
|
72
|
-
/** Explains an Arrow table query without materializing or scanning table rows. */
|
|
73
|
-
export function explainArrowTableQuery(sourceTable: ArrowTable, options: ArrowQueryOptions = {}) {
|
|
74
|
-
if (options.predicate) {
|
|
75
|
-
validateSQLPredicate(options.predicate);
|
|
76
|
-
}
|
|
77
|
-
return explainColumnarTableQuery(
|
|
78
|
-
sourceTable.data.schema.fields.map(field => field.name),
|
|
79
|
-
options,
|
|
80
|
-
ARROW_TABLE_QUERY_CAPABILITIES
|
|
81
|
-
);
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
type SQLTruthValue = boolean | null;
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* Executes a small, Arrow-native query over one in-memory table.
|
|
88
|
-
*
|
|
89
|
-
* The proof of concept supports portable predicates, column projection, and limits. Projection
|
|
90
|
-
* and limits without a predicate preserve Arrow's zero-copy views. Predicate evaluation is
|
|
91
|
-
* currently row-oriented and materializes the matching result rows; later physical operators can
|
|
92
|
-
* replace this implementation with vectorized selection handling without changing this API.
|
|
93
|
-
*/
|
|
94
|
-
export function queryArrowTable(
|
|
95
|
-
sourceTable: ArrowTable,
|
|
96
|
-
options: ArrowQueryOptions = {}
|
|
97
|
-
): ArrowTable {
|
|
98
|
-
throwIfAborted(options.signal);
|
|
99
|
-
|
|
100
|
-
if (options.join) {
|
|
101
|
-
validateArrowJoinQuery(sourceTable, options);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
if (options.join && hasRelationalOperatorsExceptJoin(options)) {
|
|
105
|
-
throw new Error('Arrow joins cannot yet be combined with other relational operators.');
|
|
106
|
-
}
|
|
107
|
-
if (options.join && !hasRelationalOperatorsExceptJoin(options)) {
|
|
108
|
-
return queryArrowJoin(sourceTable, options);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
if (hasRelationalOperators(options)) {
|
|
112
|
-
return queryArrowTableRelational(sourceTable, options);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
const sourceData = sourceTable.data;
|
|
116
|
-
const plan = planTableQuery(
|
|
117
|
-
sourceData.schema.fields.map(field => field.name),
|
|
118
|
-
options
|
|
119
|
-
);
|
|
120
|
-
const scannedData = sourceData.select([...getScanStep(plan).columns]);
|
|
121
|
-
const projectedData = scannedData.select([...getProjectStep(plan).columns]);
|
|
122
|
-
const predicate = getFilterStep(plan)?.predicate;
|
|
123
|
-
const limit = getLimitStep(plan)?.limit ?? Number.POSITIVE_INFINITY;
|
|
124
|
-
if (!predicate) {
|
|
125
|
-
return wrapArrowTable(projectedData.slice(0, limit));
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
const predicateColumns = getPredicateColumns(scannedData, getSQLPredicateColumnNames(predicate));
|
|
129
|
-
const matchingRowIndices: number[] = [];
|
|
130
|
-
for (
|
|
131
|
-
let rowIndex = 0;
|
|
132
|
-
rowIndex < scannedData.numRows && matchingRowIndices.length < limit;
|
|
133
|
-
rowIndex++
|
|
134
|
-
) {
|
|
135
|
-
throwIfAborted(options.signal);
|
|
136
|
-
if (evaluatePredicate(predicate, predicateColumns, rowIndex) === true) {
|
|
137
|
-
matchingRowIndices.push(rowIndex);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
if (matchingRowIndices.length === 0) {
|
|
142
|
-
return wrapArrowTable(projectedData.slice(0, 0));
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
const outputColumns: Record<string, arrow.Vector> = {};
|
|
146
|
-
for (const field of projectedData.schema.fields) {
|
|
147
|
-
const sourceColumn = projectedData.getChild(field.name);
|
|
148
|
-
if (!sourceColumn) {
|
|
149
|
-
throw new Error(`Arrow query could not read column "${field.name}".`);
|
|
150
|
-
}
|
|
151
|
-
const values = matchingRowIndices.map(rowIndex => sourceColumn.get(rowIndex));
|
|
152
|
-
outputColumns[field.name] = arrow.vectorFromArray(values, sourceColumn.type);
|
|
153
|
-
}
|
|
154
|
-
return wrapArrowTable(new arrow.Table(projectedData.schema, outputColumns));
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
/** Executes the relational subset that can be evaluated against one in-memory Arrow table. */
|
|
158
|
-
function queryArrowTableRelational(
|
|
159
|
-
sourceTable: ArrowTable,
|
|
160
|
-
options: ArrowQueryOptions
|
|
161
|
-
): ArrowTable {
|
|
162
|
-
const sourceData = sourceTable.data;
|
|
163
|
-
const sourceColumnNames = sourceData.schema.fields.map(field => field.name);
|
|
164
|
-
// The shared planner is predicate-generic; this executor supplies the narrower SQL AST.
|
|
165
|
-
const relationalPlan = planRelationalQuery(sourceColumnNames, options as never);
|
|
166
|
-
const scanStep = relationalPlan.tablePlan.find(step => step.kind === 'scan');
|
|
167
|
-
if (!scanStep || scanStep.kind !== 'scan') {
|
|
168
|
-
throw new Error('Arrow relational query plan is missing a scan step.');
|
|
169
|
-
}
|
|
170
|
-
const scannedTable = wrapArrowTable(sourceData.select([...scanStep.columns]));
|
|
171
|
-
const filteredTable = queryArrowTable(scannedTable, {
|
|
172
|
-
predicate: options.predicate,
|
|
173
|
-
columns: scanStep.columns,
|
|
174
|
-
signal: options.signal
|
|
175
|
-
});
|
|
176
|
-
let rows = arrowTableToRows(filteredTable.data);
|
|
177
|
-
|
|
178
|
-
for (const child of options.union || []) {
|
|
179
|
-
const childTable = options.tables?.[child.source];
|
|
180
|
-
if (!childTable) throw new Error(`Arrow union source not found: ${child.source}`);
|
|
181
|
-
const childResult = queryArrowTable(childTable, {
|
|
182
|
-
...(child.query || {}),
|
|
183
|
-
columns: getUnionChildColumns(sourceColumnNames, options),
|
|
184
|
-
signal: options.signal
|
|
185
|
-
});
|
|
186
|
-
rows = rows.concat(arrowTableToRows(childResult.data));
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
for (const expression of options.expressions || []) {
|
|
190
|
-
for (const row of rows) {
|
|
191
|
-
row[expression.name] = evaluateRelationalExpression(expression, row);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
if (options.groupBy?.length || options.aggregates?.length) {
|
|
196
|
-
rows = aggregateRows(rows, options.groupBy || [], options.aggregates || []);
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
if (options.orderBy?.length) {
|
|
200
|
-
const orderBy = options.orderBy;
|
|
201
|
-
rows = rows
|
|
202
|
-
.map((row, index) => ({row, index}))
|
|
203
|
-
.sort((left, right) => compareRows(left.row, right.row, orderBy) || left.index - right.index)
|
|
204
|
-
.map(entry => entry.row);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
const outputColumns =
|
|
208
|
-
options.columns ||
|
|
209
|
-
(options.groupBy?.length || options.aggregates?.length
|
|
210
|
-
? [...(options.groupBy || []), ...(options.aggregates || []).map(aggregate => aggregate.name)]
|
|
211
|
-
: sourceColumnNames);
|
|
212
|
-
const outputRows = rows.map(row => {
|
|
213
|
-
const output: Record<string, unknown> = {};
|
|
214
|
-
for (const columnName of outputColumns) {
|
|
215
|
-
if (!(columnName in row)) {
|
|
216
|
-
throw new Error(`Arrow relational query column not found: ${columnName}`);
|
|
217
|
-
}
|
|
218
|
-
output[columnName] = row[columnName];
|
|
219
|
-
}
|
|
220
|
-
return output;
|
|
221
|
-
});
|
|
222
|
-
const limit = options.limit === undefined ? outputRows.length : options.limit;
|
|
223
|
-
const limitedRows = outputRows.slice(0, limit);
|
|
224
|
-
return limitedRows.length
|
|
225
|
-
? convertRowsToArrowTable(limitedRows)
|
|
226
|
-
: createEmptyRelationalResult(sourceData, outputColumns, options);
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
/** Derives the source columns a UNION child needs without inheriting base-only predicate columns. */
|
|
230
|
-
function getUnionChildColumns(
|
|
231
|
-
sourceColumnNames: readonly string[],
|
|
232
|
-
options: ArrowQueryOptions
|
|
233
|
-
): string[] {
|
|
234
|
-
const expressionNames = new Set((options.expressions || []).map(expression => expression.name));
|
|
235
|
-
const aggregateNames = new Set((options.aggregates || []).map(aggregate => aggregate.name));
|
|
236
|
-
const outputColumns =
|
|
237
|
-
options.columns ||
|
|
238
|
-
(options.groupBy?.length || options.aggregates?.length
|
|
239
|
-
? [...(options.groupBy || []), ...(options.aggregates || []).map(aggregate => aggregate.name)]
|
|
240
|
-
: sourceColumnNames);
|
|
241
|
-
const requiredColumns = new Set(
|
|
242
|
-
outputColumns.filter(column => !expressionNames.has(column) && !aggregateNames.has(column))
|
|
243
|
-
);
|
|
244
|
-
|
|
245
|
-
for (const expression of options.expressions || []) {
|
|
246
|
-
const definition = expression.expression;
|
|
247
|
-
if (definition.op === 'column') requiredColumns.add(definition.column);
|
|
248
|
-
if ('left' in definition) {
|
|
249
|
-
requiredColumns.add(definition.left);
|
|
250
|
-
requiredColumns.add(definition.right);
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
for (const key of options.orderBy || []) {
|
|
254
|
-
if (!expressionNames.has(key.column) && !aggregateNames.has(key.column)) {
|
|
255
|
-
requiredColumns.add(key.column);
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
for (const column of options.groupBy || []) requiredColumns.add(column);
|
|
259
|
-
for (const aggregate of options.aggregates || []) {
|
|
260
|
-
if (aggregate.column) requiredColumns.add(aggregate.column);
|
|
261
|
-
}
|
|
262
|
-
return sourceColumnNames.filter(column => requiredColumns.has(column));
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
/** Returns whether the query uses any in-memory relational operator. */
|
|
266
|
-
function hasRelationalOperators(options: ArrowQueryOptions): boolean {
|
|
267
|
-
return Boolean(
|
|
268
|
-
options.expressions?.length ||
|
|
269
|
-
options.orderBy?.length ||
|
|
270
|
-
options.groupBy?.length ||
|
|
271
|
-
options.aggregates?.length ||
|
|
272
|
-
options.union?.length ||
|
|
273
|
-
options.join
|
|
274
|
-
);
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
/** Returns whether a join is the only relational operator requested. */
|
|
278
|
-
function hasRelationalOperatorsExceptJoin(options: ArrowQueryOptions): boolean {
|
|
279
|
-
return Boolean(
|
|
280
|
-
options.expressions?.length ||
|
|
281
|
-
options.orderBy?.length ||
|
|
282
|
-
options.groupBy?.length ||
|
|
283
|
-
options.aggregates?.length ||
|
|
284
|
-
options.union?.length
|
|
285
|
-
);
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
/** Executes a bounded nested-loop equi-join for small in-memory Arrow proof-of-concept tables. */
|
|
289
|
-
function queryArrowJoin(sourceTable: ArrowTable, options: ArrowQueryOptions): ArrowTable {
|
|
290
|
-
const join = options.join;
|
|
291
|
-
if (!join) throw new Error('Arrow join query is missing a join definition.');
|
|
292
|
-
const childTable = options.tables?.[join.child.source];
|
|
293
|
-
if (!childTable) throw new Error(`Arrow join source not found: ${join.child.source}`);
|
|
294
|
-
const leftResult = queryArrowTable(sourceTable, {
|
|
295
|
-
predicate: options.predicate,
|
|
296
|
-
columns: undefined,
|
|
297
|
-
signal: options.signal
|
|
298
|
-
});
|
|
299
|
-
const rightResult = queryArrowTable(childTable, {
|
|
300
|
-
...(join.child.query || {}),
|
|
301
|
-
signal: options.signal
|
|
302
|
-
});
|
|
303
|
-
const leftRows = arrowTableToRows(leftResult.data);
|
|
304
|
-
const rightRows = arrowTableToRows(rightResult.data);
|
|
305
|
-
const rightPrefix = `${join.child.source}.`;
|
|
306
|
-
const defaultOutputColumns = [
|
|
307
|
-
...sourceTable.data.schema.fields.map(field => field.name),
|
|
308
|
-
...rightResult.data.schema.fields.map(field => `${rightPrefix}${field.name}`)
|
|
309
|
-
];
|
|
310
|
-
const rows: Record<string, unknown>[] = [];
|
|
311
|
-
for (const leftRow of leftRows) {
|
|
312
|
-
for (const rightRow of rightRows) {
|
|
313
|
-
if (
|
|
314
|
-
leftRow[join.left] !== null &&
|
|
315
|
-
leftRow[join.left] !== undefined &&
|
|
316
|
-
rightRow[join.right] !== null &&
|
|
317
|
-
rightRow[join.right] !== undefined &&
|
|
318
|
-
compareSortValues(leftRow[join.left], rightRow[join.right]) === 0
|
|
319
|
-
) {
|
|
320
|
-
rows.push({
|
|
321
|
-
...leftRow,
|
|
322
|
-
...Object.fromEntries(
|
|
323
|
-
Object.entries(rightRow).map(([key, value]) => [`${rightPrefix}${key}`, value])
|
|
324
|
-
)
|
|
325
|
-
});
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
const outputColumns = options.columns || defaultOutputColumns;
|
|
330
|
-
const outputRows = rows.map(row =>
|
|
331
|
-
Object.fromEntries(outputColumns.map(column => [column, row[column]]))
|
|
332
|
-
);
|
|
333
|
-
const limitedRows = outputRows.slice(0, options.limit);
|
|
334
|
-
return limitedRows.length
|
|
335
|
-
? convertRowsToArrowTable(limitedRows)
|
|
336
|
-
: createEmptyRelationalResult(
|
|
337
|
-
sourceTable.data,
|
|
338
|
-
outputColumns,
|
|
339
|
-
options,
|
|
340
|
-
new Map(
|
|
341
|
-
rightResult.data.schema.fields.map(field => [
|
|
342
|
-
`${rightPrefix}${field.name}`,
|
|
343
|
-
new arrow.Field(
|
|
344
|
-
`${rightPrefix}${field.name}`,
|
|
345
|
-
field.type,
|
|
346
|
-
field.nullable,
|
|
347
|
-
field.metadata
|
|
348
|
-
)
|
|
349
|
-
])
|
|
350
|
-
)
|
|
351
|
-
);
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
/** Validates root and child projections before dispatching to the join executor. */
|
|
355
|
-
function validateArrowJoinQuery(sourceTable: ArrowTable, options: ArrowQueryOptions): void {
|
|
356
|
-
const join = options.join;
|
|
357
|
-
if (!join) return;
|
|
358
|
-
const sourceColumnNames = sourceTable.data.schema.fields.map(field => field.name);
|
|
359
|
-
const childTable = options.tables?.[join.child.source];
|
|
360
|
-
if (!childTable) throw new Error(`Arrow join source not found: ${join.child.source}`);
|
|
361
|
-
const childColumnNames = childTable.data.schema.fields.map(field => field.name);
|
|
362
|
-
if (!sourceColumnNames.includes(join.left)) {
|
|
363
|
-
throw new Error(`Arrow join column not found: ${join.left}`);
|
|
364
|
-
}
|
|
365
|
-
if (!childColumnNames.includes(join.right)) {
|
|
366
|
-
throw new Error(`Arrow join column not found: ${join.right}`);
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
const childPrefix = `${join.child.source}.`;
|
|
370
|
-
const rootColumns = options.columns?.filter(column => !column.startsWith(childPrefix));
|
|
371
|
-
planTableQuery(sourceColumnNames, {
|
|
372
|
-
predicate: options.predicate,
|
|
373
|
-
columns: rootColumns?.length ? rootColumns : undefined,
|
|
374
|
-
limit: options.limit,
|
|
375
|
-
signal: options.signal
|
|
376
|
-
});
|
|
377
|
-
|
|
378
|
-
const childQuery = join.child.query || {};
|
|
379
|
-
planTableQuery(childColumnNames, childQuery);
|
|
380
|
-
const childOutputColumns = childQuery.columns || childColumnNames;
|
|
381
|
-
if (!childOutputColumns.includes(join.right)) {
|
|
382
|
-
throw new Error(`Arrow join child projection must include: ${join.right}`);
|
|
383
|
-
}
|
|
384
|
-
const projectedChildColumns =
|
|
385
|
-
options.columns?.filter(column => column.startsWith(childPrefix)) || [];
|
|
386
|
-
const seenChildColumns = new Set<string>();
|
|
387
|
-
for (const projectedColumn of projectedChildColumns) {
|
|
388
|
-
const childColumn = projectedColumn.slice(childPrefix.length);
|
|
389
|
-
if (!childColumnNames.includes(childColumn)) {
|
|
390
|
-
throw new Error(`Arrow join column not found: ${projectedColumn}`);
|
|
391
|
-
}
|
|
392
|
-
if (seenChildColumns.has(childColumn)) {
|
|
393
|
-
throw new Error(`Arrow query column was selected more than once: ${projectedColumn}`);
|
|
394
|
-
}
|
|
395
|
-
seenChildColumns.add(childColumn);
|
|
396
|
-
if (!childOutputColumns.includes(childColumn)) {
|
|
397
|
-
throw new Error(`Arrow join child projection does not include: ${childColumn}`);
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
/** Materializes one Arrow table into row objects for the relational proof of concept. */
|
|
403
|
-
function arrowTableToRows(data: arrow.Table): Record<string, unknown>[] {
|
|
404
|
-
const columns = data.schema.fields.map(field => ({
|
|
405
|
-
name: field.name,
|
|
406
|
-
vector: data.getChild(field.name)
|
|
407
|
-
}));
|
|
408
|
-
return Array.from({length: data.numRows}, (_, rowIndex) => {
|
|
409
|
-
const row: Record<string, unknown> = {};
|
|
410
|
-
for (const column of columns) {
|
|
411
|
-
if (!column.vector) throw new Error(`Arrow relational query could not read ${column.name}`);
|
|
412
|
-
row[column.name] = column.vector.get(rowIndex);
|
|
413
|
-
}
|
|
414
|
-
return row;
|
|
415
|
-
});
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
/** Evaluates one portable scalar expression with SQL-style null propagation. */
|
|
419
|
-
function evaluateRelationalExpression(
|
|
420
|
-
expression: RelationalExpression,
|
|
421
|
-
row: Readonly<Record<string, unknown>>
|
|
422
|
-
): unknown {
|
|
423
|
-
const definition = expression.expression;
|
|
424
|
-
if (definition.op === 'literal') return definition.value;
|
|
425
|
-
if (definition.op === 'column') return row[definition.column];
|
|
426
|
-
const left = row[definition.left];
|
|
427
|
-
const right = row[definition.right];
|
|
428
|
-
if (left === null || left === undefined || right === null || right === undefined) return null;
|
|
429
|
-
if (typeof left !== 'number' || typeof right !== 'number') {
|
|
430
|
-
throw new Error(`Arrow relational expression ${expression.name} requires numeric operands.`);
|
|
431
|
-
}
|
|
432
|
-
switch (definition.op) {
|
|
433
|
-
case 'add':
|
|
434
|
-
return left + right;
|
|
435
|
-
case 'subtract':
|
|
436
|
-
return left - right;
|
|
437
|
-
case 'multiply':
|
|
438
|
-
return left * right;
|
|
439
|
-
case 'divide':
|
|
440
|
-
return right === 0 ? null : left / right;
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
/** Compares two rows using stable, multi-key relational ordering. */
|
|
445
|
-
function compareRows(
|
|
446
|
-
left: Readonly<Record<string, unknown>>,
|
|
447
|
-
right: Readonly<Record<string, unknown>>,
|
|
448
|
-
orderBy: readonly RelationalOrderKey[]
|
|
449
|
-
): number {
|
|
450
|
-
for (const key of orderBy) {
|
|
451
|
-
const leftValue = left[key.column];
|
|
452
|
-
const rightValue = right[key.column];
|
|
453
|
-
const nulls = key.nulls || 'last';
|
|
454
|
-
if (
|
|
455
|
-
leftValue === null ||
|
|
456
|
-
leftValue === undefined ||
|
|
457
|
-
rightValue === null ||
|
|
458
|
-
rightValue === undefined
|
|
459
|
-
) {
|
|
460
|
-
if (leftValue === rightValue) continue;
|
|
461
|
-
const nullResult = leftValue === null || leftValue === undefined ? -1 : 1;
|
|
462
|
-
return nulls === 'first' ? nullResult : -nullResult;
|
|
463
|
-
}
|
|
464
|
-
const result = compareSortValues(leftValue, rightValue);
|
|
465
|
-
if (result) return (key.direction === 'desc' ? -1 : 1) * result;
|
|
466
|
-
}
|
|
467
|
-
return 0;
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
/** Compares two non-null values supported by portable ordering. */
|
|
471
|
-
function compareSortValues(left: unknown, right: unknown): number {
|
|
472
|
-
if (left instanceof Date && right instanceof Date)
|
|
473
|
-
return compareNumbers(left.getTime(), right.getTime());
|
|
474
|
-
if (typeof left === 'bigint' && typeof right === 'bigint')
|
|
475
|
-
return left === right ? 0 : left < right ? -1 : 1;
|
|
476
|
-
if (typeof left === 'number' && typeof right === 'number') return compareNumbers(left, right);
|
|
477
|
-
if (typeof left === 'string' && typeof right === 'string')
|
|
478
|
-
return left === right ? 0 : left < right ? -1 : 1;
|
|
479
|
-
if (typeof left === 'boolean' && typeof right === 'boolean') return Number(left) - Number(right);
|
|
480
|
-
throw new Error(
|
|
481
|
-
`Arrow relational order cannot compare ${getValueType(left)} with ${getValueType(right)}.`
|
|
482
|
-
);
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
/** Groups rows and computes the requested aggregate output columns. */
|
|
486
|
-
function aggregateRows(
|
|
487
|
-
rows: readonly Record<string, unknown>[],
|
|
488
|
-
groupBy: readonly string[],
|
|
489
|
-
aggregates: readonly RelationalAggregate[]
|
|
490
|
-
): Record<string, unknown>[] {
|
|
491
|
-
const groups = new Map<string, Record<string, unknown>[]>();
|
|
492
|
-
for (const row of rows) {
|
|
493
|
-
const key = serializeGroupKey(groupBy.map(column => row[column]));
|
|
494
|
-
const group = groups.get(key) || [];
|
|
495
|
-
group.push(row);
|
|
496
|
-
groups.set(key, group);
|
|
497
|
-
}
|
|
498
|
-
if (!groupBy.length && !rows.length) groups.set('[]', []);
|
|
499
|
-
return [...groups.values()].map(group => {
|
|
500
|
-
const output: Record<string, unknown> = {};
|
|
501
|
-
for (const column of groupBy) output[column] = group[0]?.[column] ?? null;
|
|
502
|
-
for (const aggregate of aggregates)
|
|
503
|
-
output[aggregate.name] = evaluateAggregate(aggregate, group);
|
|
504
|
-
return output;
|
|
505
|
-
});
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
/** Produces a collision-resistant group key for Arrow scalar values, including bigint values. */
|
|
509
|
-
function serializeGroupKey(values: readonly unknown[]): string {
|
|
510
|
-
return values
|
|
511
|
-
.map(value => {
|
|
512
|
-
if (value === null || value === undefined) return 'null:';
|
|
513
|
-
if (value instanceof Date) return `date:${value.getTime()}`;
|
|
514
|
-
if (value instanceof Uint8Array) return `bytes:${Array.from(value).join('.')}`;
|
|
515
|
-
return `${typeof value}:${String(value)}`;
|
|
516
|
-
})
|
|
517
|
-
.join('\u001f');
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
/** Creates a zero-row Arrow result while preserving source and computed output fields. */
|
|
521
|
-
function createEmptyRelationalResult(
|
|
522
|
-
sourceData: arrow.Table,
|
|
523
|
-
outputColumns: readonly string[],
|
|
524
|
-
options: ArrowQueryOptions,
|
|
525
|
-
additionalFields?: ReadonlyMap<string, arrow.Field>
|
|
526
|
-
): ArrowTable {
|
|
527
|
-
const sourceFields = new Map(sourceData.schema.fields.map(field => [field.name, field]));
|
|
528
|
-
const expressions = new Map(
|
|
529
|
-
(options.expressions || []).map(expression => [expression.name, expression])
|
|
530
|
-
);
|
|
531
|
-
const aggregates = new Map(
|
|
532
|
-
(options.aggregates || []).map(aggregate => [aggregate.name, aggregate])
|
|
533
|
-
);
|
|
534
|
-
const fields = outputColumns.map(columnName => {
|
|
535
|
-
const sourceField = sourceFields.get(columnName);
|
|
536
|
-
if (sourceField) return sourceField;
|
|
537
|
-
const additionalField = additionalFields?.get(columnName);
|
|
538
|
-
if (additionalField) return additionalField;
|
|
539
|
-
const expression = expressions.get(columnName);
|
|
540
|
-
if (expression)
|
|
541
|
-
return new arrow.Field(columnName, getExpressionType(expression, sourceFields), true);
|
|
542
|
-
const aggregate = aggregates.get(columnName);
|
|
543
|
-
if (aggregate)
|
|
544
|
-
return new arrow.Field(columnName, getAggregateType(aggregate, sourceFields), true);
|
|
545
|
-
return new arrow.Field(columnName, new arrow.Utf8(), true);
|
|
546
|
-
});
|
|
547
|
-
const columns: Record<string, arrow.Vector> = {};
|
|
548
|
-
for (const field of fields) columns[field.name] = arrow.vectorFromArray([], field.type);
|
|
549
|
-
return wrapArrowTable(new arrow.Table(new arrow.Schema(fields), columns));
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
/** Infers the Arrow type for an expression in an empty relational result. */
|
|
553
|
-
function getExpressionType(
|
|
554
|
-
expression: RelationalExpression,
|
|
555
|
-
sourceFields: ReadonlyMap<string, arrow.Field>
|
|
556
|
-
): arrow.DataType {
|
|
557
|
-
if (expression.expression.op === 'literal') {
|
|
558
|
-
const value = expression.expression.value;
|
|
559
|
-
if (typeof value === 'boolean') return new arrow.Bool();
|
|
560
|
-
if (typeof value === 'number') return new arrow.Float64();
|
|
561
|
-
return new arrow.Utf8();
|
|
562
|
-
}
|
|
563
|
-
if (expression.expression.op === 'column') {
|
|
564
|
-
return sourceFields.get(expression.expression.column)?.type || new arrow.Utf8();
|
|
565
|
-
}
|
|
566
|
-
return new arrow.Float64();
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
/** Infers the Arrow type for an aggregate in an empty relational result. */
|
|
570
|
-
function getAggregateType(
|
|
571
|
-
aggregate: RelationalAggregate,
|
|
572
|
-
sourceFields: ReadonlyMap<string, arrow.Field>
|
|
573
|
-
): arrow.DataType {
|
|
574
|
-
if (aggregate.function === 'count') return new arrow.Int32();
|
|
575
|
-
return aggregate.column
|
|
576
|
-
? sourceFields.get(aggregate.column)?.type || new arrow.Float64()
|
|
577
|
-
: new arrow.Float64();
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
/** Computes one aggregate over a group while ignoring null input values. */
|
|
581
|
-
function evaluateAggregate(
|
|
582
|
-
aggregate: RelationalAggregate,
|
|
583
|
-
rows: readonly Record<string, unknown>[]
|
|
584
|
-
): unknown {
|
|
585
|
-
if (aggregate.function === 'count')
|
|
586
|
-
return aggregate.column
|
|
587
|
-
? rows.filter(row => row[aggregate.column!] != null).length
|
|
588
|
-
: rows.length;
|
|
589
|
-
const values = rows.map(row => row[aggregate.column!]).filter(value => value != null);
|
|
590
|
-
if (!values.length) return null;
|
|
591
|
-
if (!values.every(value => typeof value === 'number'))
|
|
592
|
-
throw new Error(`Arrow aggregate ${aggregate.function} requires numeric values.`);
|
|
593
|
-
const numericValues = values as number[];
|
|
594
|
-
switch (aggregate.function) {
|
|
595
|
-
case 'sum':
|
|
596
|
-
return numericValues.reduce((sum, value) => sum + value, 0);
|
|
597
|
-
case 'avg':
|
|
598
|
-
return numericValues.reduce((sum, value) => sum + value, 0) / numericValues.length;
|
|
599
|
-
case 'min':
|
|
600
|
-
return Math.min(...numericValues);
|
|
601
|
-
case 'max':
|
|
602
|
-
return Math.max(...numericValues);
|
|
603
|
-
}
|
|
604
|
-
}
|
|
605
|
-
|
|
606
|
-
/** Creates a predicate-column lookup so values are not resolved for every source row. */
|
|
607
|
-
function getPredicateColumns(
|
|
608
|
-
sourceTable: arrow.Table,
|
|
609
|
-
columnNames: readonly string[]
|
|
610
|
-
): Readonly<Record<string, arrow.Vector>> {
|
|
611
|
-
const columns: Record<string, arrow.Vector> = {};
|
|
612
|
-
for (const columnName of columnNames) {
|
|
613
|
-
const column = sourceTable.getChild(columnName);
|
|
614
|
-
if (!column) {
|
|
615
|
-
throw new Error(`Arrow query column not found: ${columnName}`);
|
|
616
|
-
}
|
|
617
|
-
columns[columnName] = column;
|
|
618
|
-
}
|
|
619
|
-
return columns;
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
/** Evaluates a predicate against one Arrow table row using SQL three-valued Boolean logic. */
|
|
623
|
-
function evaluatePredicate(
|
|
624
|
-
predicate: SQLPredicate,
|
|
625
|
-
columns: Readonly<Record<string, arrow.Vector>>,
|
|
626
|
-
rowIndex: number
|
|
627
|
-
): SQLTruthValue {
|
|
628
|
-
if (isLogicalPredicate(predicate)) {
|
|
629
|
-
const values = predicate.args.map(child => evaluatePredicate(child, columns, rowIndex));
|
|
630
|
-
return predicate.op === 'and' ? evaluateAnd(values) : evaluateOr(values);
|
|
631
|
-
}
|
|
632
|
-
if (isNotPredicate(predicate)) {
|
|
633
|
-
const value = evaluatePredicate(predicate.args[0], columns, rowIndex);
|
|
634
|
-
return value === null ? null : !value;
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
const value = getColumnValue(columns, predicate.args[0].property, rowIndex);
|
|
638
|
-
if (isNullPredicate(predicate)) {
|
|
639
|
-
return value === null;
|
|
640
|
-
}
|
|
641
|
-
if (isInPredicate(predicate)) {
|
|
642
|
-
return evaluateIn(value, predicate.args[1]);
|
|
643
|
-
}
|
|
644
|
-
return evaluateComparison(value, predicate);
|
|
645
|
-
}
|
|
646
|
-
|
|
647
|
-
/** Returns the value of one predicate column or raises an internal invariant error. */
|
|
648
|
-
function getColumnValue(
|
|
649
|
-
columns: Readonly<Record<string, arrow.Vector>>,
|
|
650
|
-
columnName: string,
|
|
651
|
-
rowIndex: number
|
|
652
|
-
): unknown {
|
|
653
|
-
const column = columns[columnName];
|
|
654
|
-
if (!column) {
|
|
655
|
-
throw new Error(`Arrow query could not read predicate column "${columnName}".`);
|
|
656
|
-
}
|
|
657
|
-
return column.get(rowIndex);
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
/** Evaluates SQL AND with null propagation. */
|
|
661
|
-
function evaluateAnd(values: readonly SQLTruthValue[]): SQLTruthValue {
|
|
662
|
-
if (values.includes(false)) {
|
|
663
|
-
return false;
|
|
664
|
-
}
|
|
665
|
-
return values.includes(null) ? null : true;
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
/** Evaluates SQL OR with null propagation. */
|
|
669
|
-
function evaluateOr(values: readonly SQLTruthValue[]): SQLTruthValue {
|
|
670
|
-
if (values.includes(true)) {
|
|
671
|
-
return true;
|
|
672
|
-
}
|
|
673
|
-
return values.includes(null) ? null : false;
|
|
674
|
-
}
|
|
675
|
-
|
|
676
|
-
/** Evaluates a membership expression with SQL null propagation. */
|
|
677
|
-
function evaluateIn(value: unknown, candidates: readonly SQLPredicateValue[]): SQLTruthValue {
|
|
678
|
-
if (value === null || value === undefined) {
|
|
679
|
-
return null;
|
|
680
|
-
}
|
|
681
|
-
for (const candidate of candidates) {
|
|
682
|
-
if (compareValues(value, candidate) === 0) {
|
|
683
|
-
return true;
|
|
684
|
-
}
|
|
685
|
-
}
|
|
686
|
-
return false;
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
/** Evaluates a binary SQL comparison with SQL null propagation. */
|
|
690
|
-
function evaluateComparison(value: unknown, predicate: SQLComparisonPredicate): SQLTruthValue {
|
|
691
|
-
if (value === null || value === undefined) {
|
|
692
|
-
return null;
|
|
693
|
-
}
|
|
694
|
-
const comparison = compareValues(value, predicate.args[1]);
|
|
695
|
-
switch (predicate.op) {
|
|
696
|
-
case '=':
|
|
697
|
-
return comparison === 0;
|
|
698
|
-
case '<>':
|
|
699
|
-
return comparison !== 0;
|
|
700
|
-
case '<':
|
|
701
|
-
return comparison < 0;
|
|
702
|
-
case '<=':
|
|
703
|
-
return comparison <= 0;
|
|
704
|
-
case '>':
|
|
705
|
-
return comparison > 0;
|
|
706
|
-
case '>=':
|
|
707
|
-
return comparison >= 0;
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
/** Compares supported predicate values and rejects comparisons without an unambiguous ordering. */
|
|
712
|
-
function compareValues(left: unknown, right: SQLPredicateValue): number {
|
|
713
|
-
if (isSQLPredicateParameter(right)) {
|
|
714
|
-
throw new Error(
|
|
715
|
-
`Arrow query parameter ":${right.parameter}" must be bound before query execution.`
|
|
716
|
-
);
|
|
717
|
-
}
|
|
718
|
-
if (left instanceof Date && right instanceof Date) {
|
|
719
|
-
return compareNumbers(left.getTime(), right.getTime());
|
|
720
|
-
}
|
|
721
|
-
if (typeof left === 'number' && right instanceof Date) {
|
|
722
|
-
return compareNumbers(left, right.getTime());
|
|
723
|
-
}
|
|
724
|
-
if (left instanceof Uint8Array && right instanceof Uint8Array) {
|
|
725
|
-
return compareBinary(left, right);
|
|
726
|
-
}
|
|
727
|
-
if (typeof left === 'bigint' && typeof right === 'bigint') {
|
|
728
|
-
return left === right ? 0 : left < right ? -1 : 1;
|
|
729
|
-
}
|
|
730
|
-
if (typeof left === 'bigint' && typeof right === 'number' && Number.isSafeInteger(right)) {
|
|
731
|
-
return compareValues(left, BigInt(right));
|
|
732
|
-
}
|
|
733
|
-
if (typeof left === 'number' && typeof right === 'bigint' && Number.isSafeInteger(left)) {
|
|
734
|
-
return compareValues(BigInt(left), right);
|
|
735
|
-
}
|
|
736
|
-
if (typeof left === 'number' && typeof right === 'number') {
|
|
737
|
-
return compareNumbers(left, right);
|
|
738
|
-
}
|
|
739
|
-
if (typeof left === 'string' && typeof right === 'string') {
|
|
740
|
-
return left === right ? 0 : left < right ? -1 : 1;
|
|
741
|
-
}
|
|
742
|
-
if (typeof left === 'boolean' && typeof right === 'boolean') {
|
|
743
|
-
return Number(left) - Number(right);
|
|
744
|
-
}
|
|
745
|
-
throw new Error(
|
|
746
|
-
`Arrow query cannot compare ${getValueType(left)} with ${getValueType(right)} values.`
|
|
747
|
-
);
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
/** Compares two finite number values. */
|
|
751
|
-
function compareNumbers(left: number, right: number): number {
|
|
752
|
-
if (!Number.isFinite(left) || !Number.isFinite(right)) {
|
|
753
|
-
throw new Error('Arrow query cannot compare non-finite numbers.');
|
|
754
|
-
}
|
|
755
|
-
return left === right ? 0 : left < right ? -1 : 1;
|
|
756
|
-
}
|
|
757
|
-
|
|
758
|
-
/** Compares binary values lexicographically. */
|
|
759
|
-
function compareBinary(left: Uint8Array, right: Uint8Array): number {
|
|
760
|
-
const commonLength = Math.min(left.length, right.length);
|
|
761
|
-
for (let index = 0; index < commonLength; index++) {
|
|
762
|
-
if (left[index] !== right[index]) {
|
|
763
|
-
return left[index] < right[index] ? -1 : 1;
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
return left.length === right.length ? 0 : left.length < right.length ? -1 : 1;
|
|
767
|
-
}
|
|
768
|
-
|
|
769
|
-
/** Returns a human-readable type name for predicate diagnostics. */
|
|
770
|
-
function getValueType(value: unknown): string {
|
|
771
|
-
if (value instanceof Date) {
|
|
772
|
-
return 'Date';
|
|
773
|
-
}
|
|
774
|
-
if (value instanceof Uint8Array) {
|
|
775
|
-
return 'Uint8Array';
|
|
776
|
-
}
|
|
777
|
-
return typeof value;
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
/** Returns whether the predicate combines two or more child predicates. */
|
|
781
|
-
function isLogicalPredicate(predicate: SQLPredicate): predicate is SQLLogicalPredicate {
|
|
782
|
-
return predicate.op === 'and' || predicate.op === 'or';
|
|
783
|
-
}
|
|
784
|
-
|
|
785
|
-
/** Returns whether the predicate negates one child predicate. */
|
|
786
|
-
function isNotPredicate(predicate: SQLPredicate): predicate is SQLNotPredicate {
|
|
787
|
-
return predicate.op === 'not';
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
/** Returns whether the predicate tests a column for null values. */
|
|
791
|
-
function isNullPredicate(predicate: SQLPredicate): predicate is SQLNullPredicate {
|
|
792
|
-
return predicate.op === 'isNull';
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
/** Returns whether the predicate tests a column against several values. */
|
|
796
|
-
function isInPredicate(predicate: SQLPredicate): predicate is SQLInPredicate {
|
|
797
|
-
return predicate.op === 'in';
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
/** Returns the mandatory source scan from a normalized table-query plan. */
|
|
801
|
-
function getScanStep(plan: readonly TableQueryPlanStep[]): TableQueryScanStep {
|
|
802
|
-
const step = plan.find((candidate): candidate is TableQueryScanStep => candidate.kind === 'scan');
|
|
803
|
-
if (!step) {
|
|
804
|
-
throw new Error('Arrow query plan is missing a scan step.');
|
|
805
|
-
}
|
|
806
|
-
return step;
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
/** Returns the optional predicate filter from a normalized table-query plan. */
|
|
810
|
-
function getFilterStep(plan: readonly TableQueryPlanStep[]): TableQueryFilterStep | undefined {
|
|
811
|
-
return plan.find((candidate): candidate is TableQueryFilterStep => candidate.kind === 'filter');
|
|
812
|
-
}
|
|
813
|
-
|
|
814
|
-
/** Returns the mandatory output projection from a normalized table-query plan. */
|
|
815
|
-
function getProjectStep(plan: readonly TableQueryPlanStep[]): TableQueryProjectStep {
|
|
816
|
-
const step = plan.find(
|
|
817
|
-
(candidate): candidate is TableQueryProjectStep => candidate.kind === 'project'
|
|
818
|
-
);
|
|
819
|
-
if (!step) {
|
|
820
|
-
throw new Error('Arrow query plan is missing a project step.');
|
|
821
|
-
}
|
|
822
|
-
return step;
|
|
823
|
-
}
|
|
824
|
-
|
|
825
|
-
/** Returns the optional result limit from a normalized table-query plan. */
|
|
826
|
-
function getLimitStep(plan: readonly TableQueryPlanStep[]): TableQueryLimitStep | undefined {
|
|
827
|
-
return plan.find((candidate): candidate is TableQueryLimitStep => candidate.kind === 'limit');
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
/** Throws a standard cancellation error if a caller has aborted the query. */
|
|
831
|
-
function throwIfAborted(signal: AbortSignal | undefined): void {
|
|
832
|
-
if (signal?.aborted) {
|
|
833
|
-
const error = new Error('Arrow query was aborted.');
|
|
834
|
-
error.name = 'AbortError';
|
|
835
|
-
throw error;
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
|
|
839
|
-
/** Wraps an Arrow table with the loaders.gl Arrow-table shape and converted schema. */
|
|
840
|
-
function wrapArrowTable(data: arrow.Table): ArrowTable {
|
|
841
|
-
return {
|
|
842
|
-
shape: 'arrow-table',
|
|
843
|
-
schema: convertArrowToSchema(data.schema),
|
|
844
|
-
data
|
|
845
|
-
};
|
|
846
|
-
}
|