@loaders.gl/scan 5.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +52 -0
- package/README.md +219 -0
- package/dist/delta.cjs +24246 -0
- package/dist/delta.cjs.map +6 -0
- package/dist/delta.d.ts +11 -0
- package/dist/delta.d.ts.map +1 -0
- package/dist/delta.js +12 -0
- package/dist/delta.js.map +1 -0
- package/dist/dist.min.js +18475 -0
- package/dist/federated-table-scan-source.d.ts +116 -0
- package/dist/federated-table-scan-source.d.ts.map +1 -0
- package/dist/federated-table-scan-source.js +691 -0
- package/dist/federated-table-scan-source.js.map +1 -0
- package/dist/hudi.cjs +24065 -0
- package/dist/hudi.cjs.map +6 -0
- package/dist/hudi.d.ts +54 -0
- package/dist/hudi.d.ts.map +1 -0
- package/dist/hudi.js +124 -0
- package/dist/hudi.js.map +1 -0
- package/dist/iceberg.cjs +25862 -0
- package/dist/iceberg.cjs.map +6 -0
- package/dist/iceberg.d.ts +12 -0
- package/dist/iceberg.d.ts.map +1 -0
- package/dist/iceberg.js +12 -0
- package/dist/iceberg.js.map +1 -0
- package/dist/index.cjs +3485 -0
- package/dist/index.cjs.map +6 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -0
- package/dist/scan-engine.d.ts +44 -0
- package/dist/scan-engine.d.ts.map +1 -0
- package/dist/scan-engine.js +49 -0
- package/dist/scan-engine.js.map +1 -0
- package/dist/scan-query.d.ts +24 -0
- package/dist/scan-query.d.ts.map +1 -0
- package/dist/scan-query.js +5 -0
- package/dist/scan-query.js.map +1 -0
- package/dist/vector-table-scan-source.d.ts +90 -0
- package/dist/vector-table-scan-source.d.ts.map +1 -0
- package/dist/vector-table-scan-source.js +215 -0
- package/dist/vector-table-scan-source.js.map +1 -0
- package/package.json +66 -0
- package/src/delta.ts +18 -0
- package/src/federated-table-scan-source.ts +991 -0
- package/src/hudi.ts +200 -0
- package/src/iceberg.ts +36 -0
- package/src/index.ts +97 -0
- package/src/scan-engine.ts +96 -0
- package/src/scan-query.ts +27 -0
- package/src/vector-table-scan-source.ts +294 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,3485 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// dist/index.js
|
|
31
|
+
var index_exports = {};
|
|
32
|
+
__export(index_exports, {
|
|
33
|
+
AddressedVectorTableScanSource: () => AddressedVectorTableScanSource,
|
|
34
|
+
FEDERATED_TABLE_QUERY_CAPABILITIES: () => FEDERATED_TABLE_QUERY_CAPABILITIES,
|
|
35
|
+
FederatedTableScanSource: () => FederatedTableScanSource,
|
|
36
|
+
VectorFeatureTableScanSource: () => VectorFeatureTableScanSource,
|
|
37
|
+
VectorTileTableScanSource: () => VectorTileTableScanSource,
|
|
38
|
+
createScanEngine: () => createScanEngine,
|
|
39
|
+
createScanQueryMetadata: () => createScanQueryMetadata,
|
|
40
|
+
executeTableScanBatches: () => executeTableScanBatches,
|
|
41
|
+
filterTableBatch: () => filterTableBatch,
|
|
42
|
+
intersectPointCloudBounds: () => intersectPointCloudBounds,
|
|
43
|
+
makeTableScanBatch: () => makeTableScanBatch,
|
|
44
|
+
parseSQLPredicate: () => parseSQLPredicate,
|
|
45
|
+
projectTableBatch: () => projectTableBatch,
|
|
46
|
+
projectTableSchema: () => projectTableSchema,
|
|
47
|
+
registerScanBackend: () => registerScanBackend,
|
|
48
|
+
selectPointCloudScanTiles: () => selectPointCloudScanTiles,
|
|
49
|
+
truncateTableBatch: () => truncateTableBatch,
|
|
50
|
+
validatePointCloudQueryOptions: () => validatePointCloudQueryOptions
|
|
51
|
+
});
|
|
52
|
+
module.exports = __toCommonJS(index_exports);
|
|
53
|
+
|
|
54
|
+
// ../loader-utils/src/lib/scan-utils/columnar-predicate.ts
|
|
55
|
+
function getColumnarPredicateColumns(predicate) {
|
|
56
|
+
return [...new Set(getColumnarPredicatePaths(predicate).map((path) => path[0]))];
|
|
57
|
+
}
|
|
58
|
+
function getColumnarPredicatePath(property) {
|
|
59
|
+
return typeof property.property === "string" ? [property.property] : [...property.property];
|
|
60
|
+
}
|
|
61
|
+
function getColumnarPredicatePaths(predicate) {
|
|
62
|
+
const paths = /* @__PURE__ */ new Map();
|
|
63
|
+
visitColumnarPredicate(predicate, (child) => {
|
|
64
|
+
if ("property" in child.args[0]) {
|
|
65
|
+
const path = getColumnarPredicatePath(child.args[0]);
|
|
66
|
+
paths.set(path.join("\0"), path);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
return [...paths.values()];
|
|
70
|
+
}
|
|
71
|
+
function validateColumnarPredicate(predicate, availableColumns) {
|
|
72
|
+
visitColumnarPredicate(predicate, (child) => {
|
|
73
|
+
if (child.op === "and" || child.op === "or") {
|
|
74
|
+
if (child.args.length < 2)
|
|
75
|
+
throw new Error(`Columnar predicate ${child.op} requires at least two child predicates`);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
if (child.op === "not") return;
|
|
79
|
+
const leaf = child;
|
|
80
|
+
const path = getColumnarPredicatePath(leaf.args[0]);
|
|
81
|
+
if (!path.length || path.some((component) => !component)) {
|
|
82
|
+
throw new Error("Columnar predicate property path must contain non-empty strings");
|
|
83
|
+
}
|
|
84
|
+
if (!availableColumns.has(path[0]))
|
|
85
|
+
throw new Error(`Columnar predicate column not found: ${path.join(".")}`);
|
|
86
|
+
if (leaf.op === "in" && leaf.args[1].length === 0) {
|
|
87
|
+
throw new Error("Columnar predicate in requires at least one value");
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
function isColumnarPredicateParameter(value) {
|
|
92
|
+
const parameterValue = value;
|
|
93
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && Object.keys(value).length === 1 && Object.hasOwn(value, "parameter") && typeof parameterValue.parameter === "string" && parameterValue.parameter.length > 0;
|
|
94
|
+
}
|
|
95
|
+
function filterColumnarRowIndices(predicate, columns, rowCount) {
|
|
96
|
+
if (!predicate) return Array.from({ length: rowCount }, (_, index) => index);
|
|
97
|
+
const result = [];
|
|
98
|
+
for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
|
|
99
|
+
if (evaluateColumnarPredicate(predicate, columns, rowIndex) === true) result.push(rowIndex);
|
|
100
|
+
}
|
|
101
|
+
return result;
|
|
102
|
+
}
|
|
103
|
+
function evaluateColumnarPredicate(predicate, columns, rowIndex) {
|
|
104
|
+
if (predicate.op === "and" || predicate.op === "or") {
|
|
105
|
+
const results = predicate.args.map(
|
|
106
|
+
(child) => evaluateColumnarPredicate(child, columns, rowIndex)
|
|
107
|
+
);
|
|
108
|
+
return predicate.op === "and" ? results.includes(false) ? false : results.includes(null) ? null : true : results.includes(true) ? true : results.includes(null) ? null : false;
|
|
109
|
+
}
|
|
110
|
+
if (predicate.op === "not") {
|
|
111
|
+
const value2 = evaluateColumnarPredicate(predicate.args[0], columns, rowIndex);
|
|
112
|
+
return value2 === null ? null : !value2;
|
|
113
|
+
}
|
|
114
|
+
const leaf = predicate;
|
|
115
|
+
const value = getColumnarValue(columns, getColumnarPredicatePath(leaf.args[0]), rowIndex);
|
|
116
|
+
if (leaf.op === "isNull") return value === null || value === void 0;
|
|
117
|
+
if (value === null || value === void 0) return null;
|
|
118
|
+
if (leaf.op === "in")
|
|
119
|
+
return leaf.args[1].some((candidate) => compareColumnarValues(value, candidate) === 0);
|
|
120
|
+
const comparison = compareColumnarValues(value, leaf.args[1]);
|
|
121
|
+
return leaf.op === "=" ? comparison === 0 : leaf.op === "<>" ? comparison !== 0 : leaf.op === "<" ? comparison < 0 : leaf.op === "<=" ? comparison <= 0 : leaf.op === ">" ? comparison > 0 : comparison >= 0;
|
|
122
|
+
}
|
|
123
|
+
function visitColumnarPredicate(predicate, visitor) {
|
|
124
|
+
visitor(predicate);
|
|
125
|
+
if (predicate.op === "and" || predicate.op === "or")
|
|
126
|
+
predicate.args.forEach((child) => visitColumnarPredicate(child, visitor));
|
|
127
|
+
else if (predicate.op === "not") visitColumnarPredicate(predicate.args[0], visitor);
|
|
128
|
+
}
|
|
129
|
+
function getColumnarValue(columns, path, rowIndex) {
|
|
130
|
+
var _a;
|
|
131
|
+
let value = (_a = columns[path[0]]) == null ? void 0 : _a[rowIndex];
|
|
132
|
+
for (let index = 1; index < path.length && value != null; index++) value = value[path[index]];
|
|
133
|
+
return value;
|
|
134
|
+
}
|
|
135
|
+
function compareColumnarValues(left, right) {
|
|
136
|
+
if (left instanceof Uint8Array && right instanceof Uint8Array) return compareBytes(left, right);
|
|
137
|
+
if (left instanceof Uint8Array || right instanceof Uint8Array)
|
|
138
|
+
throw new Error("Columnar binary predicates require binary values on both sides");
|
|
139
|
+
if (left instanceof Date || right instanceof Date) {
|
|
140
|
+
if (!(left instanceof Date) || !(right instanceof Date))
|
|
141
|
+
throw new Error("Columnar date predicates require date values on both sides");
|
|
142
|
+
return compareNumbers(left.getTime(), right.getTime());
|
|
143
|
+
}
|
|
144
|
+
const leftType = typeof left;
|
|
145
|
+
const rightType = typeof right;
|
|
146
|
+
const numeric = (leftType === "number" || leftType === "bigint") && (rightType === "number" || rightType === "bigint");
|
|
147
|
+
if (!numeric && leftType !== rightType)
|
|
148
|
+
throw new Error(`Columnar predicate cannot compare ${leftType} with ${rightType}`);
|
|
149
|
+
if (!numeric && leftType !== "string" && leftType !== "boolean")
|
|
150
|
+
throw new Error(`Columnar predicate does not support ${leftType} values`);
|
|
151
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
152
|
+
}
|
|
153
|
+
function compareNumbers(left, right) {
|
|
154
|
+
if (!Number.isFinite(left) || !Number.isFinite(right))
|
|
155
|
+
throw new Error("Non-finite values cannot be ordered by a columnar predicate");
|
|
156
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
157
|
+
}
|
|
158
|
+
function compareBytes(left, right) {
|
|
159
|
+
const length = Math.min(left.length, right.length);
|
|
160
|
+
for (let index = 0; index < length; index++)
|
|
161
|
+
if (left[index] !== right[index]) return left[index] < right[index] ? -1 : 1;
|
|
162
|
+
return left.length - right.length;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// ../loader-utils/src/lib/scan-utils/scan-query-metadata.ts
|
|
166
|
+
var import_crs = require("@math.gl/crs");
|
|
167
|
+
function emitScanExecutionTelemetry(callback, telemetry) {
|
|
168
|
+
if (!callback) return;
|
|
169
|
+
try {
|
|
170
|
+
callback(telemetry);
|
|
171
|
+
} catch {
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
function createScanQueryMetadata(options) {
|
|
175
|
+
validateScanExecutionSupport(options.queryType, options.execution);
|
|
176
|
+
const schema = Object.freeze({
|
|
177
|
+
fields: Object.freeze(
|
|
178
|
+
options.schema.fields.map(
|
|
179
|
+
(field) => Object.freeze({ ...field, metadata: Object.freeze({ ...field.metadata }) })
|
|
180
|
+
)
|
|
181
|
+
),
|
|
182
|
+
metadata: Object.freeze({ ...options.schema.metadata })
|
|
183
|
+
});
|
|
184
|
+
const columns = schema.fields.map(
|
|
185
|
+
(field) => {
|
|
186
|
+
var _a;
|
|
187
|
+
return Object.freeze({
|
|
188
|
+
name: field.name,
|
|
189
|
+
type: field.type,
|
|
190
|
+
nullable: field.nullable !== false,
|
|
191
|
+
role: ((_a = options.columnRoles) == null ? void 0 : _a[field.name]) || "attribute",
|
|
192
|
+
title: getNonEmptyMetadataValue(field.metadata, "title"),
|
|
193
|
+
description: getNonEmptyMetadataValue(field.metadata, "description"),
|
|
194
|
+
metadata: field.metadata || Object.freeze({})
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
);
|
|
198
|
+
return Object.freeze({
|
|
199
|
+
sourceType: options.sourceType,
|
|
200
|
+
queryType: options.queryType,
|
|
201
|
+
execution: Object.freeze({ ...options.execution }),
|
|
202
|
+
name: options.name,
|
|
203
|
+
description: options.description,
|
|
204
|
+
schema,
|
|
205
|
+
columns: Object.freeze(columns),
|
|
206
|
+
capabilities: Object.freeze({ ...options.capabilities }),
|
|
207
|
+
spatial: options.spatial ? Object.freeze({
|
|
208
|
+
...options.spatial,
|
|
209
|
+
bounds: options.spatial.bounds ? Object.freeze({
|
|
210
|
+
minimum: Object.freeze([...options.spatial.bounds.minimum]),
|
|
211
|
+
maximum: Object.freeze([...options.spatial.bounds.maximum])
|
|
212
|
+
}) : void 0,
|
|
213
|
+
coordinateReferenceSystems: options.spatial.coordinateReferenceSystems ? Object.freeze([...options.spatial.coordinateReferenceSystems]) : void 0,
|
|
214
|
+
spatialReference: options.spatial.spatialReference ? (0, import_crs.createSpatialReference)(options.spatial.spatialReference) : void 0
|
|
215
|
+
}) : void 0,
|
|
216
|
+
statistics: options.statistics ? Object.freeze({ ...options.statistics }) : void 0,
|
|
217
|
+
levels: options.levels ? Object.freeze(
|
|
218
|
+
options.levels.map(
|
|
219
|
+
(level) => Object.freeze({
|
|
220
|
+
...level,
|
|
221
|
+
scale: level.scale ? Object.freeze([...level.scale]) : void 0
|
|
222
|
+
})
|
|
223
|
+
)
|
|
224
|
+
) : void 0
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
function validateScanExecutionSupport(queryType, execution) {
|
|
228
|
+
if (execution.status === "metadata-only") {
|
|
229
|
+
if (!execution.reason.trim()) {
|
|
230
|
+
throw new Error("Metadata-only scan support requires a concrete reason");
|
|
231
|
+
}
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const validMethods = {
|
|
235
|
+
table: ["read", "query"],
|
|
236
|
+
raster: ["getRaster"],
|
|
237
|
+
"point-cloud": ["scan"]
|
|
238
|
+
};
|
|
239
|
+
if (!validMethods[queryType].includes(execution.method)) {
|
|
240
|
+
throw new Error(
|
|
241
|
+
`Scan execution method "${execution.method}" is not valid for ${queryType} queries`
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
function getNonEmptyMetadataValue(metadata, key) {
|
|
246
|
+
const value = metadata == null ? void 0 : metadata[key];
|
|
247
|
+
return value || void 0;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ../loader-utils/src/lib/scan-utils/table-query.ts
|
|
251
|
+
function planTableQuery(sourceColumnNames, options = {}) {
|
|
252
|
+
validateTableQueryOptions(sourceColumnNames, options);
|
|
253
|
+
const outputColumnNames = options.columns ? [...options.columns] : [...sourceColumnNames];
|
|
254
|
+
const predicateColumnNames = options.predicate ? getColumnarPredicateColumns(options.predicate) : [];
|
|
255
|
+
const requiredColumnNames = /* @__PURE__ */ new Set([...outputColumnNames, ...predicateColumnNames]);
|
|
256
|
+
const scanColumnNames = sourceColumnNames.filter(
|
|
257
|
+
(columnName) => requiredColumnNames.has(columnName)
|
|
258
|
+
);
|
|
259
|
+
const steps = [
|
|
260
|
+
Object.freeze({ kind: "scan", columns: Object.freeze(scanColumnNames) })
|
|
261
|
+
];
|
|
262
|
+
if (options.predicate) {
|
|
263
|
+
steps.push(Object.freeze({ kind: "filter", predicate: options.predicate }));
|
|
264
|
+
}
|
|
265
|
+
steps.push(Object.freeze({ kind: "project", columns: Object.freeze(outputColumnNames) }));
|
|
266
|
+
if (options.limit !== void 0) {
|
|
267
|
+
steps.push(Object.freeze({ kind: "limit", limit: options.limit }));
|
|
268
|
+
}
|
|
269
|
+
return Object.freeze(steps);
|
|
270
|
+
}
|
|
271
|
+
function validateTableQueryOptions(sourceColumnNames, options) {
|
|
272
|
+
const availableColumnNames = new Set(sourceColumnNames);
|
|
273
|
+
const outputColumnNames = options.columns ?? sourceColumnNames;
|
|
274
|
+
const seenColumnNames = /* @__PURE__ */ new Set();
|
|
275
|
+
for (const columnName of outputColumnNames) {
|
|
276
|
+
if (!availableColumnNames.has(columnName)) {
|
|
277
|
+
throw new Error(`Table query column not found: ${columnName}`);
|
|
278
|
+
}
|
|
279
|
+
if (seenColumnNames.has(columnName)) {
|
|
280
|
+
throw new Error(`Table query column was selected more than once: ${columnName}`);
|
|
281
|
+
}
|
|
282
|
+
seenColumnNames.add(columnName);
|
|
283
|
+
}
|
|
284
|
+
validateTableQueryLimit(options.limit);
|
|
285
|
+
if (options.predicate) {
|
|
286
|
+
validateColumnarPredicate(options.predicate, availableColumnNames);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
function validateTableQueryLimit(limit) {
|
|
290
|
+
if (limit !== void 0 && (!Number.isSafeInteger(limit) || limit < 0)) {
|
|
291
|
+
throw new Error("Table query limit must be a non-negative safe integer.");
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// ../loader-utils/src/lib/scan-utils/table-scan-batch.ts
|
|
296
|
+
function makeTableScanBatch(table) {
|
|
297
|
+
return { ...table, batchType: "data", length: table.data.numRows };
|
|
298
|
+
}
|
|
299
|
+
async function* executeTableScanBatches(readBatches, options = {}, operators = {}) {
|
|
300
|
+
var _a;
|
|
301
|
+
validateTableQueryLimit(options.limit);
|
|
302
|
+
const startedAt = Date.now();
|
|
303
|
+
let sourcesRead = 0;
|
|
304
|
+
let batchesRead = 0;
|
|
305
|
+
let rowsRead = 0;
|
|
306
|
+
let rowsTested = 0;
|
|
307
|
+
let rowsRetained = 0;
|
|
308
|
+
let rowsReturned = 0;
|
|
309
|
+
let bytesFetched = 0;
|
|
310
|
+
let status = "early-terminated";
|
|
311
|
+
let earlyTerminationReason;
|
|
312
|
+
let executionError;
|
|
313
|
+
let remaining = options.limit === void 0 ? Number.POSITIVE_INFINITY : Math.max(0, options.limit);
|
|
314
|
+
try {
|
|
315
|
+
if (remaining <= 0) {
|
|
316
|
+
earlyTerminationReason = "limit";
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
sourcesRead = 1;
|
|
320
|
+
for await (const batch of readBatches(
|
|
321
|
+
options.signal,
|
|
322
|
+
(byteLength) => bytesFetched += byteLength
|
|
323
|
+
)) {
|
|
324
|
+
if (remaining <= 0) {
|
|
325
|
+
earlyTerminationReason = "limit";
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
batchesRead++;
|
|
329
|
+
rowsRead += batch.length;
|
|
330
|
+
const filteredBatch = (operators.filter || filterTableBatch)(batch, options.predicate);
|
|
331
|
+
if (options.predicate) {
|
|
332
|
+
rowsTested += batch.length;
|
|
333
|
+
rowsRetained += filteredBatch.length;
|
|
334
|
+
}
|
|
335
|
+
const projectedBatch = (operators.project || projectTableBatch)(
|
|
336
|
+
filteredBatch,
|
|
337
|
+
options.columns
|
|
338
|
+
);
|
|
339
|
+
if (projectedBatch.length <= remaining) {
|
|
340
|
+
remaining -= projectedBatch.length;
|
|
341
|
+
rowsReturned += projectedBatch.length;
|
|
342
|
+
yield projectedBatch;
|
|
343
|
+
} else {
|
|
344
|
+
const outputBatch = truncateTableBatch(projectedBatch, remaining);
|
|
345
|
+
rowsReturned += outputBatch.length;
|
|
346
|
+
earlyTerminationReason = "limit";
|
|
347
|
+
yield outputBatch;
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
status = "completed";
|
|
352
|
+
} catch (error) {
|
|
353
|
+
status = ((_a = options.signal) == null ? void 0 : _a.aborted) ? "cancelled" : "failed";
|
|
354
|
+
executionError = error;
|
|
355
|
+
throw error;
|
|
356
|
+
} finally {
|
|
357
|
+
if (status === "early-terminated" && !earlyTerminationReason) {
|
|
358
|
+
earlyTerminationReason = "consumer-return";
|
|
359
|
+
}
|
|
360
|
+
emitScanExecutionTelemetry(
|
|
361
|
+
options.onTelemetry,
|
|
362
|
+
Object.freeze({
|
|
363
|
+
status,
|
|
364
|
+
sourcesPlanned: 1,
|
|
365
|
+
sourcesRead,
|
|
366
|
+
batchesRead,
|
|
367
|
+
batchesDecoded: batchesRead,
|
|
368
|
+
rowsRead,
|
|
369
|
+
rowsTested: rowsTested || void 0,
|
|
370
|
+
rowsRetained: rowsRetained || void 0,
|
|
371
|
+
rowsReturned,
|
|
372
|
+
bytesRead: bytesFetched,
|
|
373
|
+
bytesFetched,
|
|
374
|
+
filesOpened: sourcesRead,
|
|
375
|
+
tasksOpened: sourcesRead,
|
|
376
|
+
durationMilliseconds: Date.now() - startedAt,
|
|
377
|
+
earlyTerminationReason,
|
|
378
|
+
...executionError === void 0 ? {} : { error: executionError }
|
|
379
|
+
})
|
|
380
|
+
);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
function truncateTableBatch(batch, length) {
|
|
384
|
+
if (batch.shape === "object-row-table" || batch.shape === "array-row-table") {
|
|
385
|
+
return { ...batch, data: batch.data.slice(0, length), length };
|
|
386
|
+
}
|
|
387
|
+
if (batch.shape === "columnar-table") {
|
|
388
|
+
return {
|
|
389
|
+
...batch,
|
|
390
|
+
data: Object.fromEntries(
|
|
391
|
+
Object.entries(batch.data).map(([name, values]) => [
|
|
392
|
+
name,
|
|
393
|
+
Array.from(values).slice(0, length)
|
|
394
|
+
])
|
|
395
|
+
),
|
|
396
|
+
length
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
if (batch.shape === "arrow-table") return { ...batch, data: batch.data.slice(0, length), length };
|
|
400
|
+
return { ...batch, features: batch.features.slice(0, length), length };
|
|
401
|
+
}
|
|
402
|
+
function projectTableBatch(batch, columns) {
|
|
403
|
+
if (columns === void 0) return batch;
|
|
404
|
+
const schema = projectTableSchema(batch, columns);
|
|
405
|
+
if (batch.shape === "object-row-table") {
|
|
406
|
+
return {
|
|
407
|
+
...batch,
|
|
408
|
+
schema,
|
|
409
|
+
data: batch.data.map((row) => Object.fromEntries(columns.map((column) => [column, row[column]])))
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
if (batch.shape === "columnar-table") {
|
|
413
|
+
return {
|
|
414
|
+
...batch,
|
|
415
|
+
schema,
|
|
416
|
+
data: Object.fromEntries(columns.map((column) => [column, batch.data[column]]))
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
if (batch.shape === "arrow-table")
|
|
420
|
+
return { ...batch, schema, data: batch.data.select([...columns]) };
|
|
421
|
+
return batch;
|
|
422
|
+
}
|
|
423
|
+
function projectTableSchema(batch, columns) {
|
|
424
|
+
if (!batch.schema) return void 0;
|
|
425
|
+
const fieldsByName = new Map(batch.schema.fields.map((field) => [field.name, field]));
|
|
426
|
+
return {
|
|
427
|
+
...batch.schema,
|
|
428
|
+
fields: columns.flatMap((column) => {
|
|
429
|
+
const field = fieldsByName.get(column);
|
|
430
|
+
return field ? [field] : [];
|
|
431
|
+
})
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
function filterTableBatch(batch, predicate) {
|
|
435
|
+
var _a;
|
|
436
|
+
if (!predicate || batch.shape !== "object-row-table") return batch;
|
|
437
|
+
const rows = batch.data;
|
|
438
|
+
const columnNames = new Set(((_a = batch.schema) == null ? void 0 : _a.fields.map((field) => field.name)) || []);
|
|
439
|
+
validateColumnarPredicate(predicate, columnNames);
|
|
440
|
+
const columns = Object.fromEntries(
|
|
441
|
+
[...columnNames].map((name) => [name, rows.map((row) => row[name])])
|
|
442
|
+
);
|
|
443
|
+
const rowIndices = filterColumnarRowIndices(predicate, columns, rows.length);
|
|
444
|
+
return { ...batch, data: rowIndices.map((rowIndex) => rows[rowIndex]), length: rowIndices.length };
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// ../loader-utils/src/lib/scan-utils/relational-query.ts
|
|
448
|
+
function planRelationalQuery(sourceColumnNames, options = {}) {
|
|
449
|
+
var _a, _b, _c, _d, _e, _f;
|
|
450
|
+
const available = new Set(sourceColumnNames);
|
|
451
|
+
const expressionNames = /* @__PURE__ */ new Set();
|
|
452
|
+
for (const expression of options.expressions || []) {
|
|
453
|
+
if (!expression.name) throw new Error("Relational expression names must be non-empty.");
|
|
454
|
+
if (available.has(expression.name) || expressionNames.has(expression.name)) {
|
|
455
|
+
throw new Error(`Relational expression duplicates column: ${expression.name}`);
|
|
456
|
+
}
|
|
457
|
+
expressionNames.add(expression.name);
|
|
458
|
+
}
|
|
459
|
+
const aggregateNames = new Set((options.aggregates || []).map((aggregate) => aggregate.name));
|
|
460
|
+
const requiredColumns = /* @__PURE__ */ new Set();
|
|
461
|
+
const availableColumns = new Set(sourceColumnNames);
|
|
462
|
+
for (const expression of options.expressions || []) {
|
|
463
|
+
const definition = expression.expression;
|
|
464
|
+
if (definition.op === "column") {
|
|
465
|
+
validateRelationalColumnReference(definition.column, availableColumns);
|
|
466
|
+
if (available.has(definition.column)) requiredColumns.add(definition.column);
|
|
467
|
+
}
|
|
468
|
+
if ("left" in definition) {
|
|
469
|
+
validateRelationalColumnReference(definition.left, availableColumns);
|
|
470
|
+
validateRelationalColumnReference(definition.right, availableColumns);
|
|
471
|
+
if (available.has(definition.left)) requiredColumns.add(definition.left);
|
|
472
|
+
if (available.has(definition.right)) requiredColumns.add(definition.right);
|
|
473
|
+
}
|
|
474
|
+
availableColumns.add(expression.name);
|
|
475
|
+
}
|
|
476
|
+
for (const key of options.orderBy || []) requiredColumns.add(key.column);
|
|
477
|
+
for (const key of options.groupBy || []) requiredColumns.add(key);
|
|
478
|
+
for (const aggregate of options.aggregates || []) {
|
|
479
|
+
if (aggregate.column) requiredColumns.add(aggregate.column);
|
|
480
|
+
}
|
|
481
|
+
const outputColumns = (_a = options.columns) == null ? void 0 : _a.filter(
|
|
482
|
+
(column) => !expressionNames.has(column) && !aggregateNames.has(column)
|
|
483
|
+
);
|
|
484
|
+
const tablePlan = [
|
|
485
|
+
...planTableQuery(sourceColumnNames, {
|
|
486
|
+
...options,
|
|
487
|
+
columns: outputColumns,
|
|
488
|
+
predicate: options.predicate
|
|
489
|
+
})
|
|
490
|
+
];
|
|
491
|
+
const scanStep = tablePlan[0];
|
|
492
|
+
if ((scanStep == null ? void 0 : scanStep.kind) === "scan") {
|
|
493
|
+
const scanColumns = new Set(scanStep.columns);
|
|
494
|
+
for (const column of requiredColumns) {
|
|
495
|
+
if (available.has(column)) scanColumns.add(column);
|
|
496
|
+
}
|
|
497
|
+
tablePlan[0] = Object.freeze({ kind: "scan", columns: Object.freeze([...scanColumns]) });
|
|
498
|
+
}
|
|
499
|
+
for (const expression of expressionNames) available.add(expression);
|
|
500
|
+
for (const aggregate of aggregateNames) available.add(aggregate);
|
|
501
|
+
for (const key of options.orderBy || []) {
|
|
502
|
+
if (!available.has(key.column))
|
|
503
|
+
throw new Error(`Relational order column not found: ${key.column}`);
|
|
504
|
+
}
|
|
505
|
+
for (const key of options.groupBy || []) {
|
|
506
|
+
if (!available.has(key)) throw new Error(`Relational group column not found: ${key}`);
|
|
507
|
+
}
|
|
508
|
+
for (const aggregate of options.aggregates || []) {
|
|
509
|
+
if (!aggregate.name) throw new Error("Relational aggregate names must be non-empty.");
|
|
510
|
+
if (aggregate.function !== "count" && !aggregate.column) {
|
|
511
|
+
throw new Error(`Relational aggregate ${aggregate.function} requires a column.`);
|
|
512
|
+
}
|
|
513
|
+
if (aggregate.column && !available.has(aggregate.column)) {
|
|
514
|
+
throw new Error(`Relational aggregate column not found: ${aggregate.column}`);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
const relationalSteps = [];
|
|
518
|
+
if ((_b = options.expressions) == null ? void 0 : _b.length)
|
|
519
|
+
relationalSteps.push({ kind: "expression", detail: options.expressions });
|
|
520
|
+
if ((_c = options.orderBy) == null ? void 0 : _c.length) relationalSteps.push({ kind: "order", detail: options.orderBy });
|
|
521
|
+
if (((_d = options.aggregates) == null ? void 0 : _d.length) || ((_e = options.groupBy) == null ? void 0 : _e.length)) {
|
|
522
|
+
relationalSteps.push({
|
|
523
|
+
kind: "aggregate",
|
|
524
|
+
detail: { groupBy: options.groupBy, aggregates: options.aggregates }
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
if ((_f = options.union) == null ? void 0 : _f.length) relationalSteps.push({ kind: "union", detail: options.union });
|
|
528
|
+
if (options.join) relationalSteps.push({ kind: "join", detail: options.join });
|
|
529
|
+
return { tablePlan, relationalSteps: Object.freeze(relationalSteps) };
|
|
530
|
+
}
|
|
531
|
+
function validateRelationalColumnReference(columnName, availableColumns) {
|
|
532
|
+
if (!availableColumns.has(columnName)) {
|
|
533
|
+
throw new Error(`Relational expression column not found: ${columnName}`);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// ../loader-utils/src/lib/scan-utils/point-cloud-query.ts
|
|
538
|
+
function validatePointCloudQueryOptions(sourceColumnNames, options) {
|
|
539
|
+
validateTableQueryOptions(sourceColumnNames, options);
|
|
540
|
+
validatePointCloudBounds(options.bounds);
|
|
541
|
+
validateHierarchyLevel(options.minimumLevel, "minimumLevel");
|
|
542
|
+
validateHierarchyLevel(options.maximumLevel, "maximumLevel");
|
|
543
|
+
if (options.minimumLevel !== void 0 && options.maximumLevel !== void 0 && options.minimumLevel > options.maximumLevel) {
|
|
544
|
+
throw new Error("Point cloud query minimumLevel cannot exceed maximumLevel.");
|
|
545
|
+
}
|
|
546
|
+
if (options.targetSpacing !== void 0 && (!Number.isFinite(options.targetSpacing) || options.targetSpacing <= 0)) {
|
|
547
|
+
throw new Error("Point cloud query targetSpacing must be a positive finite number.");
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
async function* selectPointCloudScanTiles(rootTile, loadChildren, options = {}) {
|
|
551
|
+
const pendingTiles = [rootTile];
|
|
552
|
+
const minimumLevel = options.minimumLevel ?? 0;
|
|
553
|
+
while (pendingTiles.length > 0) {
|
|
554
|
+
throwIfPointCloudScanAborted(options.signal);
|
|
555
|
+
const tile = pendingTiles.shift();
|
|
556
|
+
if (options.bounds && !intersectPointCloudBounds(tile.bounds, options.bounds)) {
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
const satisfiesMinimumLevel = tile.level >= minimumLevel;
|
|
560
|
+
if (satisfiesMinimumLevel) {
|
|
561
|
+
yield tile;
|
|
562
|
+
}
|
|
563
|
+
const reachedMaximumLevel = options.maximumLevel !== void 0 && tile.level >= options.maximumLevel;
|
|
564
|
+
const reachedTargetSpacing = satisfiesMinimumLevel && options.targetSpacing !== void 0 && tile.geometricError <= options.targetSpacing;
|
|
565
|
+
if (reachedMaximumLevel || reachedTargetSpacing) {
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
const children = [...await loadChildren(tile)].sort(
|
|
569
|
+
(left, right) => left.id.localeCompare(right.id)
|
|
570
|
+
);
|
|
571
|
+
pendingTiles.push(...children);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
function intersectPointCloudBounds(left, right) {
|
|
575
|
+
for (let dimension = 0; dimension < 3; dimension++) {
|
|
576
|
+
if (left.maximum[dimension] < right.minimum[dimension] || left.minimum[dimension] > right.maximum[dimension]) {
|
|
577
|
+
return false;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
return true;
|
|
581
|
+
}
|
|
582
|
+
function validatePointCloudBounds(bounds) {
|
|
583
|
+
if (!bounds) {
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
for (let dimension = 0; dimension < 3; dimension++) {
|
|
587
|
+
const minimum = bounds.minimum[dimension];
|
|
588
|
+
const maximum = bounds.maximum[dimension];
|
|
589
|
+
if (!Number.isFinite(minimum) || !Number.isFinite(maximum)) {
|
|
590
|
+
throw new Error("Point cloud query bounds must contain finite coordinates.");
|
|
591
|
+
}
|
|
592
|
+
if (minimum > maximum) {
|
|
593
|
+
throw new Error("Point cloud query bounds minimum cannot exceed maximum.");
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
function validateHierarchyLevel(level, name) {
|
|
598
|
+
if (level !== void 0 && (!Number.isSafeInteger(level) || level < 0)) {
|
|
599
|
+
throw new Error(`Point cloud query ${name} must be a non-negative safe integer.`);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
function throwIfPointCloudScanAborted(signal) {
|
|
603
|
+
if (signal == null ? void 0 : signal.aborted) {
|
|
604
|
+
throw new DOMException("The operation was aborted", "AbortError");
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
// ../loader-utils/src/lib/scan-utils/table-query-explain.ts
|
|
609
|
+
function explainTableQuery(sourceColumnNames, options, capabilities) {
|
|
610
|
+
const plan = planTableQuery(sourceColumnNames, options);
|
|
611
|
+
const outputColumns = options.columns ? [...options.columns] : [...sourceColumnNames];
|
|
612
|
+
const predicateColumns = options.predicate ? getColumnarPredicateColumns(options.predicate) : [];
|
|
613
|
+
const requiredColumns = [...plan[0].kind === "scan" ? plan[0].columns : []];
|
|
614
|
+
return Object.freeze({
|
|
615
|
+
sourceColumns: Object.freeze([...sourceColumnNames]),
|
|
616
|
+
outputColumns: Object.freeze(outputColumns),
|
|
617
|
+
requiredColumns: Object.freeze(requiredColumns),
|
|
618
|
+
predicateColumns: Object.freeze(predicateColumns),
|
|
619
|
+
plan,
|
|
620
|
+
capabilities: Object.freeze({ ...capabilities }),
|
|
621
|
+
operators: Object.freeze({
|
|
622
|
+
projection: createOperatorExplanation(options.columns !== void 0, capabilities.projection),
|
|
623
|
+
predicate: createOperatorExplanation(options.predicate !== void 0, capabilities.predicate),
|
|
624
|
+
limit: createOperatorExplanation(options.limit !== void 0, capabilities.limit)
|
|
625
|
+
})
|
|
626
|
+
});
|
|
627
|
+
}
|
|
628
|
+
function createOperatorExplanation(enabled, support) {
|
|
629
|
+
return Object.freeze({ enabled, support });
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
// ../sql/src/sql-predicate-types.ts
|
|
633
|
+
function isSQLPredicateParameter(value) {
|
|
634
|
+
return isColumnarPredicateParameter(value);
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// ../sql/src/sql-predicate-schema.ts
|
|
638
|
+
var MAXIMUM_PREDICATE_DEPTH = 64;
|
|
639
|
+
var MAXIMUM_PREDICATE_NODES = 4096;
|
|
640
|
+
function validateSQLPredicate(value) {
|
|
641
|
+
const state = { nodes: 0 };
|
|
642
|
+
validatePredicateNode(value, "$", 0, state);
|
|
643
|
+
}
|
|
644
|
+
function validatePredicateNode(value, path, depth, state) {
|
|
645
|
+
if (depth > MAXIMUM_PREDICATE_DEPTH) {
|
|
646
|
+
throw new Error(`SQL predicate ${path} exceeds the maximum nesting depth`);
|
|
647
|
+
}
|
|
648
|
+
state.nodes++;
|
|
649
|
+
if (state.nodes > MAXIMUM_PREDICATE_NODES) {
|
|
650
|
+
throw new Error("SQL predicate exceeds the maximum node count");
|
|
651
|
+
}
|
|
652
|
+
if (!isRecord(value) || !hasExactKeys(value, ["op", "args"])) {
|
|
653
|
+
throw new Error(`SQL predicate ${path} must contain only op and args`);
|
|
654
|
+
}
|
|
655
|
+
if (typeof value.op !== "string" || !Array.isArray(value.args)) {
|
|
656
|
+
throw new Error(`SQL predicate ${path} requires string op and array args`);
|
|
657
|
+
}
|
|
658
|
+
switch (value.op) {
|
|
659
|
+
case "and":
|
|
660
|
+
case "or":
|
|
661
|
+
if (value.args.length < 2) {
|
|
662
|
+
throw new Error(`SQL predicate ${path}.${value.op} requires at least two arguments`);
|
|
663
|
+
}
|
|
664
|
+
for (let index = 0; index < value.args.length; index++) {
|
|
665
|
+
validatePredicateNode(value.args[index], `${path}.args[${index}]`, depth + 1, state);
|
|
666
|
+
}
|
|
667
|
+
return;
|
|
668
|
+
case "not":
|
|
669
|
+
if (value.args.length !== 1) {
|
|
670
|
+
throw new Error(`SQL predicate ${path}.not requires exactly one argument`);
|
|
671
|
+
}
|
|
672
|
+
validatePredicateNode(value.args[0], `${path}.args[0]`, depth + 1, state);
|
|
673
|
+
return;
|
|
674
|
+
case "isNull":
|
|
675
|
+
if (value.args.length !== 1) {
|
|
676
|
+
throw new Error(`SQL predicate ${path}.isNull requires exactly one argument`);
|
|
677
|
+
}
|
|
678
|
+
validatePredicateProperty(value.args[0], `${path}.args[0]`);
|
|
679
|
+
return;
|
|
680
|
+
case "in":
|
|
681
|
+
if (value.args.length !== 2 || !Array.isArray(value.args[1]) || value.args[1].length === 0) {
|
|
682
|
+
throw new Error(`SQL predicate ${path}.in requires a property and non-empty value list`);
|
|
683
|
+
}
|
|
684
|
+
validatePredicateProperty(value.args[0], `${path}.args[0]`);
|
|
685
|
+
value.args[1].forEach(
|
|
686
|
+
(item, index) => validatePredicateValue(item, `${path}.args[1][${index}]`)
|
|
687
|
+
);
|
|
688
|
+
return;
|
|
689
|
+
case "=":
|
|
690
|
+
case "<>":
|
|
691
|
+
case "<":
|
|
692
|
+
case "<=":
|
|
693
|
+
case ">":
|
|
694
|
+
case ">=":
|
|
695
|
+
if (value.args.length !== 2) {
|
|
696
|
+
throw new Error(`SQL predicate ${path}.${value.op} requires exactly two arguments`);
|
|
697
|
+
}
|
|
698
|
+
validatePredicateProperty(value.args[0], `${path}.args[0]`);
|
|
699
|
+
validatePredicateValue(value.args[1], `${path}.args[1]`);
|
|
700
|
+
return;
|
|
701
|
+
default:
|
|
702
|
+
throw new Error(`SQL predicate ${path} has unsupported operator ${JSON.stringify(value.op)}`);
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
function validatePredicateProperty(value, path) {
|
|
706
|
+
if (!isRecord(value) || !hasAllowedKeys(value, ["property", "quoted"]) || typeof value.property !== "string" || value.property.length === 0 || value.quoted !== void 0 && typeof value.quoted !== "boolean") {
|
|
707
|
+
throw new Error(`SQL predicate ${path} requires a non-empty property reference`);
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
function validatePredicateValue(value, path) {
|
|
711
|
+
if (isSQLPredicateParameter(value)) {
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
if (typeof value === "boolean" || typeof value === "bigint" || typeof value === "string" || value instanceof Uint8Array) {
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
if (value instanceof Date && Number.isFinite(value.getTime())) {
|
|
721
|
+
return;
|
|
722
|
+
}
|
|
723
|
+
throw new Error(`SQL predicate ${path} contains an unsupported scalar value`);
|
|
724
|
+
}
|
|
725
|
+
function isRecord(value) {
|
|
726
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
727
|
+
}
|
|
728
|
+
function hasExactKeys(value, expectedKeys) {
|
|
729
|
+
const keys = Object.keys(value);
|
|
730
|
+
return keys.length === expectedKeys.length && expectedKeys.every((key) => keys.includes(key));
|
|
731
|
+
}
|
|
732
|
+
function hasAllowedKeys(value, allowedKeys) {
|
|
733
|
+
return Object.keys(value).every((key) => allowedKeys.includes(key));
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
// ../sql/src/sql-utils.ts
|
|
737
|
+
var arrow5 = __toESM(require("apache-arrow"), 1);
|
|
738
|
+
|
|
739
|
+
// ../schema-utils/src/lib/schema/data-type.ts
|
|
740
|
+
function getDataTypeFromValue(value, defaultNumberType = "float32") {
|
|
741
|
+
if (value instanceof Date) {
|
|
742
|
+
return "date-millisecond";
|
|
743
|
+
}
|
|
744
|
+
if (typeof value === "number") {
|
|
745
|
+
return defaultNumberType;
|
|
746
|
+
}
|
|
747
|
+
if (typeof value === "boolean") {
|
|
748
|
+
return "bool";
|
|
749
|
+
}
|
|
750
|
+
if (typeof value === "string") {
|
|
751
|
+
return "utf8";
|
|
752
|
+
}
|
|
753
|
+
if (value === null || value === "undefined") {
|
|
754
|
+
return "null";
|
|
755
|
+
}
|
|
756
|
+
return "null";
|
|
757
|
+
}
|
|
758
|
+
function getDataTypeFromArray(array) {
|
|
759
|
+
let type = getDataTypeFromTypedArray(array);
|
|
760
|
+
if (type !== "null") {
|
|
761
|
+
return { type, nullable: false };
|
|
762
|
+
}
|
|
763
|
+
if (array.length > 0) {
|
|
764
|
+
type = getDataTypeFromValue(array[0]);
|
|
765
|
+
return { type, nullable: true };
|
|
766
|
+
}
|
|
767
|
+
return { type: "null", nullable: true };
|
|
768
|
+
}
|
|
769
|
+
function getDataTypeFromTypedArray(array) {
|
|
770
|
+
switch (array.constructor) {
|
|
771
|
+
case Int8Array:
|
|
772
|
+
return "int8";
|
|
773
|
+
case Uint8Array:
|
|
774
|
+
case Uint8ClampedArray:
|
|
775
|
+
return "uint8";
|
|
776
|
+
case Int16Array:
|
|
777
|
+
return "int16";
|
|
778
|
+
case Uint16Array:
|
|
779
|
+
return "uint16";
|
|
780
|
+
case Int32Array:
|
|
781
|
+
return "int32";
|
|
782
|
+
case Uint32Array:
|
|
783
|
+
return "uint32";
|
|
784
|
+
case Float32Array:
|
|
785
|
+
return "float32";
|
|
786
|
+
case Float64Array:
|
|
787
|
+
return "float64";
|
|
788
|
+
default:
|
|
789
|
+
return "null";
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
function getArrayTypeFromDataType(type, nullable) {
|
|
793
|
+
if (!nullable) {
|
|
794
|
+
switch (type) {
|
|
795
|
+
case "int8":
|
|
796
|
+
return Int8Array;
|
|
797
|
+
case "uint8":
|
|
798
|
+
return Uint8Array;
|
|
799
|
+
case "int16":
|
|
800
|
+
return Int16Array;
|
|
801
|
+
case "uint16":
|
|
802
|
+
return Uint16Array;
|
|
803
|
+
case "int32":
|
|
804
|
+
return Int32Array;
|
|
805
|
+
case "uint32":
|
|
806
|
+
return Uint32Array;
|
|
807
|
+
case "float32":
|
|
808
|
+
return Float32Array;
|
|
809
|
+
case "float64":
|
|
810
|
+
return Float64Array;
|
|
811
|
+
default:
|
|
812
|
+
break;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
return Array;
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
// ../schema-utils/src/lib/schema/convert-arrow-schema.ts
|
|
819
|
+
var arrow = __toESM(require("apache-arrow"), 1);
|
|
820
|
+
function convertArrowToSchema(arrowSchema) {
|
|
821
|
+
return serializeArrowSchema(arrowSchema);
|
|
822
|
+
}
|
|
823
|
+
function convertSchemaToArrow(schema, options) {
|
|
824
|
+
return deserializeArrowSchema(schema, options);
|
|
825
|
+
}
|
|
826
|
+
function serializeArrowSchema(arrowSchema) {
|
|
827
|
+
return {
|
|
828
|
+
fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
|
|
829
|
+
metadata: serializeArrowMetadata(arrowSchema.metadata)
|
|
830
|
+
};
|
|
831
|
+
}
|
|
832
|
+
function deserializeArrowSchema(schema, options) {
|
|
833
|
+
return new arrow.Schema(
|
|
834
|
+
schema.fields.map((field) => deserializeArrowField(field, options)),
|
|
835
|
+
deserializeArrowMetadata(schema.metadata)
|
|
836
|
+
);
|
|
837
|
+
}
|
|
838
|
+
function serializeArrowMetadata(arrowMetadata) {
|
|
839
|
+
return Object.fromEntries(arrowMetadata);
|
|
840
|
+
}
|
|
841
|
+
function deserializeArrowMetadata(metadata) {
|
|
842
|
+
return metadata ? new Map(Object.entries(metadata)) : /* @__PURE__ */ new Map();
|
|
843
|
+
}
|
|
844
|
+
function serializeArrowField(field) {
|
|
845
|
+
return {
|
|
846
|
+
name: field.name,
|
|
847
|
+
type: serializeArrowType(field.type),
|
|
848
|
+
nullable: field.nullable,
|
|
849
|
+
metadata: serializeArrowMetadata(field.metadata)
|
|
850
|
+
};
|
|
851
|
+
}
|
|
852
|
+
function deserializeArrowField(field, options) {
|
|
853
|
+
return new arrow.Field(
|
|
854
|
+
field.name,
|
|
855
|
+
deserializeArrowType(field.type, options),
|
|
856
|
+
field.nullable,
|
|
857
|
+
deserializeArrowMetadata(field.metadata)
|
|
858
|
+
);
|
|
859
|
+
}
|
|
860
|
+
function serializeArrowType(arrowType) {
|
|
861
|
+
if (isArrowViewType(arrowType, "BinaryView")) {
|
|
862
|
+
return "binary-view";
|
|
863
|
+
}
|
|
864
|
+
if (isArrowViewType(arrowType, "Utf8View")) {
|
|
865
|
+
return "utf8-view";
|
|
866
|
+
}
|
|
867
|
+
switch (arrowType.constructor) {
|
|
868
|
+
case arrow.Null:
|
|
869
|
+
return "null";
|
|
870
|
+
case arrow.Binary:
|
|
871
|
+
return "binary";
|
|
872
|
+
case arrow.FixedSizeBinary:
|
|
873
|
+
return {
|
|
874
|
+
type: "fixed-size-binary",
|
|
875
|
+
byteWidth: arrowType.byteWidth
|
|
876
|
+
};
|
|
877
|
+
case arrow.Bool:
|
|
878
|
+
return "bool";
|
|
879
|
+
case arrow.Int:
|
|
880
|
+
const intType = arrowType;
|
|
881
|
+
return `${intType.isSigned ? "" : "u"}int${intType.bitWidth}`;
|
|
882
|
+
case arrow.Int8:
|
|
883
|
+
return "int8";
|
|
884
|
+
case arrow.Int16:
|
|
885
|
+
return "int16";
|
|
886
|
+
case arrow.Int32:
|
|
887
|
+
return "int32";
|
|
888
|
+
case arrow.Int64:
|
|
889
|
+
return "int64";
|
|
890
|
+
case arrow.Uint8:
|
|
891
|
+
return "uint8";
|
|
892
|
+
case arrow.Uint16:
|
|
893
|
+
return "uint16";
|
|
894
|
+
case arrow.Uint32:
|
|
895
|
+
return "uint32";
|
|
896
|
+
case arrow.Uint64:
|
|
897
|
+
return "uint64";
|
|
898
|
+
case arrow.Float:
|
|
899
|
+
const precision = arrowType.precision;
|
|
900
|
+
switch (precision) {
|
|
901
|
+
case arrow.Precision.HALF:
|
|
902
|
+
return "float16";
|
|
903
|
+
case arrow.Precision.SINGLE:
|
|
904
|
+
return "float32";
|
|
905
|
+
case arrow.Precision.DOUBLE:
|
|
906
|
+
return "float64";
|
|
907
|
+
default:
|
|
908
|
+
return "float16";
|
|
909
|
+
}
|
|
910
|
+
case arrow.Float16:
|
|
911
|
+
return "float16";
|
|
912
|
+
case arrow.Float32:
|
|
913
|
+
return "float32";
|
|
914
|
+
case arrow.Float64:
|
|
915
|
+
return "float64";
|
|
916
|
+
case arrow.Utf8:
|
|
917
|
+
return "utf8";
|
|
918
|
+
case arrow.Decimal:
|
|
919
|
+
const decimal = arrowType;
|
|
920
|
+
return {
|
|
921
|
+
type: "decimal",
|
|
922
|
+
bitWidth: decimal.bitWidth,
|
|
923
|
+
precision: decimal.precision,
|
|
924
|
+
scale: decimal.scale
|
|
925
|
+
};
|
|
926
|
+
case arrow.Date_:
|
|
927
|
+
const dateUnit = arrowType.unit;
|
|
928
|
+
return dateUnit === arrow.DateUnit.DAY ? "date-day" : "date-millisecond";
|
|
929
|
+
case arrow.DateDay:
|
|
930
|
+
return "date-day";
|
|
931
|
+
case arrow.DateMillisecond:
|
|
932
|
+
return "date-millisecond";
|
|
933
|
+
case arrow.Time:
|
|
934
|
+
const timeUnit = arrowType.unit;
|
|
935
|
+
switch (timeUnit) {
|
|
936
|
+
case arrow.TimeUnit.SECOND:
|
|
937
|
+
return "time-second";
|
|
938
|
+
case arrow.TimeUnit.MILLISECOND:
|
|
939
|
+
return "time-millisecond";
|
|
940
|
+
case arrow.TimeUnit.MICROSECOND:
|
|
941
|
+
return "time-microsecond";
|
|
942
|
+
case arrow.TimeUnit.NANOSECOND:
|
|
943
|
+
return "time-nanosecond";
|
|
944
|
+
default:
|
|
945
|
+
return "time-second";
|
|
946
|
+
}
|
|
947
|
+
case arrow.TimeMillisecond:
|
|
948
|
+
return "time-millisecond";
|
|
949
|
+
case arrow.TimeSecond:
|
|
950
|
+
return "time-second";
|
|
951
|
+
case arrow.TimeMicrosecond:
|
|
952
|
+
return "time-microsecond";
|
|
953
|
+
case arrow.TimeNanosecond:
|
|
954
|
+
return "time-nanosecond";
|
|
955
|
+
case arrow.Timestamp:
|
|
956
|
+
const timeStampUnit = arrowType.unit;
|
|
957
|
+
switch (timeStampUnit) {
|
|
958
|
+
case arrow.TimeUnit.SECOND:
|
|
959
|
+
return "timestamp-second";
|
|
960
|
+
case arrow.TimeUnit.MILLISECOND:
|
|
961
|
+
return "timestamp-millisecond";
|
|
962
|
+
case arrow.TimeUnit.MICROSECOND:
|
|
963
|
+
return "timestamp-microsecond";
|
|
964
|
+
case arrow.TimeUnit.NANOSECOND:
|
|
965
|
+
return "timestamp-nanosecond";
|
|
966
|
+
default:
|
|
967
|
+
return "timestamp-second";
|
|
968
|
+
}
|
|
969
|
+
case arrow.TimestampSecond:
|
|
970
|
+
return "timestamp-second";
|
|
971
|
+
case arrow.TimestampMillisecond:
|
|
972
|
+
return "timestamp-millisecond";
|
|
973
|
+
case arrow.TimestampMicrosecond:
|
|
974
|
+
return "timestamp-microsecond";
|
|
975
|
+
case arrow.TimestampNanosecond:
|
|
976
|
+
return "timestamp-nanosecond";
|
|
977
|
+
case arrow.Interval:
|
|
978
|
+
const intervalUnit = arrowType.unit;
|
|
979
|
+
switch (intervalUnit) {
|
|
980
|
+
case arrow.IntervalUnit.DAY_TIME:
|
|
981
|
+
return "interval-daytime";
|
|
982
|
+
case arrow.IntervalUnit.YEAR_MONTH:
|
|
983
|
+
return "interval-yearmonth";
|
|
984
|
+
default:
|
|
985
|
+
return "interval-daytime";
|
|
986
|
+
}
|
|
987
|
+
case arrow.IntervalDayTime:
|
|
988
|
+
return "interval-daytime";
|
|
989
|
+
case arrow.IntervalYearMonth:
|
|
990
|
+
return "interval-yearmonth";
|
|
991
|
+
case arrow.Map_:
|
|
992
|
+
const mapType = arrowType;
|
|
993
|
+
return {
|
|
994
|
+
type: "map",
|
|
995
|
+
keysSorted: mapType.keysSorted,
|
|
996
|
+
children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
997
|
+
};
|
|
998
|
+
case arrow.List:
|
|
999
|
+
const listType = arrowType;
|
|
1000
|
+
const listField = listType.valueField;
|
|
1001
|
+
return {
|
|
1002
|
+
type: "list",
|
|
1003
|
+
children: [serializeArrowField(listField)]
|
|
1004
|
+
};
|
|
1005
|
+
case arrow.FixedSizeList:
|
|
1006
|
+
const fixedSizeList = arrowType;
|
|
1007
|
+
return {
|
|
1008
|
+
type: "fixed-size-list",
|
|
1009
|
+
listSize: fixedSizeList.listSize,
|
|
1010
|
+
children: [serializeArrowField(fixedSizeList.children[0])]
|
|
1011
|
+
};
|
|
1012
|
+
case arrow.Struct:
|
|
1013
|
+
const structType = arrowType;
|
|
1014
|
+
return {
|
|
1015
|
+
type: "struct",
|
|
1016
|
+
children: structType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
1017
|
+
};
|
|
1018
|
+
case arrow.Dictionary:
|
|
1019
|
+
const dictionaryType = arrowType;
|
|
1020
|
+
return {
|
|
1021
|
+
type: "dictionary",
|
|
1022
|
+
id: dictionaryType.id,
|
|
1023
|
+
indices: serializeArrowDictionaryKeyType(dictionaryType.indices),
|
|
1024
|
+
dictionary: serializeArrowType(dictionaryType.dictionary),
|
|
1025
|
+
isOrdered: dictionaryType.isOrdered
|
|
1026
|
+
};
|
|
1027
|
+
default:
|
|
1028
|
+
throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
function deserializeArrowType(dataType, options) {
|
|
1032
|
+
if (typeof dataType === "object") {
|
|
1033
|
+
switch (dataType.type) {
|
|
1034
|
+
case "decimal":
|
|
1035
|
+
return new arrow.Decimal(dataType.scale, dataType.precision, dataType.bitWidth);
|
|
1036
|
+
case "map": {
|
|
1037
|
+
const children = dataType.children.map(
|
|
1038
|
+
(arrowField) => deserializeArrowField(arrowField, options)
|
|
1039
|
+
);
|
|
1040
|
+
return new arrow.Map_(children[0], dataType.keysSorted);
|
|
1041
|
+
}
|
|
1042
|
+
case "list": {
|
|
1043
|
+
const field = deserializeArrowField(dataType.children[0], options);
|
|
1044
|
+
return new arrow.List(field);
|
|
1045
|
+
}
|
|
1046
|
+
case "fixed-size-list": {
|
|
1047
|
+
const child = deserializeArrowField(dataType.children[0], options);
|
|
1048
|
+
return new arrow.FixedSizeList(dataType.listSize, child);
|
|
1049
|
+
}
|
|
1050
|
+
case "fixed-size-binary":
|
|
1051
|
+
return new arrow.FixedSizeBinary(dataType.byteWidth);
|
|
1052
|
+
case "struct": {
|
|
1053
|
+
const children = dataType.children.map(
|
|
1054
|
+
(arrowField) => deserializeArrowField(arrowField, options)
|
|
1055
|
+
);
|
|
1056
|
+
return new arrow.Struct(children);
|
|
1057
|
+
}
|
|
1058
|
+
case "dictionary": {
|
|
1059
|
+
return new arrow.Dictionary(
|
|
1060
|
+
deserializeArrowType(dataType.dictionary, options),
|
|
1061
|
+
deserializeArrowDictionaryKeyType(dataType.indices),
|
|
1062
|
+
dataType.id,
|
|
1063
|
+
dataType.isOrdered
|
|
1064
|
+
);
|
|
1065
|
+
}
|
|
1066
|
+
default:
|
|
1067
|
+
throw new Error("array type not supported");
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
switch (dataType) {
|
|
1071
|
+
case "null":
|
|
1072
|
+
return new arrow.Null();
|
|
1073
|
+
case "binary":
|
|
1074
|
+
return makeVariableWidthArrowType("BinaryView", new arrow.Binary(), options);
|
|
1075
|
+
case "binary-view":
|
|
1076
|
+
return makeRequiredArrowViewType("BinaryView");
|
|
1077
|
+
case "bool":
|
|
1078
|
+
return new arrow.Bool();
|
|
1079
|
+
case "int8":
|
|
1080
|
+
return new arrow.Int8();
|
|
1081
|
+
case "int16":
|
|
1082
|
+
return new arrow.Int16();
|
|
1083
|
+
case "int32":
|
|
1084
|
+
return new arrow.Int32();
|
|
1085
|
+
case "int64":
|
|
1086
|
+
return new arrow.Int64();
|
|
1087
|
+
case "uint8":
|
|
1088
|
+
return new arrow.Uint8();
|
|
1089
|
+
case "uint16":
|
|
1090
|
+
return new arrow.Uint16();
|
|
1091
|
+
case "uint32":
|
|
1092
|
+
return new arrow.Uint32();
|
|
1093
|
+
case "uint64":
|
|
1094
|
+
return new arrow.Uint64();
|
|
1095
|
+
case "float16":
|
|
1096
|
+
return new arrow.Float16();
|
|
1097
|
+
case "float32":
|
|
1098
|
+
return new arrow.Float32();
|
|
1099
|
+
case "float64":
|
|
1100
|
+
return new arrow.Float64();
|
|
1101
|
+
case "utf8":
|
|
1102
|
+
return makeVariableWidthArrowType("Utf8View", new arrow.Utf8(), options);
|
|
1103
|
+
case "utf8-view":
|
|
1104
|
+
return makeRequiredArrowViewType("Utf8View");
|
|
1105
|
+
case "date-day":
|
|
1106
|
+
return new arrow.DateDay();
|
|
1107
|
+
case "date-millisecond":
|
|
1108
|
+
return new arrow.DateMillisecond();
|
|
1109
|
+
case "time-second":
|
|
1110
|
+
return new arrow.TimeSecond();
|
|
1111
|
+
case "time-millisecond":
|
|
1112
|
+
return new arrow.TimeMillisecond();
|
|
1113
|
+
case "time-microsecond":
|
|
1114
|
+
return new arrow.TimeMicrosecond();
|
|
1115
|
+
case "time-nanosecond":
|
|
1116
|
+
return new arrow.TimeNanosecond();
|
|
1117
|
+
case "timestamp-second":
|
|
1118
|
+
return new arrow.TimestampSecond();
|
|
1119
|
+
case "timestamp-millisecond":
|
|
1120
|
+
return new arrow.TimestampMillisecond();
|
|
1121
|
+
case "timestamp-microsecond":
|
|
1122
|
+
return new arrow.TimestampMicrosecond();
|
|
1123
|
+
case "timestamp-nanosecond":
|
|
1124
|
+
return new arrow.TimestampNanosecond();
|
|
1125
|
+
case "interval-daytime":
|
|
1126
|
+
return new arrow.IntervalDayTime();
|
|
1127
|
+
case "interval-yearmonth":
|
|
1128
|
+
return new arrow.IntervalYearMonth();
|
|
1129
|
+
default:
|
|
1130
|
+
throw new Error("array type not supported");
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
function getArrowViewConstructors() {
|
|
1134
|
+
return arrow;
|
|
1135
|
+
}
|
|
1136
|
+
function isArrowViewType(arrowType, viewTypeName) {
|
|
1137
|
+
const constructor = getArrowViewConstructors()[viewTypeName];
|
|
1138
|
+
if (constructor && arrowType instanceof constructor) {
|
|
1139
|
+
return true;
|
|
1140
|
+
}
|
|
1141
|
+
const guards = arrow.DataType;
|
|
1142
|
+
const guard = viewTypeName === "BinaryView" ? guards.isBinaryView : guards.isUtf8View;
|
|
1143
|
+
return Boolean(guard == null ? void 0 : guard(arrowType));
|
|
1144
|
+
}
|
|
1145
|
+
function makeVariableWidthArrowType(viewTypeName, fallbackType, options) {
|
|
1146
|
+
const viewTypeMode = (options == null ? void 0 : options.viewTypes) || "never";
|
|
1147
|
+
if (viewTypeMode === "never") {
|
|
1148
|
+
return fallbackType;
|
|
1149
|
+
}
|
|
1150
|
+
const constructor = getArrowViewConstructors()[viewTypeName];
|
|
1151
|
+
if (constructor) {
|
|
1152
|
+
return new constructor();
|
|
1153
|
+
}
|
|
1154
|
+
if (viewTypeMode === "require") {
|
|
1155
|
+
throw new Error(
|
|
1156
|
+
`${viewTypeName} requires apache-arrow 21.2.0 or later; the installed runtime does not support it`
|
|
1157
|
+
);
|
|
1158
|
+
}
|
|
1159
|
+
return fallbackType;
|
|
1160
|
+
}
|
|
1161
|
+
function makeRequiredArrowViewType(viewTypeName) {
|
|
1162
|
+
const constructor = getArrowViewConstructors()[viewTypeName];
|
|
1163
|
+
if (!constructor) {
|
|
1164
|
+
throw new Error(
|
|
1165
|
+
`${viewTypeName} requires apache-arrow 21.2.0 or later; the installed runtime does not support it`
|
|
1166
|
+
);
|
|
1167
|
+
}
|
|
1168
|
+
return new constructor();
|
|
1169
|
+
}
|
|
1170
|
+
function serializeArrowDictionaryKeyType(arrowType) {
|
|
1171
|
+
if (arrowType instanceof arrow.Int) {
|
|
1172
|
+
const prefix = arrowType.isSigned ? "int" : "uint";
|
|
1173
|
+
const keyType = `${prefix}${arrowType.bitWidth}`;
|
|
1174
|
+
if (isSchemaDictionaryKeyType(keyType)) {
|
|
1175
|
+
return keyType;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
switch (arrowType.constructor) {
|
|
1179
|
+
case arrow.Int8:
|
|
1180
|
+
return "int8";
|
|
1181
|
+
case arrow.Int16:
|
|
1182
|
+
return "int16";
|
|
1183
|
+
case arrow.Int32:
|
|
1184
|
+
return "int32";
|
|
1185
|
+
case arrow.Uint8:
|
|
1186
|
+
return "uint8";
|
|
1187
|
+
case arrow.Uint16:
|
|
1188
|
+
return "uint16";
|
|
1189
|
+
case arrow.Uint32:
|
|
1190
|
+
return "uint32";
|
|
1191
|
+
default:
|
|
1192
|
+
throw new Error(`arrow dictionary index type not supported: ${arrowType.constructor.name}`);
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
function isSchemaDictionaryKeyType(keyType) {
|
|
1196
|
+
return ["int8", "int16", "int32", "uint8", "uint16", "uint32"].includes(keyType);
|
|
1197
|
+
}
|
|
1198
|
+
function deserializeArrowDictionaryKeyType(keyType) {
|
|
1199
|
+
switch (keyType) {
|
|
1200
|
+
case "int8":
|
|
1201
|
+
return new arrow.Int8();
|
|
1202
|
+
case "int16":
|
|
1203
|
+
return new arrow.Int16();
|
|
1204
|
+
case "int32":
|
|
1205
|
+
return new arrow.Int32();
|
|
1206
|
+
case "uint8":
|
|
1207
|
+
return new arrow.Uint8();
|
|
1208
|
+
case "uint16":
|
|
1209
|
+
return new arrow.Uint16();
|
|
1210
|
+
case "uint32":
|
|
1211
|
+
return new arrow.Uint32();
|
|
1212
|
+
default:
|
|
1213
|
+
throw new Error(`schema dictionary index type not supported: ${keyType}`);
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
// ../schema-utils/src/lib/schema/deduce-table-schema.ts
|
|
1218
|
+
function deduceTableSchema(table) {
|
|
1219
|
+
switch (table.shape) {
|
|
1220
|
+
case "array-row-table":
|
|
1221
|
+
case "object-row-table":
|
|
1222
|
+
return deduceSchemaFromRows(table.data);
|
|
1223
|
+
case "geojson-table":
|
|
1224
|
+
return deduceSchemaFromGeoJSON(table.features);
|
|
1225
|
+
case "columnar-table":
|
|
1226
|
+
return deduceSchemaFromColumns(table.data);
|
|
1227
|
+
case "arrow-table":
|
|
1228
|
+
default:
|
|
1229
|
+
throw new Error("Deduce schema");
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
function deduceSchemaFromColumns(columnarTable) {
|
|
1233
|
+
const fields = [];
|
|
1234
|
+
for (const [columnName, column] of Object.entries(columnarTable)) {
|
|
1235
|
+
const field = deduceFieldFromColumn(column, columnName);
|
|
1236
|
+
fields.push(field);
|
|
1237
|
+
}
|
|
1238
|
+
return { fields, metadata: {} };
|
|
1239
|
+
}
|
|
1240
|
+
function deduceSchemaFromRows(rowTable) {
|
|
1241
|
+
if (!rowTable.length) {
|
|
1242
|
+
throw new Error("deduce from empty table");
|
|
1243
|
+
}
|
|
1244
|
+
const fields = [];
|
|
1245
|
+
const row0 = rowTable[0];
|
|
1246
|
+
for (const [columnName, value] of Object.entries(row0)) {
|
|
1247
|
+
fields.push(deduceFieldFromValue(value, columnName));
|
|
1248
|
+
}
|
|
1249
|
+
return { fields, metadata: {} };
|
|
1250
|
+
}
|
|
1251
|
+
function deduceSchemaFromGeoJSON(features) {
|
|
1252
|
+
if (!features.length) {
|
|
1253
|
+
throw new Error("deduce from empty table");
|
|
1254
|
+
}
|
|
1255
|
+
const fields = [];
|
|
1256
|
+
const row0 = features[0].properties || {};
|
|
1257
|
+
for (const [columnName, value] of Object.entries(row0)) {
|
|
1258
|
+
fields.push(deduceFieldFromValue(value, columnName));
|
|
1259
|
+
}
|
|
1260
|
+
return { fields, metadata: {} };
|
|
1261
|
+
}
|
|
1262
|
+
function deduceFieldFromColumn(column, name) {
|
|
1263
|
+
if (ArrayBuffer.isView(column)) {
|
|
1264
|
+
const type = getDataTypeFromArray(column);
|
|
1265
|
+
return {
|
|
1266
|
+
name,
|
|
1267
|
+
type: type.type || "null",
|
|
1268
|
+
nullable: type.nullable
|
|
1269
|
+
// metadata: {}
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
if (Array.isArray(column) && column.length > 0) {
|
|
1273
|
+
const value = column[0];
|
|
1274
|
+
const type = getDataTypeFromValue(value);
|
|
1275
|
+
return {
|
|
1276
|
+
name,
|
|
1277
|
+
type,
|
|
1278
|
+
nullable: true
|
|
1279
|
+
// metadata: {},
|
|
1280
|
+
};
|
|
1281
|
+
}
|
|
1282
|
+
throw new Error("empty table");
|
|
1283
|
+
}
|
|
1284
|
+
function deduceFieldFromValue(value, name) {
|
|
1285
|
+
const type = getDataTypeFromValue(value);
|
|
1286
|
+
return {
|
|
1287
|
+
name,
|
|
1288
|
+
type,
|
|
1289
|
+
nullable: true
|
|
1290
|
+
// metadata: {}
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
// ../schema-utils/src/lib/table/tables/make-table.ts
|
|
1295
|
+
var arrow2 = __toESM(require("apache-arrow"), 1);
|
|
1296
|
+
function makeTableFromData(data) {
|
|
1297
|
+
let table;
|
|
1298
|
+
switch (getTableShapeFromData(data)) {
|
|
1299
|
+
case "array-row-table":
|
|
1300
|
+
table = { shape: "array-row-table", data };
|
|
1301
|
+
break;
|
|
1302
|
+
case "object-row-table":
|
|
1303
|
+
table = { shape: "object-row-table", data };
|
|
1304
|
+
break;
|
|
1305
|
+
case "columnar-table":
|
|
1306
|
+
table = { shape: "columnar-table", data };
|
|
1307
|
+
break;
|
|
1308
|
+
case "arrow-table":
|
|
1309
|
+
table = { shape: "arrow-table", data };
|
|
1310
|
+
break;
|
|
1311
|
+
default:
|
|
1312
|
+
throw new Error("table");
|
|
1313
|
+
}
|
|
1314
|
+
const schema = deduceTableSchema(table);
|
|
1315
|
+
return { ...table, schema };
|
|
1316
|
+
}
|
|
1317
|
+
function getTableShapeFromData(data) {
|
|
1318
|
+
if (data instanceof arrow2.Table) {
|
|
1319
|
+
return "arrow-table";
|
|
1320
|
+
}
|
|
1321
|
+
if (Array.isArray(data)) {
|
|
1322
|
+
if (data.length === 0) {
|
|
1323
|
+
throw new Error("cannot deduce type of empty table");
|
|
1324
|
+
}
|
|
1325
|
+
const firstRow = data[0];
|
|
1326
|
+
if (Array.isArray(firstRow)) {
|
|
1327
|
+
return "array-row-table";
|
|
1328
|
+
}
|
|
1329
|
+
if (firstRow && typeof firstRow === "object") {
|
|
1330
|
+
return "object-row-table";
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
if (data && typeof data === "object") {
|
|
1334
|
+
return "columnar-table";
|
|
1335
|
+
}
|
|
1336
|
+
throw new Error("invalid table");
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
// ../schema-utils/src/lib/table/tables/table-accessors.ts
|
|
1340
|
+
function getTableLength(table) {
|
|
1341
|
+
switch (table.shape) {
|
|
1342
|
+
case "array-row-table":
|
|
1343
|
+
case "object-row-table":
|
|
1344
|
+
return table.data.length;
|
|
1345
|
+
case "geojson-table":
|
|
1346
|
+
return table.features.length;
|
|
1347
|
+
case "arrow-table":
|
|
1348
|
+
const arrowTable = table.data;
|
|
1349
|
+
return arrowTable.numRows;
|
|
1350
|
+
case "columnar-table":
|
|
1351
|
+
for (const column of Object.values(table.data)) {
|
|
1352
|
+
return column.length || 0;
|
|
1353
|
+
}
|
|
1354
|
+
return 0;
|
|
1355
|
+
default:
|
|
1356
|
+
throw new Error("table");
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
function getTableNumCols(table) {
|
|
1360
|
+
if (table.schema) {
|
|
1361
|
+
return table.schema.fields.length;
|
|
1362
|
+
}
|
|
1363
|
+
if (getTableLength(table) === 0) {
|
|
1364
|
+
throw new Error("empty table");
|
|
1365
|
+
}
|
|
1366
|
+
switch (table.shape) {
|
|
1367
|
+
case "array-row-table":
|
|
1368
|
+
return table.data[0].length;
|
|
1369
|
+
case "object-row-table":
|
|
1370
|
+
return Object.keys(table.data[0]).length;
|
|
1371
|
+
case "geojson-table":
|
|
1372
|
+
return Object.keys(table.features[0]).length;
|
|
1373
|
+
case "columnar-table":
|
|
1374
|
+
return Object.keys(table.data).length;
|
|
1375
|
+
case "arrow-table":
|
|
1376
|
+
const arrowTable = table.data;
|
|
1377
|
+
return arrowTable.numCols;
|
|
1378
|
+
default:
|
|
1379
|
+
throw new Error("table");
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
function getTableCell(table, rowIndex, columnName) {
|
|
1383
|
+
var _a;
|
|
1384
|
+
switch (table.shape) {
|
|
1385
|
+
case "array-row-table":
|
|
1386
|
+
const columnIndex = getTableColumnIndex(table, columnName);
|
|
1387
|
+
return table.data[rowIndex][columnIndex];
|
|
1388
|
+
case "object-row-table":
|
|
1389
|
+
return table.data[rowIndex][columnName];
|
|
1390
|
+
case "geojson-table":
|
|
1391
|
+
return table.features[rowIndex][columnName];
|
|
1392
|
+
case "columnar-table":
|
|
1393
|
+
const column = table.data[columnName];
|
|
1394
|
+
return column[rowIndex];
|
|
1395
|
+
case "arrow-table":
|
|
1396
|
+
const arrowTable = table.data;
|
|
1397
|
+
const arrowColumnIndex = arrowTable.schema.fields.findIndex(
|
|
1398
|
+
(field) => field.name === columnName
|
|
1399
|
+
);
|
|
1400
|
+
return (_a = arrowTable.getChildAt(arrowColumnIndex)) == null ? void 0 : _a.get(rowIndex);
|
|
1401
|
+
default:
|
|
1402
|
+
throw new Error("todo");
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
function getTableCellAt(table, rowIndex, columnIndex) {
|
|
1406
|
+
var _a;
|
|
1407
|
+
switch (table.shape) {
|
|
1408
|
+
case "array-row-table":
|
|
1409
|
+
return table.data[rowIndex][columnIndex];
|
|
1410
|
+
case "object-row-table":
|
|
1411
|
+
const columnName1 = getTableColumnName(table, columnIndex);
|
|
1412
|
+
return table.data[rowIndex][columnName1];
|
|
1413
|
+
case "geojson-table":
|
|
1414
|
+
const columnName2 = getTableColumnName(table, columnIndex);
|
|
1415
|
+
return table.features[rowIndex][columnName2];
|
|
1416
|
+
case "columnar-table":
|
|
1417
|
+
const columnName3 = getTableColumnName(table, columnIndex);
|
|
1418
|
+
const column = table.data[columnName3];
|
|
1419
|
+
return column[rowIndex];
|
|
1420
|
+
case "arrow-table":
|
|
1421
|
+
const arrowTable = table.data;
|
|
1422
|
+
return (_a = arrowTable.getChildAt(columnIndex)) == null ? void 0 : _a.get(rowIndex);
|
|
1423
|
+
default:
|
|
1424
|
+
throw new Error("todo");
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
function getTableColumnIndex(table, columnName) {
|
|
1428
|
+
var _a;
|
|
1429
|
+
const columnIndex = (_a = table.schema) == null ? void 0 : _a.fields.findIndex((field) => field.name === columnName);
|
|
1430
|
+
if (columnIndex === void 0) {
|
|
1431
|
+
throw new Error(columnName);
|
|
1432
|
+
}
|
|
1433
|
+
return columnIndex;
|
|
1434
|
+
}
|
|
1435
|
+
function getTableColumnName(table, columnIndex) {
|
|
1436
|
+
var _a, _b;
|
|
1437
|
+
const columnName = (_b = (_a = table.schema) == null ? void 0 : _a.fields[columnIndex]) == null ? void 0 : _b.name;
|
|
1438
|
+
if (!columnName) {
|
|
1439
|
+
throw new Error(`${columnIndex}`);
|
|
1440
|
+
}
|
|
1441
|
+
return columnName;
|
|
1442
|
+
}
|
|
1443
|
+
function getTableRowAsObject(table, rowIndex, target, copy) {
|
|
1444
|
+
switch (table.shape) {
|
|
1445
|
+
case "object-row-table":
|
|
1446
|
+
return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
|
|
1447
|
+
case "array-row-table":
|
|
1448
|
+
if (table.schema) {
|
|
1449
|
+
const objectRow2 = target || {};
|
|
1450
|
+
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
1451
|
+
objectRow2[table.schema.fields[i].name] = table.data[rowIndex][i];
|
|
1452
|
+
}
|
|
1453
|
+
return objectRow2;
|
|
1454
|
+
}
|
|
1455
|
+
throw new Error("no schema");
|
|
1456
|
+
case "geojson-table":
|
|
1457
|
+
if (table.schema) {
|
|
1458
|
+
const objectRow2 = target || {};
|
|
1459
|
+
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
1460
|
+
objectRow2[table.schema.fields[i].name] = table.features[rowIndex][table.schema.fields[i].name];
|
|
1461
|
+
}
|
|
1462
|
+
return objectRow2;
|
|
1463
|
+
}
|
|
1464
|
+
throw new Error("no schema");
|
|
1465
|
+
case "columnar-table":
|
|
1466
|
+
if (table.schema) {
|
|
1467
|
+
const objectRow2 = target || {};
|
|
1468
|
+
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
1469
|
+
objectRow2[table.schema.fields[i].name] = table.data[table.schema.fields[i].name][rowIndex];
|
|
1470
|
+
}
|
|
1471
|
+
return objectRow2;
|
|
1472
|
+
} else {
|
|
1473
|
+
const objectRow2 = target || {};
|
|
1474
|
+
for (const [name, column] of Object.entries(table.data)) {
|
|
1475
|
+
objectRow2[name] = column[rowIndex];
|
|
1476
|
+
}
|
|
1477
|
+
return objectRow2;
|
|
1478
|
+
}
|
|
1479
|
+
case "arrow-table":
|
|
1480
|
+
const arrowTable = table.data;
|
|
1481
|
+
const objectRow = target || {};
|
|
1482
|
+
const row = arrowTable.get(rowIndex);
|
|
1483
|
+
const schema = arrowTable.schema;
|
|
1484
|
+
for (let i = 0; i < schema.fields.length; i++) {
|
|
1485
|
+
objectRow[schema.fields[i].name] = row == null ? void 0 : row[schema.fields[i].name];
|
|
1486
|
+
}
|
|
1487
|
+
return objectRow;
|
|
1488
|
+
default:
|
|
1489
|
+
throw new Error("shape");
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
function getTableRowAsArray(table, rowIndex, target, copy) {
|
|
1493
|
+
switch (table.shape) {
|
|
1494
|
+
case "array-row-table":
|
|
1495
|
+
return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
|
|
1496
|
+
case "object-row-table":
|
|
1497
|
+
if (table.schema) {
|
|
1498
|
+
const arrayRow2 = target || [];
|
|
1499
|
+
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
1500
|
+
arrayRow2[i] = table.data[rowIndex][table.schema.fields[i].name];
|
|
1501
|
+
}
|
|
1502
|
+
return arrayRow2;
|
|
1503
|
+
}
|
|
1504
|
+
return Object.values(table.data[rowIndex]);
|
|
1505
|
+
case "geojson-table":
|
|
1506
|
+
if (table.schema) {
|
|
1507
|
+
const arrayRow2 = target || [];
|
|
1508
|
+
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
1509
|
+
arrayRow2[i] = table.features[rowIndex][table.schema.fields[i].name];
|
|
1510
|
+
}
|
|
1511
|
+
return arrayRow2;
|
|
1512
|
+
}
|
|
1513
|
+
return Object.values(table.features[rowIndex]);
|
|
1514
|
+
case "columnar-table":
|
|
1515
|
+
if (table.schema) {
|
|
1516
|
+
const arrayRow2 = target || [];
|
|
1517
|
+
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
1518
|
+
arrayRow2[i] = table.data[table.schema.fields[i].name][rowIndex];
|
|
1519
|
+
}
|
|
1520
|
+
return arrayRow2;
|
|
1521
|
+
} else {
|
|
1522
|
+
const arrayRow2 = target || [];
|
|
1523
|
+
let i = 0;
|
|
1524
|
+
for (const column of Object.values(table.data)) {
|
|
1525
|
+
arrayRow2[i] = column[rowIndex];
|
|
1526
|
+
i++;
|
|
1527
|
+
}
|
|
1528
|
+
return arrayRow2;
|
|
1529
|
+
}
|
|
1530
|
+
case "arrow-table":
|
|
1531
|
+
const arrowTable = table.data;
|
|
1532
|
+
const arrayRow = target || [];
|
|
1533
|
+
const row = arrowTable.get(rowIndex);
|
|
1534
|
+
const schema = arrowTable.schema;
|
|
1535
|
+
for (let i = 0; i < schema.fields.length; i++) {
|
|
1536
|
+
arrayRow[i] = row == null ? void 0 : row[schema.fields[i].name];
|
|
1537
|
+
}
|
|
1538
|
+
return arrayRow;
|
|
1539
|
+
default:
|
|
1540
|
+
throw new Error("shape");
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
// ../schema-utils/src/lib/table/tables/table-column.ts
|
|
1545
|
+
function makeColumnFromField(field, length) {
|
|
1546
|
+
const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
|
|
1547
|
+
return new ArrayType(length);
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
// ../schema-utils/src/lib/table/tables/convert-arrow-table.ts
|
|
1551
|
+
var arrow4 = __toESM(require("apache-arrow"), 1);
|
|
1552
|
+
|
|
1553
|
+
// ../schema-utils/src/lib/table/batches/make-arrow-batch-iterator.ts
|
|
1554
|
+
var arrow3 = __toESM(require("apache-arrow"), 1);
|
|
1555
|
+
function* makeArrowRecordBatchIterator(table, options) {
|
|
1556
|
+
const arrowSchema = convertSchemaToArrow(table.schema);
|
|
1557
|
+
const length = getTableLength(table);
|
|
1558
|
+
const numColumns = getTableNumCols(table);
|
|
1559
|
+
const batchSize = (options == null ? void 0 : options.batchSize) || length;
|
|
1560
|
+
const builders = arrowSchema == null ? void 0 : arrowSchema.fields.map(
|
|
1561
|
+
(arrowField) => arrow3.makeBuilder({
|
|
1562
|
+
type: arrowField.type,
|
|
1563
|
+
nullValues: [null, void 0]
|
|
1564
|
+
})
|
|
1565
|
+
);
|
|
1566
|
+
const structField = new arrow3.Struct(arrowSchema.fields);
|
|
1567
|
+
let batchLength = 0;
|
|
1568
|
+
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
1569
|
+
for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
|
|
1570
|
+
const value = getTableCellAt(table, rowIndex, columnIndex);
|
|
1571
|
+
const builder = builders[columnIndex];
|
|
1572
|
+
builder.append(value);
|
|
1573
|
+
}
|
|
1574
|
+
batchLength++;
|
|
1575
|
+
if (batchLength >= batchSize) {
|
|
1576
|
+
const datas = builders.map((builder) => builder.flush());
|
|
1577
|
+
const structData = new arrow3.Data(structField, 0, batchLength, 0, void 0, datas);
|
|
1578
|
+
yield new arrow3.RecordBatch(arrowSchema, structData);
|
|
1579
|
+
batchLength = 0;
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
if (batchLength > 0) {
|
|
1583
|
+
const datas = builders.map((builder) => builder.flush());
|
|
1584
|
+
const structData = new arrow3.Data(structField, 0, batchLength, 0, void 0, datas);
|
|
1585
|
+
yield new arrow3.RecordBatch(arrowSchema, structData);
|
|
1586
|
+
batchLength = 0;
|
|
1587
|
+
}
|
|
1588
|
+
builders.map((builder) => builder.finish());
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
// ../schema-utils/src/lib/table/tables/convert-arrow-table.ts
|
|
1592
|
+
function convertTableToArrow(table, options) {
|
|
1593
|
+
switch (table.shape) {
|
|
1594
|
+
case "arrow-table":
|
|
1595
|
+
return table.data;
|
|
1596
|
+
case "columnar-table":
|
|
1597
|
+
// TODO - optimized implementation is possible
|
|
1598
|
+
// return convertColumnarTableToArrow(table, options);
|
|
1599
|
+
// fall through
|
|
1600
|
+
default:
|
|
1601
|
+
const arrowBatchIterator = makeArrowRecordBatchIterator(table, options);
|
|
1602
|
+
return new arrow4.Table(arrowBatchIterator);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
// ../schema-utils/src/lib/table/tables/convert-table.ts
|
|
1607
|
+
function convertTable(table, shape) {
|
|
1608
|
+
switch (shape) {
|
|
1609
|
+
case "object-row-table":
|
|
1610
|
+
return convertToObjectRowTable(table);
|
|
1611
|
+
case "array-row-table":
|
|
1612
|
+
return convertToArrayRowTable(table);
|
|
1613
|
+
case "columnar-table":
|
|
1614
|
+
return convertToColumnarTable(table);
|
|
1615
|
+
case "arrow-table":
|
|
1616
|
+
return convertToArrowTable(table);
|
|
1617
|
+
default:
|
|
1618
|
+
throw new Error(shape);
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
function convertToColumnarTable(table) {
|
|
1622
|
+
const schema = table.schema || deduceTableSchema(table);
|
|
1623
|
+
const fields = schema.fields;
|
|
1624
|
+
if (table.shape === "columnar-table") {
|
|
1625
|
+
return { ...table, schema };
|
|
1626
|
+
}
|
|
1627
|
+
const length = getTableLength(table);
|
|
1628
|
+
const columns = {};
|
|
1629
|
+
for (const field of fields) {
|
|
1630
|
+
const column = makeColumnFromField(field, length);
|
|
1631
|
+
columns[field.name] = column;
|
|
1632
|
+
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
1633
|
+
column[rowIndex] = getTableCell(table, rowIndex, field.name);
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
return {
|
|
1637
|
+
shape: "columnar-table",
|
|
1638
|
+
schema,
|
|
1639
|
+
data: columns
|
|
1640
|
+
};
|
|
1641
|
+
}
|
|
1642
|
+
function convertToArrayRowTable(table) {
|
|
1643
|
+
if (table.shape === "array-row-table") {
|
|
1644
|
+
return table;
|
|
1645
|
+
}
|
|
1646
|
+
const length = getTableLength(table);
|
|
1647
|
+
const data = new Array(length);
|
|
1648
|
+
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
1649
|
+
data[rowIndex] = getTableRowAsArray(table, rowIndex);
|
|
1650
|
+
}
|
|
1651
|
+
return {
|
|
1652
|
+
shape: "array-row-table",
|
|
1653
|
+
schema: table.schema,
|
|
1654
|
+
data
|
|
1655
|
+
};
|
|
1656
|
+
}
|
|
1657
|
+
function convertToObjectRowTable(table) {
|
|
1658
|
+
if (table.shape === "object-row-table") {
|
|
1659
|
+
return table;
|
|
1660
|
+
}
|
|
1661
|
+
const length = getTableLength(table);
|
|
1662
|
+
const data = new Array(length);
|
|
1663
|
+
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
1664
|
+
data[rowIndex] = getTableRowAsObject(table, rowIndex);
|
|
1665
|
+
}
|
|
1666
|
+
return {
|
|
1667
|
+
shape: "object-row-table",
|
|
1668
|
+
schema: table.schema,
|
|
1669
|
+
data
|
|
1670
|
+
};
|
|
1671
|
+
}
|
|
1672
|
+
function convertToArrowTable(table) {
|
|
1673
|
+
const schema = table.schema || deduceTableSchema(table);
|
|
1674
|
+
const arrowTable = convertTableToArrow({ ...table, schema });
|
|
1675
|
+
return {
|
|
1676
|
+
shape: "arrow-table",
|
|
1677
|
+
schema,
|
|
1678
|
+
data: arrowTable
|
|
1679
|
+
};
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
// ../schema-utils/src/lib/table/batches/convert-batches.ts
|
|
1683
|
+
function convertBatch(batch, shape) {
|
|
1684
|
+
switch (shape) {
|
|
1685
|
+
case "object-row-table":
|
|
1686
|
+
return { ...batch, ...convertTable(batch, "object-row-table") };
|
|
1687
|
+
case "array-row-table":
|
|
1688
|
+
return { ...batch, ...convertTable(batch, "array-row-table") };
|
|
1689
|
+
case "columnar-table":
|
|
1690
|
+
return { ...batch, ...convertTable(batch, "columnar-table") };
|
|
1691
|
+
case "arrow-table":
|
|
1692
|
+
return { ...batch, ...convertTable(batch, "arrow-table") };
|
|
1693
|
+
default:
|
|
1694
|
+
throw new Error(shape);
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
// ../sql/src/sql-utils.ts
|
|
1699
|
+
function convertRowsToArrowTable(rows) {
|
|
1700
|
+
const table = rows.length > 0 ? makeTableFromData(rows) : { shape: "object-row-table", schema: { fields: [], metadata: {} }, data: rows };
|
|
1701
|
+
const schema = deducePrimitiveAwareSchema(table);
|
|
1702
|
+
if (table.data.length === 0 || schema.fields.length === 0) {
|
|
1703
|
+
return {
|
|
1704
|
+
shape: "arrow-table",
|
|
1705
|
+
schema,
|
|
1706
|
+
data: new arrow5.Table(new arrow5.Schema([]))
|
|
1707
|
+
};
|
|
1708
|
+
}
|
|
1709
|
+
return {
|
|
1710
|
+
shape: "arrow-table",
|
|
1711
|
+
schema,
|
|
1712
|
+
data: arrow5.tableFromJSON(table.data)
|
|
1713
|
+
};
|
|
1714
|
+
}
|
|
1715
|
+
function deducePrimitiveAwareSchema(table) {
|
|
1716
|
+
return {
|
|
1717
|
+
...table.schema,
|
|
1718
|
+
fields: table.schema.fields.map((field) => {
|
|
1719
|
+
if (field.type !== "null") {
|
|
1720
|
+
return field;
|
|
1721
|
+
}
|
|
1722
|
+
const columnValues = table.data.map((row) => row[field.name]);
|
|
1723
|
+
const inferredType = getDataTypeFromArray(columnValues);
|
|
1724
|
+
return { ...field, type: inferredType.type, nullable: inferredType.nullable };
|
|
1725
|
+
})
|
|
1726
|
+
};
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
// ../sql/src/table-query-capabilities.ts
|
|
1730
|
+
var ARROW_TABLE_QUERY_CAPABILITIES = Object.freeze({
|
|
1731
|
+
projection: "residual",
|
|
1732
|
+
predicate: "residual",
|
|
1733
|
+
limit: "residual",
|
|
1734
|
+
streaming: false,
|
|
1735
|
+
cancellation: true,
|
|
1736
|
+
expressions: "residual",
|
|
1737
|
+
orderBy: "residual",
|
|
1738
|
+
aggregates: "residual"
|
|
1739
|
+
});
|
|
1740
|
+
var SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES = Object.freeze({
|
|
1741
|
+
projection: "pushdown",
|
|
1742
|
+
predicate: "pushdown",
|
|
1743
|
+
limit: "pushdown",
|
|
1744
|
+
streaming: false,
|
|
1745
|
+
cancellation: false,
|
|
1746
|
+
expressions: "pushdown",
|
|
1747
|
+
orderBy: "pushdown",
|
|
1748
|
+
aggregates: "pushdown"
|
|
1749
|
+
});
|
|
1750
|
+
|
|
1751
|
+
// ../sql/src/query-arrow-table.ts
|
|
1752
|
+
var arrow6 = __toESM(require("apache-arrow"), 1);
|
|
1753
|
+
|
|
1754
|
+
// ../sql/src/table-query.ts
|
|
1755
|
+
function planTableQuery2(sourceColumnNames, options = {}) {
|
|
1756
|
+
if (options.predicate) {
|
|
1757
|
+
validateSQLPredicate(options.predicate);
|
|
1758
|
+
}
|
|
1759
|
+
return planTableQuery(sourceColumnNames, options);
|
|
1760
|
+
}
|
|
1761
|
+
function getSQLPredicateColumnNames(predicate) {
|
|
1762
|
+
return getColumnarPredicateColumns(predicate);
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
// ../sql/src/query-arrow-table.ts
|
|
1766
|
+
function explainArrowTableQuery(sourceTable, options = {}) {
|
|
1767
|
+
if (options.predicate) {
|
|
1768
|
+
validateSQLPredicate(options.predicate);
|
|
1769
|
+
}
|
|
1770
|
+
return explainTableQuery(
|
|
1771
|
+
sourceTable.data.schema.fields.map((field) => field.name),
|
|
1772
|
+
options,
|
|
1773
|
+
ARROW_TABLE_QUERY_CAPABILITIES
|
|
1774
|
+
);
|
|
1775
|
+
}
|
|
1776
|
+
function queryArrowTable(sourceTable, options = {}) {
|
|
1777
|
+
var _a, _b;
|
|
1778
|
+
throwIfAborted(options.signal);
|
|
1779
|
+
if (options.join) {
|
|
1780
|
+
validateArrowJoinQuery(sourceTable, options);
|
|
1781
|
+
}
|
|
1782
|
+
if (options.join && hasRelationalOperatorsExceptJoin(options)) {
|
|
1783
|
+
throw new Error("Arrow joins cannot yet be combined with other relational operators.");
|
|
1784
|
+
}
|
|
1785
|
+
if (options.join && !hasRelationalOperatorsExceptJoin(options)) {
|
|
1786
|
+
return queryArrowJoin(sourceTable, options);
|
|
1787
|
+
}
|
|
1788
|
+
if (hasRelationalOperators(options)) {
|
|
1789
|
+
return queryArrowTableRelational(sourceTable, options);
|
|
1790
|
+
}
|
|
1791
|
+
const sourceData = sourceTable.data;
|
|
1792
|
+
const plan = planTableQuery2(
|
|
1793
|
+
sourceData.schema.fields.map((field) => field.name),
|
|
1794
|
+
options
|
|
1795
|
+
);
|
|
1796
|
+
const scannedData = sourceData.select([...getScanStep(plan).columns]);
|
|
1797
|
+
const projectedData = scannedData.select([...getProjectStep(plan).columns]);
|
|
1798
|
+
const predicate = (_a = getFilterStep(plan)) == null ? void 0 : _a.predicate;
|
|
1799
|
+
const limit = ((_b = getLimitStep(plan)) == null ? void 0 : _b.limit) ?? Number.POSITIVE_INFINITY;
|
|
1800
|
+
if (!predicate) {
|
|
1801
|
+
return wrapArrowTable(projectedData.slice(0, limit));
|
|
1802
|
+
}
|
|
1803
|
+
const predicateColumns = getPredicateColumns(scannedData, getSQLPredicateColumnNames(predicate));
|
|
1804
|
+
const matchingRowIndices = [];
|
|
1805
|
+
for (let rowIndex = 0; rowIndex < scannedData.numRows && matchingRowIndices.length < limit; rowIndex++) {
|
|
1806
|
+
throwIfAborted(options.signal);
|
|
1807
|
+
if (evaluatePredicate(predicate, predicateColumns, rowIndex) === true) {
|
|
1808
|
+
matchingRowIndices.push(rowIndex);
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1811
|
+
if (matchingRowIndices.length === 0) {
|
|
1812
|
+
return wrapArrowTable(projectedData.slice(0, 0));
|
|
1813
|
+
}
|
|
1814
|
+
const outputColumns = {};
|
|
1815
|
+
for (const field of projectedData.schema.fields) {
|
|
1816
|
+
const sourceColumn = projectedData.getChild(field.name);
|
|
1817
|
+
if (!sourceColumn) {
|
|
1818
|
+
throw new Error(`Arrow query could not read column "${field.name}".`);
|
|
1819
|
+
}
|
|
1820
|
+
const values = matchingRowIndices.map((rowIndex) => sourceColumn.get(rowIndex));
|
|
1821
|
+
outputColumns[field.name] = arrow6.vectorFromArray(values, sourceColumn.type);
|
|
1822
|
+
}
|
|
1823
|
+
return wrapArrowTable(new arrow6.Table(projectedData.schema, outputColumns));
|
|
1824
|
+
}
|
|
1825
|
+
function queryArrowTableRelational(sourceTable, options) {
|
|
1826
|
+
var _a, _b, _c, _d, _e, _f;
|
|
1827
|
+
const sourceData = sourceTable.data;
|
|
1828
|
+
const sourceColumnNames = sourceData.schema.fields.map((field) => field.name);
|
|
1829
|
+
const relationalPlan = planRelationalQuery(sourceColumnNames, options);
|
|
1830
|
+
const scanStep = relationalPlan.tablePlan.find((step) => step.kind === "scan");
|
|
1831
|
+
if (!scanStep || scanStep.kind !== "scan") {
|
|
1832
|
+
throw new Error("Arrow relational query plan is missing a scan step.");
|
|
1833
|
+
}
|
|
1834
|
+
const scannedTable = wrapArrowTable(sourceData.select([...scanStep.columns]));
|
|
1835
|
+
const filteredTable = queryArrowTable(scannedTable, {
|
|
1836
|
+
predicate: options.predicate,
|
|
1837
|
+
columns: scanStep.columns,
|
|
1838
|
+
signal: options.signal
|
|
1839
|
+
});
|
|
1840
|
+
let rows = arrowTableToRows(filteredTable.data);
|
|
1841
|
+
for (const child of options.union || []) {
|
|
1842
|
+
const childTable = (_a = options.tables) == null ? void 0 : _a[child.source];
|
|
1843
|
+
if (!childTable) throw new Error(`Arrow union source not found: ${child.source}`);
|
|
1844
|
+
const childResult = queryArrowTable(childTable, {
|
|
1845
|
+
...child.query || {},
|
|
1846
|
+
columns: getUnionChildColumns(sourceColumnNames, options),
|
|
1847
|
+
signal: options.signal
|
|
1848
|
+
});
|
|
1849
|
+
rows = rows.concat(arrowTableToRows(childResult.data));
|
|
1850
|
+
}
|
|
1851
|
+
for (const expression of options.expressions || []) {
|
|
1852
|
+
for (const row of rows) {
|
|
1853
|
+
row[expression.name] = evaluateRelationalExpression(expression, row);
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
if (((_b = options.groupBy) == null ? void 0 : _b.length) || ((_c = options.aggregates) == null ? void 0 : _c.length)) {
|
|
1857
|
+
rows = aggregateRows(rows, options.groupBy || [], options.aggregates || []);
|
|
1858
|
+
}
|
|
1859
|
+
if ((_d = options.orderBy) == null ? void 0 : _d.length) {
|
|
1860
|
+
const orderBy = options.orderBy;
|
|
1861
|
+
rows = rows.map((row, index) => ({ row, index })).sort((left, right) => compareRows(left.row, right.row, orderBy) || left.index - right.index).map((entry) => entry.row);
|
|
1862
|
+
}
|
|
1863
|
+
const outputColumns = options.columns || (((_e = options.groupBy) == null ? void 0 : _e.length) || ((_f = options.aggregates) == null ? void 0 : _f.length) ? [...options.groupBy || [], ...(options.aggregates || []).map((aggregate) => aggregate.name)] : sourceColumnNames);
|
|
1864
|
+
const outputRows = rows.map((row) => {
|
|
1865
|
+
const output = {};
|
|
1866
|
+
for (const columnName of outputColumns) {
|
|
1867
|
+
if (!(columnName in row)) {
|
|
1868
|
+
throw new Error(`Arrow relational query column not found: ${columnName}`);
|
|
1869
|
+
}
|
|
1870
|
+
output[columnName] = row[columnName];
|
|
1871
|
+
}
|
|
1872
|
+
return output;
|
|
1873
|
+
});
|
|
1874
|
+
const limit = options.limit === void 0 ? outputRows.length : options.limit;
|
|
1875
|
+
const limitedRows = outputRows.slice(0, limit);
|
|
1876
|
+
return limitedRows.length ? convertRowsToArrowTable(limitedRows) : createEmptyRelationalResult(sourceData, outputColumns, options);
|
|
1877
|
+
}
|
|
1878
|
+
function getUnionChildColumns(sourceColumnNames, options) {
|
|
1879
|
+
var _a, _b;
|
|
1880
|
+
const expressionNames = new Set((options.expressions || []).map((expression) => expression.name));
|
|
1881
|
+
const aggregateNames = new Set((options.aggregates || []).map((aggregate) => aggregate.name));
|
|
1882
|
+
const outputColumns = options.columns || (((_a = options.groupBy) == null ? void 0 : _a.length) || ((_b = options.aggregates) == null ? void 0 : _b.length) ? [...options.groupBy || [], ...(options.aggregates || []).map((aggregate) => aggregate.name)] : sourceColumnNames);
|
|
1883
|
+
const requiredColumns = new Set(
|
|
1884
|
+
outputColumns.filter((column) => !expressionNames.has(column) && !aggregateNames.has(column))
|
|
1885
|
+
);
|
|
1886
|
+
for (const expression of options.expressions || []) {
|
|
1887
|
+
const definition = expression.expression;
|
|
1888
|
+
if (definition.op === "column") requiredColumns.add(definition.column);
|
|
1889
|
+
if ("left" in definition) {
|
|
1890
|
+
requiredColumns.add(definition.left);
|
|
1891
|
+
requiredColumns.add(definition.right);
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
for (const key of options.orderBy || []) {
|
|
1895
|
+
if (!expressionNames.has(key.column) && !aggregateNames.has(key.column)) {
|
|
1896
|
+
requiredColumns.add(key.column);
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
for (const column of options.groupBy || []) requiredColumns.add(column);
|
|
1900
|
+
for (const aggregate of options.aggregates || []) {
|
|
1901
|
+
if (aggregate.column) requiredColumns.add(aggregate.column);
|
|
1902
|
+
}
|
|
1903
|
+
return sourceColumnNames.filter((column) => requiredColumns.has(column));
|
|
1904
|
+
}
|
|
1905
|
+
function hasRelationalOperators(options) {
|
|
1906
|
+
var _a, _b, _c, _d, _e;
|
|
1907
|
+
return Boolean(
|
|
1908
|
+
((_a = options.expressions) == null ? void 0 : _a.length) || ((_b = options.orderBy) == null ? void 0 : _b.length) || ((_c = options.groupBy) == null ? void 0 : _c.length) || ((_d = options.aggregates) == null ? void 0 : _d.length) || ((_e = options.union) == null ? void 0 : _e.length) || options.join
|
|
1909
|
+
);
|
|
1910
|
+
}
|
|
1911
|
+
function hasRelationalOperatorsExceptJoin(options) {
|
|
1912
|
+
var _a, _b, _c, _d, _e;
|
|
1913
|
+
return Boolean(
|
|
1914
|
+
((_a = options.expressions) == null ? void 0 : _a.length) || ((_b = options.orderBy) == null ? void 0 : _b.length) || ((_c = options.groupBy) == null ? void 0 : _c.length) || ((_d = options.aggregates) == null ? void 0 : _d.length) || ((_e = options.union) == null ? void 0 : _e.length)
|
|
1915
|
+
);
|
|
1916
|
+
}
|
|
1917
|
+
function queryArrowJoin(sourceTable, options) {
|
|
1918
|
+
var _a;
|
|
1919
|
+
const join = options.join;
|
|
1920
|
+
if (!join) throw new Error("Arrow join query is missing a join definition.");
|
|
1921
|
+
const childTable = (_a = options.tables) == null ? void 0 : _a[join.child.source];
|
|
1922
|
+
if (!childTable) throw new Error(`Arrow join source not found: ${join.child.source}`);
|
|
1923
|
+
const leftResult = queryArrowTable(sourceTable, {
|
|
1924
|
+
predicate: options.predicate,
|
|
1925
|
+
columns: void 0,
|
|
1926
|
+
signal: options.signal
|
|
1927
|
+
});
|
|
1928
|
+
const rightResult = queryArrowTable(childTable, {
|
|
1929
|
+
...join.child.query || {},
|
|
1930
|
+
signal: options.signal
|
|
1931
|
+
});
|
|
1932
|
+
const leftRows = arrowTableToRows(leftResult.data);
|
|
1933
|
+
const rightRows = arrowTableToRows(rightResult.data);
|
|
1934
|
+
const rightPrefix = `${join.child.source}.`;
|
|
1935
|
+
const defaultOutputColumns = [
|
|
1936
|
+
...sourceTable.data.schema.fields.map((field) => field.name),
|
|
1937
|
+
...rightResult.data.schema.fields.map((field) => `${rightPrefix}${field.name}`)
|
|
1938
|
+
];
|
|
1939
|
+
const rows = [];
|
|
1940
|
+
for (const leftRow of leftRows) {
|
|
1941
|
+
for (const rightRow of rightRows) {
|
|
1942
|
+
if (leftRow[join.left] !== null && leftRow[join.left] !== void 0 && rightRow[join.right] !== null && rightRow[join.right] !== void 0 && compareSortValues(leftRow[join.left], rightRow[join.right]) === 0) {
|
|
1943
|
+
rows.push({
|
|
1944
|
+
...leftRow,
|
|
1945
|
+
...Object.fromEntries(
|
|
1946
|
+
Object.entries(rightRow).map(([key, value]) => [`${rightPrefix}${key}`, value])
|
|
1947
|
+
)
|
|
1948
|
+
});
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
const outputColumns = options.columns || defaultOutputColumns;
|
|
1953
|
+
const outputRows = rows.map(
|
|
1954
|
+
(row) => Object.fromEntries(outputColumns.map((column) => [column, row[column]]))
|
|
1955
|
+
);
|
|
1956
|
+
const limitedRows = outputRows.slice(0, options.limit);
|
|
1957
|
+
return limitedRows.length ? convertRowsToArrowTable(limitedRows) : createEmptyRelationalResult(
|
|
1958
|
+
sourceTable.data,
|
|
1959
|
+
outputColumns,
|
|
1960
|
+
options,
|
|
1961
|
+
new Map(
|
|
1962
|
+
rightResult.data.schema.fields.map((field) => [
|
|
1963
|
+
`${rightPrefix}${field.name}`,
|
|
1964
|
+
new arrow6.Field(
|
|
1965
|
+
`${rightPrefix}${field.name}`,
|
|
1966
|
+
field.type,
|
|
1967
|
+
field.nullable,
|
|
1968
|
+
field.metadata
|
|
1969
|
+
)
|
|
1970
|
+
])
|
|
1971
|
+
)
|
|
1972
|
+
);
|
|
1973
|
+
}
|
|
1974
|
+
function validateArrowJoinQuery(sourceTable, options) {
|
|
1975
|
+
var _a, _b, _c;
|
|
1976
|
+
const join = options.join;
|
|
1977
|
+
if (!join) return;
|
|
1978
|
+
const sourceColumnNames = sourceTable.data.schema.fields.map((field) => field.name);
|
|
1979
|
+
const childTable = (_a = options.tables) == null ? void 0 : _a[join.child.source];
|
|
1980
|
+
if (!childTable) throw new Error(`Arrow join source not found: ${join.child.source}`);
|
|
1981
|
+
const childColumnNames = childTable.data.schema.fields.map((field) => field.name);
|
|
1982
|
+
if (!sourceColumnNames.includes(join.left)) {
|
|
1983
|
+
throw new Error(`Arrow join column not found: ${join.left}`);
|
|
1984
|
+
}
|
|
1985
|
+
if (!childColumnNames.includes(join.right)) {
|
|
1986
|
+
throw new Error(`Arrow join column not found: ${join.right}`);
|
|
1987
|
+
}
|
|
1988
|
+
const childPrefix = `${join.child.source}.`;
|
|
1989
|
+
const rootColumns = (_b = options.columns) == null ? void 0 : _b.filter((column) => !column.startsWith(childPrefix));
|
|
1990
|
+
planTableQuery2(sourceColumnNames, {
|
|
1991
|
+
predicate: options.predicate,
|
|
1992
|
+
columns: (rootColumns == null ? void 0 : rootColumns.length) ? rootColumns : void 0,
|
|
1993
|
+
limit: options.limit,
|
|
1994
|
+
signal: options.signal
|
|
1995
|
+
});
|
|
1996
|
+
const childQuery = join.child.query || {};
|
|
1997
|
+
planTableQuery2(childColumnNames, childQuery);
|
|
1998
|
+
const childOutputColumns = childQuery.columns || childColumnNames;
|
|
1999
|
+
if (!childOutputColumns.includes(join.right)) {
|
|
2000
|
+
throw new Error(`Arrow join child projection must include: ${join.right}`);
|
|
2001
|
+
}
|
|
2002
|
+
const projectedChildColumns = ((_c = options.columns) == null ? void 0 : _c.filter((column) => column.startsWith(childPrefix))) || [];
|
|
2003
|
+
const seenChildColumns = /* @__PURE__ */ new Set();
|
|
2004
|
+
for (const projectedColumn of projectedChildColumns) {
|
|
2005
|
+
const childColumn = projectedColumn.slice(childPrefix.length);
|
|
2006
|
+
if (!childColumnNames.includes(childColumn)) {
|
|
2007
|
+
throw new Error(`Arrow join column not found: ${projectedColumn}`);
|
|
2008
|
+
}
|
|
2009
|
+
if (seenChildColumns.has(childColumn)) {
|
|
2010
|
+
throw new Error(`Arrow query column was selected more than once: ${projectedColumn}`);
|
|
2011
|
+
}
|
|
2012
|
+
seenChildColumns.add(childColumn);
|
|
2013
|
+
if (!childOutputColumns.includes(childColumn)) {
|
|
2014
|
+
throw new Error(`Arrow join child projection does not include: ${childColumn}`);
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
function arrowTableToRows(data) {
|
|
2019
|
+
const columns = data.schema.fields.map((field) => ({
|
|
2020
|
+
name: field.name,
|
|
2021
|
+
vector: data.getChild(field.name)
|
|
2022
|
+
}));
|
|
2023
|
+
return Array.from({ length: data.numRows }, (_, rowIndex) => {
|
|
2024
|
+
const row = {};
|
|
2025
|
+
for (const column of columns) {
|
|
2026
|
+
if (!column.vector) throw new Error(`Arrow relational query could not read ${column.name}`);
|
|
2027
|
+
row[column.name] = column.vector.get(rowIndex);
|
|
2028
|
+
}
|
|
2029
|
+
return row;
|
|
2030
|
+
});
|
|
2031
|
+
}
|
|
2032
|
+
function evaluateRelationalExpression(expression, row) {
|
|
2033
|
+
const definition = expression.expression;
|
|
2034
|
+
if (definition.op === "literal") return definition.value;
|
|
2035
|
+
if (definition.op === "column") return row[definition.column];
|
|
2036
|
+
const left = row[definition.left];
|
|
2037
|
+
const right = row[definition.right];
|
|
2038
|
+
if (left === null || left === void 0 || right === null || right === void 0) return null;
|
|
2039
|
+
if (typeof left !== "number" || typeof right !== "number") {
|
|
2040
|
+
throw new Error(`Arrow relational expression ${expression.name} requires numeric operands.`);
|
|
2041
|
+
}
|
|
2042
|
+
switch (definition.op) {
|
|
2043
|
+
case "add":
|
|
2044
|
+
return left + right;
|
|
2045
|
+
case "subtract":
|
|
2046
|
+
return left - right;
|
|
2047
|
+
case "multiply":
|
|
2048
|
+
return left * right;
|
|
2049
|
+
case "divide":
|
|
2050
|
+
return right === 0 ? null : left / right;
|
|
2051
|
+
}
|
|
2052
|
+
}
|
|
2053
|
+
function compareRows(left, right, orderBy) {
|
|
2054
|
+
for (const key of orderBy) {
|
|
2055
|
+
const leftValue = left[key.column];
|
|
2056
|
+
const rightValue = right[key.column];
|
|
2057
|
+
const nulls = key.nulls || "last";
|
|
2058
|
+
if (leftValue === null || leftValue === void 0 || rightValue === null || rightValue === void 0) {
|
|
2059
|
+
if (leftValue === rightValue) continue;
|
|
2060
|
+
const nullResult = leftValue === null || leftValue === void 0 ? -1 : 1;
|
|
2061
|
+
return nulls === "first" ? nullResult : -nullResult;
|
|
2062
|
+
}
|
|
2063
|
+
const result = compareSortValues(leftValue, rightValue);
|
|
2064
|
+
if (result) return (key.direction === "desc" ? -1 : 1) * result;
|
|
2065
|
+
}
|
|
2066
|
+
return 0;
|
|
2067
|
+
}
|
|
2068
|
+
function compareSortValues(left, right) {
|
|
2069
|
+
if (left instanceof Date && right instanceof Date)
|
|
2070
|
+
return compareNumbers2(left.getTime(), right.getTime());
|
|
2071
|
+
if (typeof left === "bigint" && typeof right === "bigint")
|
|
2072
|
+
return left === right ? 0 : left < right ? -1 : 1;
|
|
2073
|
+
if (typeof left === "number" && typeof right === "number") return compareNumbers2(left, right);
|
|
2074
|
+
if (typeof left === "string" && typeof right === "string")
|
|
2075
|
+
return left === right ? 0 : left < right ? -1 : 1;
|
|
2076
|
+
if (typeof left === "boolean" && typeof right === "boolean") return Number(left) - Number(right);
|
|
2077
|
+
throw new Error(
|
|
2078
|
+
`Arrow relational order cannot compare ${getValueType(left)} with ${getValueType(right)}.`
|
|
2079
|
+
);
|
|
2080
|
+
}
|
|
2081
|
+
function aggregateRows(rows, groupBy, aggregates) {
|
|
2082
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2083
|
+
for (const row of rows) {
|
|
2084
|
+
const key = serializeGroupKey(groupBy.map((column) => row[column]));
|
|
2085
|
+
const group = groups.get(key) || [];
|
|
2086
|
+
group.push(row);
|
|
2087
|
+
groups.set(key, group);
|
|
2088
|
+
}
|
|
2089
|
+
if (!groupBy.length && !rows.length) groups.set("[]", []);
|
|
2090
|
+
return [...groups.values()].map((group) => {
|
|
2091
|
+
var _a;
|
|
2092
|
+
const output = {};
|
|
2093
|
+
for (const column of groupBy) output[column] = ((_a = group[0]) == null ? void 0 : _a[column]) ?? null;
|
|
2094
|
+
for (const aggregate of aggregates)
|
|
2095
|
+
output[aggregate.name] = evaluateAggregate(aggregate, group);
|
|
2096
|
+
return output;
|
|
2097
|
+
});
|
|
2098
|
+
}
|
|
2099
|
+
function serializeGroupKey(values) {
|
|
2100
|
+
return values.map((value) => {
|
|
2101
|
+
if (value === null || value === void 0) return "null:";
|
|
2102
|
+
if (value instanceof Date) return `date:${value.getTime()}`;
|
|
2103
|
+
if (value instanceof Uint8Array) return `bytes:${Array.from(value).join(".")}`;
|
|
2104
|
+
return `${typeof value}:${String(value)}`;
|
|
2105
|
+
}).join("");
|
|
2106
|
+
}
|
|
2107
|
+
function createEmptyRelationalResult(sourceData, outputColumns, options, additionalFields) {
|
|
2108
|
+
const sourceFields = new Map(sourceData.schema.fields.map((field) => [field.name, field]));
|
|
2109
|
+
const expressions = new Map(
|
|
2110
|
+
(options.expressions || []).map((expression) => [expression.name, expression])
|
|
2111
|
+
);
|
|
2112
|
+
const aggregates = new Map(
|
|
2113
|
+
(options.aggregates || []).map((aggregate) => [aggregate.name, aggregate])
|
|
2114
|
+
);
|
|
2115
|
+
const fields = outputColumns.map((columnName) => {
|
|
2116
|
+
const sourceField = sourceFields.get(columnName);
|
|
2117
|
+
if (sourceField) return sourceField;
|
|
2118
|
+
const additionalField = additionalFields == null ? void 0 : additionalFields.get(columnName);
|
|
2119
|
+
if (additionalField) return additionalField;
|
|
2120
|
+
const expression = expressions.get(columnName);
|
|
2121
|
+
if (expression)
|
|
2122
|
+
return new arrow6.Field(columnName, getExpressionType(expression, sourceFields), true);
|
|
2123
|
+
const aggregate = aggregates.get(columnName);
|
|
2124
|
+
if (aggregate)
|
|
2125
|
+
return new arrow6.Field(columnName, getAggregateType(aggregate, sourceFields), true);
|
|
2126
|
+
return new arrow6.Field(columnName, new arrow6.Utf8(), true);
|
|
2127
|
+
});
|
|
2128
|
+
const columns = {};
|
|
2129
|
+
for (const field of fields) columns[field.name] = arrow6.vectorFromArray([], field.type);
|
|
2130
|
+
return wrapArrowTable(new arrow6.Table(new arrow6.Schema(fields), columns));
|
|
2131
|
+
}
|
|
2132
|
+
function getExpressionType(expression, sourceFields) {
|
|
2133
|
+
var _a;
|
|
2134
|
+
if (expression.expression.op === "literal") {
|
|
2135
|
+
const value = expression.expression.value;
|
|
2136
|
+
if (typeof value === "boolean") return new arrow6.Bool();
|
|
2137
|
+
if (typeof value === "number") return new arrow6.Float64();
|
|
2138
|
+
return new arrow6.Utf8();
|
|
2139
|
+
}
|
|
2140
|
+
if (expression.expression.op === "column") {
|
|
2141
|
+
return ((_a = sourceFields.get(expression.expression.column)) == null ? void 0 : _a.type) || new arrow6.Utf8();
|
|
2142
|
+
}
|
|
2143
|
+
return new arrow6.Float64();
|
|
2144
|
+
}
|
|
2145
|
+
function getAggregateType(aggregate, sourceFields) {
|
|
2146
|
+
var _a;
|
|
2147
|
+
if (aggregate.function === "count") return new arrow6.Int32();
|
|
2148
|
+
return aggregate.column ? ((_a = sourceFields.get(aggregate.column)) == null ? void 0 : _a.type) || new arrow6.Float64() : new arrow6.Float64();
|
|
2149
|
+
}
|
|
2150
|
+
function evaluateAggregate(aggregate, rows) {
|
|
2151
|
+
if (aggregate.function === "count")
|
|
2152
|
+
return aggregate.column ? rows.filter((row) => row[aggregate.column] != null).length : rows.length;
|
|
2153
|
+
const values = rows.map((row) => row[aggregate.column]).filter((value) => value != null);
|
|
2154
|
+
if (!values.length) return null;
|
|
2155
|
+
if (!values.every((value) => typeof value === "number"))
|
|
2156
|
+
throw new Error(`Arrow aggregate ${aggregate.function} requires numeric values.`);
|
|
2157
|
+
const numericValues = values;
|
|
2158
|
+
switch (aggregate.function) {
|
|
2159
|
+
case "sum":
|
|
2160
|
+
return numericValues.reduce((sum, value) => sum + value, 0);
|
|
2161
|
+
case "avg":
|
|
2162
|
+
return numericValues.reduce((sum, value) => sum + value, 0) / numericValues.length;
|
|
2163
|
+
case "min":
|
|
2164
|
+
return Math.min(...numericValues);
|
|
2165
|
+
case "max":
|
|
2166
|
+
return Math.max(...numericValues);
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
function getPredicateColumns(sourceTable, columnNames) {
|
|
2170
|
+
const columns = {};
|
|
2171
|
+
for (const columnName of columnNames) {
|
|
2172
|
+
const column = sourceTable.getChild(columnName);
|
|
2173
|
+
if (!column) {
|
|
2174
|
+
throw new Error(`Arrow query column not found: ${columnName}`);
|
|
2175
|
+
}
|
|
2176
|
+
columns[columnName] = column;
|
|
2177
|
+
}
|
|
2178
|
+
return columns;
|
|
2179
|
+
}
|
|
2180
|
+
function evaluatePredicate(predicate, columns, rowIndex) {
|
|
2181
|
+
if (isLogicalPredicate(predicate)) {
|
|
2182
|
+
const values = predicate.args.map((child) => evaluatePredicate(child, columns, rowIndex));
|
|
2183
|
+
return predicate.op === "and" ? evaluateAnd(values) : evaluateOr(values);
|
|
2184
|
+
}
|
|
2185
|
+
if (isNotPredicate(predicate)) {
|
|
2186
|
+
const value2 = evaluatePredicate(predicate.args[0], columns, rowIndex);
|
|
2187
|
+
return value2 === null ? null : !value2;
|
|
2188
|
+
}
|
|
2189
|
+
const value = getColumnValue(columns, predicate.args[0].property, rowIndex);
|
|
2190
|
+
if (isNullPredicate(predicate)) {
|
|
2191
|
+
return value === null;
|
|
2192
|
+
}
|
|
2193
|
+
if (isInPredicate(predicate)) {
|
|
2194
|
+
return evaluateIn(value, predicate.args[1]);
|
|
2195
|
+
}
|
|
2196
|
+
return evaluateComparison(value, predicate);
|
|
2197
|
+
}
|
|
2198
|
+
function getColumnValue(columns, columnName, rowIndex) {
|
|
2199
|
+
const column = columns[columnName];
|
|
2200
|
+
if (!column) {
|
|
2201
|
+
throw new Error(`Arrow query could not read predicate column "${columnName}".`);
|
|
2202
|
+
}
|
|
2203
|
+
return column.get(rowIndex);
|
|
2204
|
+
}
|
|
2205
|
+
function evaluateAnd(values) {
|
|
2206
|
+
if (values.includes(false)) {
|
|
2207
|
+
return false;
|
|
2208
|
+
}
|
|
2209
|
+
return values.includes(null) ? null : true;
|
|
2210
|
+
}
|
|
2211
|
+
function evaluateOr(values) {
|
|
2212
|
+
if (values.includes(true)) {
|
|
2213
|
+
return true;
|
|
2214
|
+
}
|
|
2215
|
+
return values.includes(null) ? null : false;
|
|
2216
|
+
}
|
|
2217
|
+
function evaluateIn(value, candidates) {
|
|
2218
|
+
if (value === null || value === void 0) {
|
|
2219
|
+
return null;
|
|
2220
|
+
}
|
|
2221
|
+
for (const candidate of candidates) {
|
|
2222
|
+
if (compareValues(value, candidate) === 0) {
|
|
2223
|
+
return true;
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
return false;
|
|
2227
|
+
}
|
|
2228
|
+
function evaluateComparison(value, predicate) {
|
|
2229
|
+
if (value === null || value === void 0) {
|
|
2230
|
+
return null;
|
|
2231
|
+
}
|
|
2232
|
+
const comparison = compareValues(value, predicate.args[1]);
|
|
2233
|
+
switch (predicate.op) {
|
|
2234
|
+
case "=":
|
|
2235
|
+
return comparison === 0;
|
|
2236
|
+
case "<>":
|
|
2237
|
+
return comparison !== 0;
|
|
2238
|
+
case "<":
|
|
2239
|
+
return comparison < 0;
|
|
2240
|
+
case "<=":
|
|
2241
|
+
return comparison <= 0;
|
|
2242
|
+
case ">":
|
|
2243
|
+
return comparison > 0;
|
|
2244
|
+
case ">=":
|
|
2245
|
+
return comparison >= 0;
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
function compareValues(left, right) {
|
|
2249
|
+
if (isSQLPredicateParameter(right)) {
|
|
2250
|
+
throw new Error(
|
|
2251
|
+
`Arrow query parameter ":${right.parameter}" must be bound before query execution.`
|
|
2252
|
+
);
|
|
2253
|
+
}
|
|
2254
|
+
if (left instanceof Date && right instanceof Date) {
|
|
2255
|
+
return compareNumbers2(left.getTime(), right.getTime());
|
|
2256
|
+
}
|
|
2257
|
+
if (left instanceof Uint8Array && right instanceof Uint8Array) {
|
|
2258
|
+
return compareBinary(left, right);
|
|
2259
|
+
}
|
|
2260
|
+
if (typeof left === "bigint" && typeof right === "bigint") {
|
|
2261
|
+
return left === right ? 0 : left < right ? -1 : 1;
|
|
2262
|
+
}
|
|
2263
|
+
if (typeof left === "bigint" && typeof right === "number" && Number.isSafeInteger(right)) {
|
|
2264
|
+
return compareValues(left, BigInt(right));
|
|
2265
|
+
}
|
|
2266
|
+
if (typeof left === "number" && typeof right === "bigint" && Number.isSafeInteger(left)) {
|
|
2267
|
+
return compareValues(BigInt(left), right);
|
|
2268
|
+
}
|
|
2269
|
+
if (typeof left === "number" && typeof right === "number") {
|
|
2270
|
+
return compareNumbers2(left, right);
|
|
2271
|
+
}
|
|
2272
|
+
if (typeof left === "string" && typeof right === "string") {
|
|
2273
|
+
return left === right ? 0 : left < right ? -1 : 1;
|
|
2274
|
+
}
|
|
2275
|
+
if (typeof left === "boolean" && typeof right === "boolean") {
|
|
2276
|
+
return Number(left) - Number(right);
|
|
2277
|
+
}
|
|
2278
|
+
throw new Error(
|
|
2279
|
+
`Arrow query cannot compare ${getValueType(left)} with ${getValueType(right)} values.`
|
|
2280
|
+
);
|
|
2281
|
+
}
|
|
2282
|
+
function compareNumbers2(left, right) {
|
|
2283
|
+
if (!Number.isFinite(left) || !Number.isFinite(right)) {
|
|
2284
|
+
throw new Error("Arrow query cannot compare non-finite numbers.");
|
|
2285
|
+
}
|
|
2286
|
+
return left === right ? 0 : left < right ? -1 : 1;
|
|
2287
|
+
}
|
|
2288
|
+
function compareBinary(left, right) {
|
|
2289
|
+
const commonLength = Math.min(left.length, right.length);
|
|
2290
|
+
for (let index = 0; index < commonLength; index++) {
|
|
2291
|
+
if (left[index] !== right[index]) {
|
|
2292
|
+
return left[index] < right[index] ? -1 : 1;
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
return left.length === right.length ? 0 : left.length < right.length ? -1 : 1;
|
|
2296
|
+
}
|
|
2297
|
+
function getValueType(value) {
|
|
2298
|
+
if (value instanceof Date) {
|
|
2299
|
+
return "Date";
|
|
2300
|
+
}
|
|
2301
|
+
if (value instanceof Uint8Array) {
|
|
2302
|
+
return "Uint8Array";
|
|
2303
|
+
}
|
|
2304
|
+
return typeof value;
|
|
2305
|
+
}
|
|
2306
|
+
function isLogicalPredicate(predicate) {
|
|
2307
|
+
return predicate.op === "and" || predicate.op === "or";
|
|
2308
|
+
}
|
|
2309
|
+
function isNotPredicate(predicate) {
|
|
2310
|
+
return predicate.op === "not";
|
|
2311
|
+
}
|
|
2312
|
+
function isNullPredicate(predicate) {
|
|
2313
|
+
return predicate.op === "isNull";
|
|
2314
|
+
}
|
|
2315
|
+
function isInPredicate(predicate) {
|
|
2316
|
+
return predicate.op === "in";
|
|
2317
|
+
}
|
|
2318
|
+
function getScanStep(plan) {
|
|
2319
|
+
const step = plan.find((candidate) => candidate.kind === "scan");
|
|
2320
|
+
if (!step) {
|
|
2321
|
+
throw new Error("Arrow query plan is missing a scan step.");
|
|
2322
|
+
}
|
|
2323
|
+
return step;
|
|
2324
|
+
}
|
|
2325
|
+
function getFilterStep(plan) {
|
|
2326
|
+
return plan.find((candidate) => candidate.kind === "filter");
|
|
2327
|
+
}
|
|
2328
|
+
function getProjectStep(plan) {
|
|
2329
|
+
const step = plan.find(
|
|
2330
|
+
(candidate) => candidate.kind === "project"
|
|
2331
|
+
);
|
|
2332
|
+
if (!step) {
|
|
2333
|
+
throw new Error("Arrow query plan is missing a project step.");
|
|
2334
|
+
}
|
|
2335
|
+
return step;
|
|
2336
|
+
}
|
|
2337
|
+
function getLimitStep(plan) {
|
|
2338
|
+
return plan.find((candidate) => candidate.kind === "limit");
|
|
2339
|
+
}
|
|
2340
|
+
function throwIfAborted(signal) {
|
|
2341
|
+
if (signal == null ? void 0 : signal.aborted) {
|
|
2342
|
+
const error = new Error("Arrow query was aborted.");
|
|
2343
|
+
error.name = "AbortError";
|
|
2344
|
+
throw error;
|
|
2345
|
+
}
|
|
2346
|
+
}
|
|
2347
|
+
function wrapArrowTable(data) {
|
|
2348
|
+
return {
|
|
2349
|
+
shape: "arrow-table",
|
|
2350
|
+
schema: convertArrowToSchema(data.schema),
|
|
2351
|
+
data
|
|
2352
|
+
};
|
|
2353
|
+
}
|
|
2354
|
+
|
|
2355
|
+
// ../sql/src/parse-sql-predicate.ts
|
|
2356
|
+
var MAXIMUM_SQL_LENGTH = 65536;
|
|
2357
|
+
var MAXIMUM_SQL_TOKENS = 4096;
|
|
2358
|
+
var MAXIMUM_EXPRESSION_DEPTH = 64;
|
|
2359
|
+
function parseSQLPredicate(source, options = {}) {
|
|
2360
|
+
const parser = new SQLPredicateParser(
|
|
2361
|
+
source,
|
|
2362
|
+
options.parameters ?? {},
|
|
2363
|
+
options.preserveParameters === true
|
|
2364
|
+
);
|
|
2365
|
+
const predicate = parser.parse();
|
|
2366
|
+
validateSQLPredicate(predicate);
|
|
2367
|
+
return predicate;
|
|
2368
|
+
}
|
|
2369
|
+
var SQLPredicateParser = class {
|
|
2370
|
+
/** Token stream parsed by this parser instance. */
|
|
2371
|
+
tokens;
|
|
2372
|
+
/** Named scalar parameters available to the expression. */
|
|
2373
|
+
parameters;
|
|
2374
|
+
/** Whether named parameters remain references in the parsed AST. */
|
|
2375
|
+
preserveParameters;
|
|
2376
|
+
/** Current index in the token stream. */
|
|
2377
|
+
position = 0;
|
|
2378
|
+
constructor(source, parameters, preserveParameters) {
|
|
2379
|
+
this.tokens = tokenizeSQLPredicate(source);
|
|
2380
|
+
this.parameters = parameters;
|
|
2381
|
+
this.preserveParameters = preserveParameters;
|
|
2382
|
+
}
|
|
2383
|
+
/** Parses the complete token stream. */
|
|
2384
|
+
parse() {
|
|
2385
|
+
const predicate = this.parseOr(0);
|
|
2386
|
+
this.takeSymbol(";");
|
|
2387
|
+
const token = this.peek();
|
|
2388
|
+
if (token) {
|
|
2389
|
+
throw new Error(`SQL predicate contains unexpected token ${JSON.stringify(token.value)}`);
|
|
2390
|
+
}
|
|
2391
|
+
return predicate;
|
|
2392
|
+
}
|
|
2393
|
+
/** Parses left-associative disjunction. */
|
|
2394
|
+
parseOr(depth) {
|
|
2395
|
+
let predicate = this.parseAnd(depth + 1);
|
|
2396
|
+
while (this.takeKeyword("OR")) {
|
|
2397
|
+
predicate = combineLogicalPredicate("or", predicate, this.parseAnd(depth + 1));
|
|
2398
|
+
}
|
|
2399
|
+
return predicate;
|
|
2400
|
+
}
|
|
2401
|
+
/** Parses left-associative conjunction. */
|
|
2402
|
+
parseAnd(depth) {
|
|
2403
|
+
let predicate = this.parseNot(depth + 1);
|
|
2404
|
+
while (this.takeKeyword("AND")) {
|
|
2405
|
+
predicate = combineLogicalPredicate("and", predicate, this.parseNot(depth + 1));
|
|
2406
|
+
}
|
|
2407
|
+
return predicate;
|
|
2408
|
+
}
|
|
2409
|
+
/** Parses boolean negation and parenthesized predicates. */
|
|
2410
|
+
parseNot(depth) {
|
|
2411
|
+
this.assertDepth(depth);
|
|
2412
|
+
if (this.takeKeyword("NOT")) {
|
|
2413
|
+
return { op: "not", args: [this.parseNot(depth + 1)] };
|
|
2414
|
+
}
|
|
2415
|
+
if (this.takeSymbol("(")) {
|
|
2416
|
+
const predicate = this.parseOr(depth + 1);
|
|
2417
|
+
this.expectSymbol(")");
|
|
2418
|
+
return predicate;
|
|
2419
|
+
}
|
|
2420
|
+
return this.parseLeaf();
|
|
2421
|
+
}
|
|
2422
|
+
/** Parses one comparison, membership, or null predicate. */
|
|
2423
|
+
parseLeaf() {
|
|
2424
|
+
var _a, _b;
|
|
2425
|
+
const property = this.parseProperty();
|
|
2426
|
+
if (this.takeKeyword("IS")) {
|
|
2427
|
+
const negated = this.takeKeyword("NOT");
|
|
2428
|
+
this.expectKeyword("NULL");
|
|
2429
|
+
const predicate = { op: "isNull", args: [property] };
|
|
2430
|
+
return negated ? { op: "not", args: [predicate] } : predicate;
|
|
2431
|
+
}
|
|
2432
|
+
const negatedMembership = this.takeKeyword("NOT");
|
|
2433
|
+
if (negatedMembership || this.peekKeyword("IN")) {
|
|
2434
|
+
this.expectKeyword("IN");
|
|
2435
|
+
this.expectSymbol("(");
|
|
2436
|
+
if (((_a = this.peek()) == null ? void 0 : _a.kind) === "symbol" && ((_b = this.peek()) == null ? void 0 : _b.value) === ")") {
|
|
2437
|
+
throw new Error("SQL predicate IN requires at least one scalar value");
|
|
2438
|
+
}
|
|
2439
|
+
const values = [this.parseValue()];
|
|
2440
|
+
while (this.takeSymbol(",")) {
|
|
2441
|
+
values.push(this.parseValue());
|
|
2442
|
+
}
|
|
2443
|
+
this.expectSymbol(")");
|
|
2444
|
+
const predicate = { op: "in", args: [property, values] };
|
|
2445
|
+
return negatedMembership ? { op: "not", args: [predicate] } : predicate;
|
|
2446
|
+
}
|
|
2447
|
+
const token = this.take();
|
|
2448
|
+
if ((token == null ? void 0 : token.kind) !== "symbol" || !["=", "!=", "<>", "<", "<=", ">", ">="].includes(token.value)) {
|
|
2449
|
+
throw new Error("SQL predicate expected a comparison, IN, or IS NULL operator");
|
|
2450
|
+
}
|
|
2451
|
+
const operator = token.value === "!=" ? "<>" : token.value;
|
|
2452
|
+
return {
|
|
2453
|
+
op: operator,
|
|
2454
|
+
args: [property, this.parseValue()]
|
|
2455
|
+
};
|
|
2456
|
+
}
|
|
2457
|
+
/** Parses an identifier into a property reference. */
|
|
2458
|
+
parseProperty() {
|
|
2459
|
+
const token = this.take();
|
|
2460
|
+
if ((token == null ? void 0 : token.kind) !== "identifier") {
|
|
2461
|
+
throw new Error("SQL predicate expected a column identifier");
|
|
2462
|
+
}
|
|
2463
|
+
return token.quoted ? { property: token.value, quoted: true } : { property: token.value };
|
|
2464
|
+
}
|
|
2465
|
+
/** Parses one scalar literal or named parameter. */
|
|
2466
|
+
parseValue() {
|
|
2467
|
+
const negative = this.takeSymbol("-");
|
|
2468
|
+
const token = this.take();
|
|
2469
|
+
if (!token) {
|
|
2470
|
+
throw new Error("SQL predicate expected a scalar value");
|
|
2471
|
+
}
|
|
2472
|
+
if (negative && token.kind !== "number") {
|
|
2473
|
+
throw new Error("SQL predicate only permits unary minus on numeric literals");
|
|
2474
|
+
}
|
|
2475
|
+
if (token.kind === "number") {
|
|
2476
|
+
return parseNumericLiteral(negative ? `-${token.value}` : token.value);
|
|
2477
|
+
}
|
|
2478
|
+
if (negative) {
|
|
2479
|
+
throw new Error("SQL predicate expected a numeric literal after unary minus");
|
|
2480
|
+
}
|
|
2481
|
+
if (token.kind === "string") {
|
|
2482
|
+
return token.value;
|
|
2483
|
+
}
|
|
2484
|
+
if (token.kind === "parameter") {
|
|
2485
|
+
if (this.preserveParameters) {
|
|
2486
|
+
return { parameter: token.value };
|
|
2487
|
+
}
|
|
2488
|
+
if (!Object.hasOwn(this.parameters, token.value)) {
|
|
2489
|
+
throw new Error(`SQL predicate parameter ":${token.value}" requires a value`);
|
|
2490
|
+
}
|
|
2491
|
+
const value = this.parameters[token.value];
|
|
2492
|
+
validateParameterValue(value, token.value);
|
|
2493
|
+
return value;
|
|
2494
|
+
}
|
|
2495
|
+
if (token.kind === "identifier" && /^(TRUE|FALSE)$/i.test(token.value)) {
|
|
2496
|
+
return /^TRUE$/i.test(token.value);
|
|
2497
|
+
}
|
|
2498
|
+
if (token.kind === "identifier" && /^NULL$/i.test(token.value)) {
|
|
2499
|
+
throw new Error("SQL predicate comparisons with NULL must use IS NULL or IS NOT NULL");
|
|
2500
|
+
}
|
|
2501
|
+
throw new Error(`SQL predicate does not support scalar token ${JSON.stringify(token.value)}`);
|
|
2502
|
+
}
|
|
2503
|
+
/** Rejects excessively nested inputs before recursion can exhaust the stack. */
|
|
2504
|
+
assertDepth(depth) {
|
|
2505
|
+
if (depth > MAXIMUM_EXPRESSION_DEPTH) {
|
|
2506
|
+
throw new Error("SQL predicate nesting exceeds its safe limit");
|
|
2507
|
+
}
|
|
2508
|
+
}
|
|
2509
|
+
/** Consumes one required keyword. */
|
|
2510
|
+
expectKeyword(keyword) {
|
|
2511
|
+
if (!this.takeKeyword(keyword)) {
|
|
2512
|
+
throw new Error(`SQL predicate expected ${keyword}`);
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
/** Returns whether the next token is a keyword without consuming it. */
|
|
2516
|
+
peekKeyword(keyword) {
|
|
2517
|
+
const token = this.peek();
|
|
2518
|
+
return (token == null ? void 0 : token.kind) === "identifier" && token.value.toUpperCase() === keyword;
|
|
2519
|
+
}
|
|
2520
|
+
/** Consumes one optional keyword. */
|
|
2521
|
+
takeKeyword(keyword) {
|
|
2522
|
+
if (!this.peekKeyword(keyword)) {
|
|
2523
|
+
return false;
|
|
2524
|
+
}
|
|
2525
|
+
this.position++;
|
|
2526
|
+
return true;
|
|
2527
|
+
}
|
|
2528
|
+
/** Consumes one required symbol. */
|
|
2529
|
+
expectSymbol(symbol) {
|
|
2530
|
+
if (!this.takeSymbol(symbol)) {
|
|
2531
|
+
throw new Error(`SQL predicate expected ${JSON.stringify(symbol)}`);
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
/** Consumes one optional symbol. */
|
|
2535
|
+
takeSymbol(symbol) {
|
|
2536
|
+
const token = this.peek();
|
|
2537
|
+
if ((token == null ? void 0 : token.kind) !== "symbol" || token.value !== symbol) {
|
|
2538
|
+
return false;
|
|
2539
|
+
}
|
|
2540
|
+
this.position++;
|
|
2541
|
+
return true;
|
|
2542
|
+
}
|
|
2543
|
+
/** Consumes and returns the next token. */
|
|
2544
|
+
take() {
|
|
2545
|
+
const token = this.tokens[this.position];
|
|
2546
|
+
if (token) {
|
|
2547
|
+
this.position++;
|
|
2548
|
+
}
|
|
2549
|
+
return token;
|
|
2550
|
+
}
|
|
2551
|
+
/** Returns the next token without consuming it. */
|
|
2552
|
+
peek() {
|
|
2553
|
+
return this.tokens[this.position];
|
|
2554
|
+
}
|
|
2555
|
+
};
|
|
2556
|
+
function combineLogicalPredicate(operator, left, right) {
|
|
2557
|
+
return left.op === operator ? { op: operator, args: [...left.args, right] } : { op: operator, args: [left, right] };
|
|
2558
|
+
}
|
|
2559
|
+
function parseNumericLiteral(source) {
|
|
2560
|
+
if (/^-?\d+$/.test(source)) {
|
|
2561
|
+
const bigint = BigInt(source);
|
|
2562
|
+
if (bigint > BigInt(Number.MAX_SAFE_INTEGER) || bigint < BigInt(Number.MIN_SAFE_INTEGER)) {
|
|
2563
|
+
return bigint;
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
const number = Number(source);
|
|
2567
|
+
if (!Number.isFinite(number)) {
|
|
2568
|
+
throw new Error("SQL predicate numeric literals must be finite");
|
|
2569
|
+
}
|
|
2570
|
+
return number;
|
|
2571
|
+
}
|
|
2572
|
+
function validateParameterValue(value, name) {
|
|
2573
|
+
const probe = { op: "=", args: [{ property: "_" }, value] };
|
|
2574
|
+
try {
|
|
2575
|
+
validateSQLPredicate(probe);
|
|
2576
|
+
} catch {
|
|
2577
|
+
throw new Error(`SQL predicate parameter ":${name}" has an unsupported value`);
|
|
2578
|
+
}
|
|
2579
|
+
}
|
|
2580
|
+
function tokenizeSQLPredicate(source) {
|
|
2581
|
+
if (typeof source !== "string" || source.trim().length === 0 || source.length > MAXIMUM_SQL_LENGTH) {
|
|
2582
|
+
throw new Error("SQL predicate requires a non-empty, safely bounded expression");
|
|
2583
|
+
}
|
|
2584
|
+
const tokens = [];
|
|
2585
|
+
let position = 0;
|
|
2586
|
+
while (position < source.length) {
|
|
2587
|
+
const character = source[position];
|
|
2588
|
+
if (/\s/.test(character)) {
|
|
2589
|
+
position++;
|
|
2590
|
+
continue;
|
|
2591
|
+
}
|
|
2592
|
+
if (character === "'") {
|
|
2593
|
+
const result = readQuotedToken(source, position, "'", "'");
|
|
2594
|
+
tokens.push({ kind: "string", value: result.value });
|
|
2595
|
+
position = result.position;
|
|
2596
|
+
} else if (character === '"') {
|
|
2597
|
+
const result = readQuotedToken(source, position, '"', '"');
|
|
2598
|
+
tokens.push({ kind: "identifier", value: result.value, quoted: true });
|
|
2599
|
+
position = result.position;
|
|
2600
|
+
} else if (character === ":") {
|
|
2601
|
+
const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(source.slice(position + 1));
|
|
2602
|
+
if (!match) {
|
|
2603
|
+
throw new Error(`SQL predicate contains an invalid parameter at character ${position}`);
|
|
2604
|
+
}
|
|
2605
|
+
tokens.push({ kind: "parameter", value: match[0] });
|
|
2606
|
+
position += match[0].length + 1;
|
|
2607
|
+
} else {
|
|
2608
|
+
const numberMatch = /^(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?/.exec(source.slice(position));
|
|
2609
|
+
const identifierMatch = /^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*/.exec(
|
|
2610
|
+
source.slice(position)
|
|
2611
|
+
);
|
|
2612
|
+
if (numberMatch) {
|
|
2613
|
+
tokens.push({ kind: "number", value: numberMatch[0] });
|
|
2614
|
+
position += numberMatch[0].length;
|
|
2615
|
+
} else if (identifierMatch) {
|
|
2616
|
+
tokens.push({ kind: "identifier", value: identifierMatch[0] });
|
|
2617
|
+
position += identifierMatch[0].length;
|
|
2618
|
+
} else {
|
|
2619
|
+
const twoCharacters = source.slice(position, position + 2);
|
|
2620
|
+
if (["<=", ">=", "<>", "!="].includes(twoCharacters)) {
|
|
2621
|
+
tokens.push({ kind: "symbol", value: twoCharacters });
|
|
2622
|
+
position += 2;
|
|
2623
|
+
} else if ("(),;-=<>".includes(character)) {
|
|
2624
|
+
tokens.push({ kind: "symbol", value: character });
|
|
2625
|
+
position++;
|
|
2626
|
+
} else {
|
|
2627
|
+
throw new Error(`SQL predicate contains an unsupported token at character ${position}`);
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
if (tokens.length > MAXIMUM_SQL_TOKENS) {
|
|
2632
|
+
throw new Error("SQL predicate exceeds its safe token limit");
|
|
2633
|
+
}
|
|
2634
|
+
}
|
|
2635
|
+
return tokens;
|
|
2636
|
+
}
|
|
2637
|
+
function readQuotedToken(source, start, quote, escapedQuote) {
|
|
2638
|
+
let value = "";
|
|
2639
|
+
let position = start + 1;
|
|
2640
|
+
while (position < source.length) {
|
|
2641
|
+
if (source[position] !== quote) {
|
|
2642
|
+
value += source[position++];
|
|
2643
|
+
continue;
|
|
2644
|
+
}
|
|
2645
|
+
if (source[position + 1] === escapedQuote) {
|
|
2646
|
+
value += quote;
|
|
2647
|
+
position += 2;
|
|
2648
|
+
continue;
|
|
2649
|
+
}
|
|
2650
|
+
return { value, position: position + 1 };
|
|
2651
|
+
}
|
|
2652
|
+
throw new Error(`SQL predicate contains an unterminated quoted value at character ${start}`);
|
|
2653
|
+
}
|
|
2654
|
+
|
|
2655
|
+
// dist/scan-engine.js
|
|
2656
|
+
var scanBackendLoaders = /* @__PURE__ */ new Map();
|
|
2657
|
+
var arrowScanBackend = Object.freeze({
|
|
2658
|
+
name: "arrow",
|
|
2659
|
+
query: queryArrowTable,
|
|
2660
|
+
queryAsync: async (sourceTable, options) => queryArrowTable(sourceTable, options),
|
|
2661
|
+
explain: explainArrowTableQuery
|
|
2662
|
+
});
|
|
2663
|
+
scanBackendLoaders.set("arrow", () => arrowScanBackend);
|
|
2664
|
+
function registerScanBackend(name, loader) {
|
|
2665
|
+
if (!name || !loader) {
|
|
2666
|
+
throw new Error("A scan backend name and loader are required");
|
|
2667
|
+
}
|
|
2668
|
+
scanBackendLoaders.set(name, loader);
|
|
2669
|
+
}
|
|
2670
|
+
async function createScanEngine(options = {}) {
|
|
2671
|
+
const backendName = options.backend || "arrow";
|
|
2672
|
+
const loader = scanBackendLoaders.get(backendName);
|
|
2673
|
+
if (!loader) {
|
|
2674
|
+
throw new Error(`Scan backend "${backendName}" is not registered. The built-in backend is "arrow".`);
|
|
2675
|
+
}
|
|
2676
|
+
const backend = await loader();
|
|
2677
|
+
if (backend.name !== backendName) {
|
|
2678
|
+
throw new Error(`Scan backend loader returned "${backend.name}" while "${backendName}" was requested.`);
|
|
2679
|
+
}
|
|
2680
|
+
const query = backend.query.bind(backend);
|
|
2681
|
+
const explain = backend.explain.bind(backend);
|
|
2682
|
+
const queryAsync = backend.queryAsync ? backend.queryAsync.bind(backend) : async (sourceTable, queryOptions) => backend.query.call(backend, sourceTable, queryOptions);
|
|
2683
|
+
return Object.freeze({ ...backend, query, explain, queryAsync });
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
// dist/federated-table-scan-source.js
|
|
2687
|
+
var arrow7 = __toESM(require("apache-arrow"), 1);
|
|
2688
|
+
var FEDERATED_TABLE_QUERY_CAPABILITIES = Object.freeze({
|
|
2689
|
+
predicate: "residual",
|
|
2690
|
+
projection: "pushdown+residual",
|
|
2691
|
+
limit: "pushdown+residual",
|
|
2692
|
+
streaming: true,
|
|
2693
|
+
cancellation: true
|
|
2694
|
+
});
|
|
2695
|
+
var nextFederatedSourceId = 0;
|
|
2696
|
+
var FederatedTableScanSource = class {
|
|
2697
|
+
/** Shared manager used as the authoritative named-source registry. */
|
|
2698
|
+
dataSourceManager;
|
|
2699
|
+
/** Immutable sources in physical append order. */
|
|
2700
|
+
sources;
|
|
2701
|
+
/** Schema reconciliation policy. */
|
|
2702
|
+
schemaPolicy;
|
|
2703
|
+
/** Optional caller-declared normalized output schema. */
|
|
2704
|
+
outputSchema;
|
|
2705
|
+
/** Optional display name reported through scan metadata. */
|
|
2706
|
+
name;
|
|
2707
|
+
/** Human-readable description reported through scan metadata. */
|
|
2708
|
+
description;
|
|
2709
|
+
/** Stable id used to isolate this adapter's manager subscriptions. */
|
|
2710
|
+
instanceId = nextFederatedSourceId++;
|
|
2711
|
+
/** Monotonic id used to isolate overlapping operations on this adapter. */
|
|
2712
|
+
nextOperationId = 0;
|
|
2713
|
+
/** Creates an ordered federated source backed by one shared DataSourceManager. */
|
|
2714
|
+
constructor(dataSourceManager, options) {
|
|
2715
|
+
if (!options.sources.length) {
|
|
2716
|
+
throw new Error("Federated table scans require at least one source.");
|
|
2717
|
+
}
|
|
2718
|
+
if (options.schemaPolicy && options.schemaPolicy !== "strict" && options.schemaPolicy !== "union") {
|
|
2719
|
+
throw new Error(`Unsupported federated schema policy: ${options.schemaPolicy}`);
|
|
2720
|
+
}
|
|
2721
|
+
this.dataSourceManager = dataSourceManager;
|
|
2722
|
+
this.sources = Object.freeze(options.sources.map(cloneSourceEntry));
|
|
2723
|
+
this.schemaPolicy = options.schemaPolicy || "strict";
|
|
2724
|
+
this.outputSchema = options.outputSchema ? cloneSchema(options.outputSchema) : void 0;
|
|
2725
|
+
this.name = options.name;
|
|
2726
|
+
this.description = options.description || `${this.sources.length} ordered table sources`;
|
|
2727
|
+
}
|
|
2728
|
+
/** Discovers and reconciles every child schema without reading federated result batches. */
|
|
2729
|
+
async getQueryMetadata(options = {}) {
|
|
2730
|
+
const consumerId = this.createConsumerId("metadata");
|
|
2731
|
+
try {
|
|
2732
|
+
const plan = await this.resolvePlan(consumerId, options.signal);
|
|
2733
|
+
return createScanQueryMetadata({
|
|
2734
|
+
sourceType: "federated-table",
|
|
2735
|
+
queryType: "table",
|
|
2736
|
+
execution: { status: "supported", method: "read" },
|
|
2737
|
+
name: this.name,
|
|
2738
|
+
description: this.description,
|
|
2739
|
+
schema: plan.schema,
|
|
2740
|
+
capabilities: { table: FEDERATED_TABLE_QUERY_CAPABILITIES },
|
|
2741
|
+
statistics: getFederatedStatistics(plan.sources)
|
|
2742
|
+
});
|
|
2743
|
+
} finally {
|
|
2744
|
+
this.dataSourceManager.unsubscribe({ consumerId });
|
|
2745
|
+
}
|
|
2746
|
+
}
|
|
2747
|
+
/** Explains source resolution, schema reconciliation, and the global table query. */
|
|
2748
|
+
async explain(options = {}) {
|
|
2749
|
+
const consumerId = this.createConsumerId("explain");
|
|
2750
|
+
try {
|
|
2751
|
+
const plan = await this.resolvePlan(consumerId, options.signal);
|
|
2752
|
+
const explanation = explainTableQuery(plan.schema.fields.map((field) => field.name), options, FEDERATED_TABLE_QUERY_CAPABILITIES);
|
|
2753
|
+
return Object.freeze({
|
|
2754
|
+
...explanation,
|
|
2755
|
+
schemaPolicy: this.schemaPolicy,
|
|
2756
|
+
sources: Object.freeze(plan.sources.map((source) => createSourceExplanation(source, plan.schema)))
|
|
2757
|
+
});
|
|
2758
|
+
} finally {
|
|
2759
|
+
this.dataSourceManager.unsubscribe({ consumerId });
|
|
2760
|
+
}
|
|
2761
|
+
}
|
|
2762
|
+
/**
|
|
2763
|
+
* Emits source and batch ordered Arrow results with one global limit and batch provenance.
|
|
2764
|
+
*
|
|
2765
|
+
* Returning early from the iterator closes the active child iterator and releases every manager
|
|
2766
|
+
* subscription. Later sources are never read once the global limit has been satisfied.
|
|
2767
|
+
*/
|
|
2768
|
+
async *read(options = {}) {
|
|
2769
|
+
var _a, _b;
|
|
2770
|
+
const consumerId = this.createConsumerId("read");
|
|
2771
|
+
const startedAt = Date.now();
|
|
2772
|
+
const telemetry = {
|
|
2773
|
+
status: "early-terminated",
|
|
2774
|
+
sourcesPlanned: this.sources.length,
|
|
2775
|
+
sourcesRead: 0,
|
|
2776
|
+
batchesRead: 0,
|
|
2777
|
+
rowsRead: 0,
|
|
2778
|
+
rowsTested: 0,
|
|
2779
|
+
rowsRetained: 0,
|
|
2780
|
+
rowsReturned: 0,
|
|
2781
|
+
bytesFetched: 0,
|
|
2782
|
+
filesOpened: 0,
|
|
2783
|
+
tasksOpened: 0,
|
|
2784
|
+
rowsPruned: 0,
|
|
2785
|
+
durationMilliseconds: 0,
|
|
2786
|
+
sources: []
|
|
2787
|
+
};
|
|
2788
|
+
try {
|
|
2789
|
+
const plan = await this.resolvePlan(consumerId, options.signal);
|
|
2790
|
+
const scanStep = planTableQuery(plan.schema.fields.map((field) => field.name), options)[0];
|
|
2791
|
+
let remaining = options.limit ?? Number.POSITIVE_INFINITY;
|
|
2792
|
+
if (remaining <= 0) {
|
|
2793
|
+
telemetry.earlyTerminationReason = "limit";
|
|
2794
|
+
return;
|
|
2795
|
+
}
|
|
2796
|
+
for (const resolvedSource of plan.sources) {
|
|
2797
|
+
throwIfAborted2(options.signal);
|
|
2798
|
+
if (remaining <= 0) {
|
|
2799
|
+
telemetry.earlyTerminationReason = "limit";
|
|
2800
|
+
return;
|
|
2801
|
+
}
|
|
2802
|
+
const sourceStartedAt = Date.now();
|
|
2803
|
+
let childTelemetry;
|
|
2804
|
+
let sourceCompleted = false;
|
|
2805
|
+
let sourceError;
|
|
2806
|
+
let sourceBatchesDecoded = 0;
|
|
2807
|
+
let sourceRowsRead = 0;
|
|
2808
|
+
let sourceRowsTested = 0;
|
|
2809
|
+
let sourceRowsRetained = 0;
|
|
2810
|
+
let sourceRowsReturned = 0;
|
|
2811
|
+
const sourceReadOptions = createSourceReadOptions(resolvedSource, scanStep.columns, remaining, options, (value) => {
|
|
2812
|
+
childTelemetry = value;
|
|
2813
|
+
});
|
|
2814
|
+
const sourceResidualPredicate = getSourceResidualPredicate(resolvedSource);
|
|
2815
|
+
let sourceRemaining = ((_a = resolvedSource.entry.query) == null ? void 0 : _a.limit) ?? Number.POSITIVE_INFINITY;
|
|
2816
|
+
telemetry.sourcesRead++;
|
|
2817
|
+
let sourceBatchIndex = 0;
|
|
2818
|
+
try {
|
|
2819
|
+
for await (const batch of resolvedSource.source.read(sourceReadOptions)) {
|
|
2820
|
+
throwIfAborted2(options.signal);
|
|
2821
|
+
if (remaining <= 0) {
|
|
2822
|
+
telemetry.earlyTerminationReason = "limit";
|
|
2823
|
+
return;
|
|
2824
|
+
}
|
|
2825
|
+
if (sourceRemaining <= 0)
|
|
2826
|
+
break;
|
|
2827
|
+
if (batch.batchType !== "data")
|
|
2828
|
+
continue;
|
|
2829
|
+
telemetry.batchesRead++;
|
|
2830
|
+
telemetry.rowsRead += batch.length;
|
|
2831
|
+
sourceBatchesDecoded++;
|
|
2832
|
+
sourceRowsRead += batch.length;
|
|
2833
|
+
if (batch.length <= 0)
|
|
2834
|
+
continue;
|
|
2835
|
+
const physicalTable = convertBatch(batch.schema ? batch : { ...batch, schema: resolvedSource.metadata.schema }, "arrow-table");
|
|
2836
|
+
const sourceResult = queryArrowTable(physicalTable, {
|
|
2837
|
+
predicate: sourceResidualPredicate,
|
|
2838
|
+
limit: Number.isFinite(sourceRemaining) ? sourceRemaining : void 0,
|
|
2839
|
+
signal: options.signal
|
|
2840
|
+
});
|
|
2841
|
+
if (sourceResidualPredicate) {
|
|
2842
|
+
sourceRowsTested += physicalTable.data.numRows;
|
|
2843
|
+
sourceRowsRetained += sourceResult.data.numRows;
|
|
2844
|
+
}
|
|
2845
|
+
sourceRemaining -= sourceResult.data.numRows;
|
|
2846
|
+
const currentSourceBatchIndex = sourceBatchIndex++;
|
|
2847
|
+
if (!sourceResult.data.numRows)
|
|
2848
|
+
continue;
|
|
2849
|
+
const canonicalTable = createCanonicalArrowTable(plan.schema, resolvedSource, sourceResult);
|
|
2850
|
+
const result = queryArrowTable(canonicalTable, {
|
|
2851
|
+
predicate: options.predicate,
|
|
2852
|
+
columns: options.columns,
|
|
2853
|
+
limit: Number.isFinite(remaining) ? remaining : void 0,
|
|
2854
|
+
signal: options.signal
|
|
2855
|
+
});
|
|
2856
|
+
if (options.predicate) {
|
|
2857
|
+
telemetry.rowsTested += canonicalTable.data.numRows;
|
|
2858
|
+
telemetry.rowsRetained += result.data.numRows;
|
|
2859
|
+
}
|
|
2860
|
+
const resultLength = result.data.numRows;
|
|
2861
|
+
if (!resultLength)
|
|
2862
|
+
continue;
|
|
2863
|
+
const provenance = Object.freeze({
|
|
2864
|
+
sourceId: resolvedSource.entry.dataSourceId,
|
|
2865
|
+
sourceIndex: resolvedSource.sourceIndex,
|
|
2866
|
+
sourceBatchIndex: currentSourceBatchIndex,
|
|
2867
|
+
sourceMetadata: batch.metadata
|
|
2868
|
+
});
|
|
2869
|
+
telemetry.rowsReturned += resultLength;
|
|
2870
|
+
sourceRowsReturned += resultLength;
|
|
2871
|
+
yield {
|
|
2872
|
+
batchType: "data",
|
|
2873
|
+
shape: "arrow-table",
|
|
2874
|
+
schema: convertArrowToSchema(result.data.schema),
|
|
2875
|
+
data: result.data,
|
|
2876
|
+
length: resultLength,
|
|
2877
|
+
metadata: provenance,
|
|
2878
|
+
sourceId: provenance.sourceId,
|
|
2879
|
+
sourceIndex: provenance.sourceIndex,
|
|
2880
|
+
sourceBatchIndex: provenance.sourceBatchIndex
|
|
2881
|
+
};
|
|
2882
|
+
remaining -= resultLength;
|
|
2883
|
+
}
|
|
2884
|
+
sourceCompleted = true;
|
|
2885
|
+
} catch (error) {
|
|
2886
|
+
sourceError = error;
|
|
2887
|
+
throw error;
|
|
2888
|
+
} finally {
|
|
2889
|
+
const sourceTelemetry = createSourceTelemetry({
|
|
2890
|
+
resolvedSource,
|
|
2891
|
+
childTelemetry,
|
|
2892
|
+
sourceStartedAt,
|
|
2893
|
+
sourceCompleted,
|
|
2894
|
+
sourceError,
|
|
2895
|
+
signal: options.signal,
|
|
2896
|
+
batchesDecoded: sourceBatchesDecoded,
|
|
2897
|
+
rowsRead: sourceRowsRead,
|
|
2898
|
+
rowsTested: sourceRowsTested,
|
|
2899
|
+
rowsRetained: sourceRowsRetained,
|
|
2900
|
+
rowsReturned: sourceRowsReturned
|
|
2901
|
+
});
|
|
2902
|
+
telemetry.sources.push(sourceTelemetry);
|
|
2903
|
+
telemetry.bytesFetched += sourceTelemetry.bytesFetched || 0;
|
|
2904
|
+
telemetry.filesOpened += sourceTelemetry.filesOpened || 0;
|
|
2905
|
+
telemetry.tasksOpened += sourceTelemetry.tasksOpened || 0;
|
|
2906
|
+
telemetry.rowsPruned += sourceTelemetry.rowsPruned || 0;
|
|
2907
|
+
}
|
|
2908
|
+
}
|
|
2909
|
+
telemetry.status = "completed";
|
|
2910
|
+
} catch (error) {
|
|
2911
|
+
telemetry.status = ((_b = options.signal) == null ? void 0 : _b.aborted) ? "cancelled" : "failed";
|
|
2912
|
+
telemetry.error = error;
|
|
2913
|
+
throw error;
|
|
2914
|
+
} finally {
|
|
2915
|
+
this.dataSourceManager.unsubscribe({ consumerId });
|
|
2916
|
+
telemetry.durationMilliseconds = Date.now() - startedAt;
|
|
2917
|
+
if (telemetry.status === "early-terminated" && !telemetry.earlyTerminationReason) {
|
|
2918
|
+
telemetry.earlyTerminationReason = "consumer-return";
|
|
2919
|
+
}
|
|
2920
|
+
emitScanExecutionTelemetry(options.onTelemetry, Object.freeze({
|
|
2921
|
+
...telemetry,
|
|
2922
|
+
batchesDecoded: telemetry.batchesRead,
|
|
2923
|
+
sources: Object.freeze([...telemetry.sources])
|
|
2924
|
+
}));
|
|
2925
|
+
}
|
|
2926
|
+
}
|
|
2927
|
+
/** Resolves managed sources, discovers their schemas, and creates one compatibility plan. */
|
|
2928
|
+
async resolvePlan(consumerId, signal) {
|
|
2929
|
+
const sources = [];
|
|
2930
|
+
for (let sourceIndex = 0; sourceIndex < this.sources.length; sourceIndex++) {
|
|
2931
|
+
throwIfAborted2(signal);
|
|
2932
|
+
const entry = this.sources[sourceIndex];
|
|
2933
|
+
const source = await resolveManagedSource(this.dataSourceManager, entry.dataSourceId, consumerId, String(sourceIndex), signal);
|
|
2934
|
+
if (typeof source.getQueryMetadata !== "function" || typeof source.read !== "function") {
|
|
2935
|
+
throw new Error(`Managed source does not implement TableScanSource: ${entry.dataSourceId}`);
|
|
2936
|
+
}
|
|
2937
|
+
const metadata = await source.getQueryMetadata({ signal });
|
|
2938
|
+
if (metadata.execution.status !== "supported" || metadata.execution.method !== "read") {
|
|
2939
|
+
throw new Error(`Federated table source does not support read(): ${entry.dataSourceId}`);
|
|
2940
|
+
}
|
|
2941
|
+
if (metadata.queryType !== "table") {
|
|
2942
|
+
throw new Error(`Federated source is not a table source: ${entry.dataSourceId}`);
|
|
2943
|
+
}
|
|
2944
|
+
const fields = resolveMappedFields(entry, metadata);
|
|
2945
|
+
sources.push({
|
|
2946
|
+
entry,
|
|
2947
|
+
source,
|
|
2948
|
+
metadata,
|
|
2949
|
+
sourceIndex,
|
|
2950
|
+
fields,
|
|
2951
|
+
sourceNameByOutputName: new Map(fields.map((field) => [field.outputName, field.sourceName]))
|
|
2952
|
+
});
|
|
2953
|
+
}
|
|
2954
|
+
return Object.freeze({
|
|
2955
|
+
schema: reconcileSchemas(sources, this.schemaPolicy, this.outputSchema),
|
|
2956
|
+
sources: Object.freeze(sources)
|
|
2957
|
+
});
|
|
2958
|
+
}
|
|
2959
|
+
/** Returns a collision-free manager consumer id for one operation. */
|
|
2960
|
+
createConsumerId(operation) {
|
|
2961
|
+
return `federated-table:${this.instanceId}:${operation}:${this.nextOperationId++}`;
|
|
2962
|
+
}
|
|
2963
|
+
};
|
|
2964
|
+
function cloneSourceEntry(entry) {
|
|
2965
|
+
if (!entry.dataSourceId) {
|
|
2966
|
+
throw new Error("Federated table source ids must be non-empty.");
|
|
2967
|
+
}
|
|
2968
|
+
return Object.freeze({
|
|
2969
|
+
dataSourceId: entry.dataSourceId,
|
|
2970
|
+
query: entry.query ? Object.freeze({
|
|
2971
|
+
...entry.query,
|
|
2972
|
+
columns: entry.query.columns ? Object.freeze([...entry.query.columns]) : void 0
|
|
2973
|
+
}) : void 0,
|
|
2974
|
+
columnMapping: entry.columnMapping ? Object.freeze({ ...entry.columnMapping }) : void 0
|
|
2975
|
+
});
|
|
2976
|
+
}
|
|
2977
|
+
function resolveMappedFields(entry, metadata) {
|
|
2978
|
+
var _a;
|
|
2979
|
+
const sourceFields = new Map(metadata.schema.fields.map((field) => [field.name, field]));
|
|
2980
|
+
for (const sourceName of Object.keys(entry.columnMapping || {})) {
|
|
2981
|
+
if (!sourceFields.has(sourceName)) {
|
|
2982
|
+
throw new Error(`Federated column mapping source not found in ${entry.dataSourceId}: ${sourceName}`);
|
|
2983
|
+
}
|
|
2984
|
+
}
|
|
2985
|
+
const selectedNames = ((_a = entry.query) == null ? void 0 : _a.columns) || metadata.schema.fields.map((field) => field.name);
|
|
2986
|
+
if (!selectedNames.length) {
|
|
2987
|
+
throw new Error(`Federated source projection must not be empty: ${entry.dataSourceId}`);
|
|
2988
|
+
}
|
|
2989
|
+
const outputNames = /* @__PURE__ */ new Set();
|
|
2990
|
+
return Object.freeze(selectedNames.map((sourceName) => {
|
|
2991
|
+
var _a2;
|
|
2992
|
+
const field = sourceFields.get(sourceName);
|
|
2993
|
+
if (!field) {
|
|
2994
|
+
throw new Error(`Federated source column not found in ${entry.dataSourceId}: ${sourceName}`);
|
|
2995
|
+
}
|
|
2996
|
+
const outputName = ((_a2 = entry.columnMapping) == null ? void 0 : _a2[sourceName]) ?? sourceName;
|
|
2997
|
+
if (!outputName) {
|
|
2998
|
+
throw new Error(`Federated output column names must be non-empty: ${sourceName}`);
|
|
2999
|
+
}
|
|
3000
|
+
if (outputNames.has(outputName)) {
|
|
3001
|
+
throw new Error(`Federated column mapping creates duplicate output column in ${entry.dataSourceId}: ${outputName}`);
|
|
3002
|
+
}
|
|
3003
|
+
outputNames.add(outputName);
|
|
3004
|
+
return Object.freeze({ sourceName, outputName, field });
|
|
3005
|
+
}));
|
|
3006
|
+
}
|
|
3007
|
+
function reconcileSchemas(sources, schemaPolicy, outputSchema) {
|
|
3008
|
+
const outputFields = /* @__PURE__ */ new Map();
|
|
3009
|
+
const firstSourceNames = sources[0].fields.map((field) => field.outputName);
|
|
3010
|
+
for (const source of sources) {
|
|
3011
|
+
const sourceNames = new Set(source.fields.map((field) => field.outputName));
|
|
3012
|
+
if (schemaPolicy === "strict") {
|
|
3013
|
+
const missing = firstSourceNames.filter((name) => !sourceNames.has(name));
|
|
3014
|
+
const extra = source.fields.map((field) => field.outputName).filter((name) => !firstSourceNames.includes(name));
|
|
3015
|
+
if (missing.length || extra.length) {
|
|
3016
|
+
throw new Error(`Federated strict schema mismatch for ${source.entry.dataSourceId}: missing [${missing.join(", ")}], extra [${extra.join(", ")}]`);
|
|
3017
|
+
}
|
|
3018
|
+
}
|
|
3019
|
+
for (const mappedField of source.fields) {
|
|
3020
|
+
const current = outputFields.get(mappedField.outputName);
|
|
3021
|
+
if (current) {
|
|
3022
|
+
if (!outputSchema && !areDataTypesEqual(current.type, mappedField.field.type)) {
|
|
3023
|
+
throw new Error(`Federated column type mismatch for ${mappedField.outputName}: ${formatDataType(current.type)} versus ${formatDataType(mappedField.field.type)} in ${source.entry.dataSourceId}`);
|
|
3024
|
+
}
|
|
3025
|
+
current.nullable = Boolean(current.nullable || mappedField.field.nullable);
|
|
3026
|
+
current.presentCount++;
|
|
3027
|
+
} else {
|
|
3028
|
+
outputFields.set(mappedField.outputName, {
|
|
3029
|
+
...mappedField.field,
|
|
3030
|
+
name: mappedField.outputName,
|
|
3031
|
+
metadata: mappedField.field.metadata ? { ...mappedField.field.metadata } : void 0,
|
|
3032
|
+
presentCount: 1
|
|
3033
|
+
});
|
|
3034
|
+
}
|
|
3035
|
+
}
|
|
3036
|
+
}
|
|
3037
|
+
const fields = [...outputFields.values()].map(({ presentCount, ...field }) => ({
|
|
3038
|
+
...field,
|
|
3039
|
+
nullable: Boolean(field.nullable || presentCount < sources.length)
|
|
3040
|
+
}));
|
|
3041
|
+
const inferredSchema = { fields, metadata: {} };
|
|
3042
|
+
return outputSchema ? validateAndCloneOutputSchema(outputSchema, sources, inferredSchema, schemaPolicy) : inferredSchema;
|
|
3043
|
+
}
|
|
3044
|
+
function validateAndCloneOutputSchema(outputSchema, sources, inferredSchema, schemaPolicy) {
|
|
3045
|
+
const outputFields = /* @__PURE__ */ new Map();
|
|
3046
|
+
for (const field of outputSchema.fields) {
|
|
3047
|
+
if (!field.name || outputFields.has(field.name)) {
|
|
3048
|
+
throw new Error(`Federated output schema contains an invalid or duplicate field: ${field.name}`);
|
|
3049
|
+
}
|
|
3050
|
+
outputFields.set(field.name, field);
|
|
3051
|
+
}
|
|
3052
|
+
const inferredNames = new Set(inferredSchema.fields.map((field) => field.name));
|
|
3053
|
+
const missingFromContract = [...inferredNames].filter((name) => !outputFields.has(name));
|
|
3054
|
+
const absentFromSources = outputSchema.fields.map((field) => field.name).filter((name) => !inferredNames.has(name));
|
|
3055
|
+
if (missingFromContract.length || schemaPolicy === "strict" && absentFromSources.length) {
|
|
3056
|
+
throw new Error(`Federated output schema mismatch: missing [${missingFromContract.join(", ")}], absent [${absentFromSources.join(", ")}]`);
|
|
3057
|
+
}
|
|
3058
|
+
for (const outputField of outputSchema.fields) {
|
|
3059
|
+
const presentFields = sources.flatMap((source) => source.fields.filter((field) => field.outputName === outputField.name).map((field) => ({ sourceId: source.entry.dataSourceId, field: field.field })));
|
|
3060
|
+
if (!presentFields.length) {
|
|
3061
|
+
if (!outputField.nullable) {
|
|
3062
|
+
throw new Error(`Federated output field is absent from every source and must be nullable: ${outputField.name}`);
|
|
3063
|
+
}
|
|
3064
|
+
continue;
|
|
3065
|
+
}
|
|
3066
|
+
for (const presentField of presentFields) {
|
|
3067
|
+
if (presentField.field.nullable && !outputField.nullable) {
|
|
3068
|
+
throw new Error(`Federated output field cannot remove nullability for ${outputField.name} in ${presentField.sourceId}`);
|
|
3069
|
+
}
|
|
3070
|
+
if (!isLosslessDataTypeCast(presentField.field.type, outputField.type)) {
|
|
3071
|
+
throw new Error(`Unsupported federated normalization for ${outputField.name}: ${formatDataType(presentField.field.type)} to ${formatDataType(outputField.type)} in ${presentField.sourceId}`);
|
|
3072
|
+
}
|
|
3073
|
+
}
|
|
3074
|
+
if (presentFields.length < sources.length && !outputField.nullable) {
|
|
3075
|
+
throw new Error(`Federated union field must be nullable when absent from a source: ${outputField.name}`);
|
|
3076
|
+
}
|
|
3077
|
+
}
|
|
3078
|
+
return cloneSchema(outputSchema);
|
|
3079
|
+
}
|
|
3080
|
+
function createSourceReadOptions(source, requiredOutputColumns, remaining, options, onTelemetry) {
|
|
3081
|
+
var _a, _b;
|
|
3082
|
+
const sourceResidualPredicate = getSourceResidualPredicate(source);
|
|
3083
|
+
const requiredSourceColumns = requiredOutputColumns.map((outputName) => source.sourceNameByOutputName.get(outputName)).filter((sourceName) => Boolean(sourceName));
|
|
3084
|
+
if (sourceResidualPredicate) {
|
|
3085
|
+
requiredSourceColumns.push(...getColumnarPredicateColumns(sourceResidualPredicate));
|
|
3086
|
+
}
|
|
3087
|
+
if (!requiredSourceColumns.length) {
|
|
3088
|
+
requiredSourceColumns.push(source.fields[0].sourceName);
|
|
3089
|
+
}
|
|
3090
|
+
const sourceLimit = (_a = source.entry.query) == null ? void 0 : _a.limit;
|
|
3091
|
+
const globalLimitCanBePushed = options.predicate === void 0 && Number.isFinite(remaining);
|
|
3092
|
+
const limit = globalLimitCanBePushed ? Math.min(sourceLimit ?? Number.POSITIVE_INFINITY, remaining) : sourceLimit;
|
|
3093
|
+
return {
|
|
3094
|
+
...source.entry.query,
|
|
3095
|
+
columns: Object.freeze([...new Set(requiredSourceColumns)]),
|
|
3096
|
+
predicate: sourceResidualPredicate ? void 0 : (_b = source.entry.query) == null ? void 0 : _b.predicate,
|
|
3097
|
+
limit: sourceResidualPredicate ? void 0 : Number.isFinite(limit) ? limit : void 0,
|
|
3098
|
+
signal: options.signal,
|
|
3099
|
+
onTelemetry
|
|
3100
|
+
};
|
|
3101
|
+
}
|
|
3102
|
+
function getSourceResidualPredicate(source) {
|
|
3103
|
+
var _a, _b;
|
|
3104
|
+
return ((_a = source.metadata.capabilities.table) == null ? void 0 : _a.predicate) === "unsupported" ? (_b = source.entry.query) == null ? void 0 : _b.predicate : void 0;
|
|
3105
|
+
}
|
|
3106
|
+
function createCanonicalArrowTable(schema, source, physicalTable) {
|
|
3107
|
+
const arrowSchema = convertSchemaToArrow(schema);
|
|
3108
|
+
const columns = {};
|
|
3109
|
+
for (const field of arrowSchema.fields) {
|
|
3110
|
+
const sourceName = source.sourceNameByOutputName.get(field.name);
|
|
3111
|
+
const vector = sourceName ? physicalTable.data.getChild(sourceName) : null;
|
|
3112
|
+
const mappedField = source.fields.find((sourceField) => sourceField.outputName === field.name);
|
|
3113
|
+
columns[field.name] = vector ? mappedField && !areDataTypesEqual(mappedField.field.type, schema.fields.find((schemaField) => schemaField.name === field.name).type) ? arrow7.vectorFromArray(Array.from(vector), field.type) : vector : arrow7.vectorFromArray(Array.from({ length: physicalTable.data.numRows }, () => null), field.type);
|
|
3114
|
+
}
|
|
3115
|
+
return {
|
|
3116
|
+
shape: "arrow-table",
|
|
3117
|
+
schema,
|
|
3118
|
+
data: new arrow7.Table(arrowSchema, columns)
|
|
3119
|
+
};
|
|
3120
|
+
}
|
|
3121
|
+
function createSourceTelemetry(options) {
|
|
3122
|
+
var _a;
|
|
3123
|
+
const { resolvedSource, childTelemetry } = options;
|
|
3124
|
+
const status = (childTelemetry == null ? void 0 : childTelemetry.status) || (((_a = options.signal) == null ? void 0 : _a.aborted) ? "cancelled" : options.sourceError ? "failed" : options.sourceCompleted ? "completed" : "early-terminated");
|
|
3125
|
+
const details = childTelemetry ? Object.freeze(Object.fromEntries(Object.entries(childTelemetry).filter(([key]) => ![
|
|
3126
|
+
"status",
|
|
3127
|
+
"sourcesPlanned",
|
|
3128
|
+
"sourcesRead",
|
|
3129
|
+
"batchesRead",
|
|
3130
|
+
"batchesDecoded",
|
|
3131
|
+
"rowsRead",
|
|
3132
|
+
"rowsTested",
|
|
3133
|
+
"rowsRetained",
|
|
3134
|
+
"rowsReturned",
|
|
3135
|
+
"bytesRead",
|
|
3136
|
+
"bytesFetched",
|
|
3137
|
+
"filesOpened",
|
|
3138
|
+
"tasksOpened",
|
|
3139
|
+
"rowsPruned",
|
|
3140
|
+
"durationMilliseconds",
|
|
3141
|
+
"sources",
|
|
3142
|
+
"details",
|
|
3143
|
+
"error"
|
|
3144
|
+
].includes(key)))) : void 0;
|
|
3145
|
+
return Object.freeze({
|
|
3146
|
+
sourceId: resolvedSource.entry.dataSourceId,
|
|
3147
|
+
sourceType: resolvedSource.metadata.sourceType,
|
|
3148
|
+
sourceIndex: resolvedSource.sourceIndex,
|
|
3149
|
+
status,
|
|
3150
|
+
filesOpened: childTelemetry == null ? void 0 : childTelemetry.filesOpened,
|
|
3151
|
+
tasksOpened: childTelemetry == null ? void 0 : childTelemetry.tasksOpened,
|
|
3152
|
+
bytesFetched: (childTelemetry == null ? void 0 : childTelemetry.bytesFetched) ?? (childTelemetry == null ? void 0 : childTelemetry.bytesRead),
|
|
3153
|
+
batchesDecoded: (childTelemetry == null ? void 0 : childTelemetry.batchesDecoded) ?? options.batchesDecoded,
|
|
3154
|
+
rowsRead: (childTelemetry == null ? void 0 : childTelemetry.rowsRead) ?? options.rowsRead,
|
|
3155
|
+
rowsTested: (childTelemetry == null ? void 0 : childTelemetry.rowsTested) || options.rowsTested || void 0,
|
|
3156
|
+
rowsRetained: (childTelemetry == null ? void 0 : childTelemetry.rowsRetained) || options.rowsRetained || void 0,
|
|
3157
|
+
rowsReturned: options.rowsReturned,
|
|
3158
|
+
rowsPruned: childTelemetry == null ? void 0 : childTelemetry.rowsPruned,
|
|
3159
|
+
durationMilliseconds: Date.now() - options.sourceStartedAt,
|
|
3160
|
+
details: (childTelemetry == null ? void 0 : childTelemetry.details) ?? (details && Object.keys(details).length ? details : void 0),
|
|
3161
|
+
error: (childTelemetry == null ? void 0 : childTelemetry.error) ?? options.sourceError
|
|
3162
|
+
});
|
|
3163
|
+
}
|
|
3164
|
+
function getFederatedStatistics(sources) {
|
|
3165
|
+
if (sources.some((source) => {
|
|
3166
|
+
var _a, _b, _c;
|
|
3167
|
+
return ((_a = source.entry.query) == null ? void 0 : _a.predicate) || ((_b = source.entry.query) == null ? void 0 : _b.limit) !== void 0 || ((_c = source.metadata.statistics) == null ? void 0 : _c.rowCount) === void 0;
|
|
3168
|
+
})) {
|
|
3169
|
+
return void 0;
|
|
3170
|
+
}
|
|
3171
|
+
const counts = sources.map((source) => source.metadata.statistics.rowCount);
|
|
3172
|
+
return {
|
|
3173
|
+
rowCount: counts.some((count) => typeof count === "bigint") ? counts.reduce((sum, count) => sum + BigInt(count), 0n) : counts.reduce((sum, count) => sum + Number(count), 0)
|
|
3174
|
+
};
|
|
3175
|
+
}
|
|
3176
|
+
function createSourceExplanation(source, outputSchema) {
|
|
3177
|
+
const normalizedTypes = Object.fromEntries(source.fields.filter((field) => {
|
|
3178
|
+
const outputField = outputSchema.fields.find((schemaField) => schemaField.name === field.outputName);
|
|
3179
|
+
return outputField && !areDataTypesEqual(outputField.type, field.field.type);
|
|
3180
|
+
}).map((field) => [
|
|
3181
|
+
field.outputName,
|
|
3182
|
+
outputSchema.fields.find((schemaField) => schemaField.name === field.outputName).type
|
|
3183
|
+
]));
|
|
3184
|
+
return Object.freeze({
|
|
3185
|
+
sourceId: source.entry.dataSourceId,
|
|
3186
|
+
sourceIndex: source.sourceIndex,
|
|
3187
|
+
sourceType: source.metadata.sourceType,
|
|
3188
|
+
sourceColumns: Object.freeze(source.fields.map((field) => field.sourceName)),
|
|
3189
|
+
outputColumns: Object.freeze(source.fields.map((field) => field.outputName)),
|
|
3190
|
+
columnMapping: Object.freeze({ ...source.entry.columnMapping }),
|
|
3191
|
+
normalizedTypes: Object.freeze(normalizedTypes)
|
|
3192
|
+
});
|
|
3193
|
+
}
|
|
3194
|
+
function isLosslessDataTypeCast(sourceType, outputType) {
|
|
3195
|
+
if (areDataTypesEqual(sourceType, outputType) || sourceType === "null")
|
|
3196
|
+
return true;
|
|
3197
|
+
if (typeof sourceType === "object" && sourceType.type === "dictionary") {
|
|
3198
|
+
return isLosslessDataTypeCast(sourceType.dictionary, outputType);
|
|
3199
|
+
}
|
|
3200
|
+
if (sourceType === "utf8-view" && outputType === "utf8")
|
|
3201
|
+
return true;
|
|
3202
|
+
if (sourceType === "binary-view" && outputType === "binary")
|
|
3203
|
+
return true;
|
|
3204
|
+
const sourceInteger = getIntegerType(sourceType);
|
|
3205
|
+
const outputInteger = getIntegerType(outputType);
|
|
3206
|
+
if (sourceInteger && outputInteger) {
|
|
3207
|
+
return sourceInteger.signed === outputInteger.signed && sourceInteger.bits <= outputInteger.bits || !sourceInteger.signed && outputInteger.signed && sourceInteger.bits < outputInteger.bits;
|
|
3208
|
+
}
|
|
3209
|
+
const sourceFloatBits = getFloatBits(sourceType);
|
|
3210
|
+
const outputFloatBits = getFloatBits(outputType);
|
|
3211
|
+
if (sourceFloatBits && outputFloatBits)
|
|
3212
|
+
return sourceFloatBits <= outputFloatBits;
|
|
3213
|
+
return Boolean(sourceInteger && sourceInteger.bits <= 32 && outputFloatBits === 64);
|
|
3214
|
+
}
|
|
3215
|
+
function getIntegerType(dataType) {
|
|
3216
|
+
const types = {
|
|
3217
|
+
int: { signed: true, bits: 32 },
|
|
3218
|
+
int8: { signed: true, bits: 8 },
|
|
3219
|
+
int16: { signed: true, bits: 16 },
|
|
3220
|
+
int32: { signed: true, bits: 32 },
|
|
3221
|
+
int64: { signed: true, bits: 64 },
|
|
3222
|
+
uint8: { signed: false, bits: 8 },
|
|
3223
|
+
uint16: { signed: false, bits: 16 },
|
|
3224
|
+
uint32: { signed: false, bits: 32 },
|
|
3225
|
+
uint64: { signed: false, bits: 64 }
|
|
3226
|
+
};
|
|
3227
|
+
return typeof dataType === "string" ? types[dataType] || null : null;
|
|
3228
|
+
}
|
|
3229
|
+
function getFloatBits(dataType) {
|
|
3230
|
+
const types = { float: 32, float16: 16, float32: 32, float64: 64 };
|
|
3231
|
+
return typeof dataType === "string" ? types[dataType] || null : null;
|
|
3232
|
+
}
|
|
3233
|
+
function cloneSchema(schema) {
|
|
3234
|
+
return convertArrowToSchema(convertSchemaToArrow(schema));
|
|
3235
|
+
}
|
|
3236
|
+
function areDataTypesEqual(left, right) {
|
|
3237
|
+
return formatDataType(left) === formatDataType(right);
|
|
3238
|
+
}
|
|
3239
|
+
function formatDataType(dataType) {
|
|
3240
|
+
return typeof dataType === "string" ? dataType : JSON.stringify(canonicalizeValue(dataType));
|
|
3241
|
+
}
|
|
3242
|
+
function canonicalizeValue(value) {
|
|
3243
|
+
if (ArrayBuffer.isView(value)) {
|
|
3244
|
+
return Array.from(value);
|
|
3245
|
+
}
|
|
3246
|
+
if (Array.isArray(value)) {
|
|
3247
|
+
return value.map(canonicalizeValue);
|
|
3248
|
+
}
|
|
3249
|
+
if (value && typeof value === "object") {
|
|
3250
|
+
return Object.fromEntries(Object.entries(value).sort(([leftKey], [rightKey]) => leftKey.localeCompare(rightKey)).map(([key, child]) => [key, canonicalizeValue(child)]));
|
|
3251
|
+
}
|
|
3252
|
+
return value;
|
|
3253
|
+
}
|
|
3254
|
+
function resolveManagedSource(dataSourceManager, dataSourceId, consumerId, requestId, signal) {
|
|
3255
|
+
let resolveDeferredSource;
|
|
3256
|
+
let rejectDeferredSource;
|
|
3257
|
+
const deferredSource = new Promise((resolve, reject) => {
|
|
3258
|
+
resolveDeferredSource = resolve;
|
|
3259
|
+
rejectDeferredSource = reject;
|
|
3260
|
+
});
|
|
3261
|
+
let sourceGeneration = 0;
|
|
3262
|
+
const acceptSource = (source) => {
|
|
3263
|
+
const generation = ++sourceGeneration;
|
|
3264
|
+
if (source) {
|
|
3265
|
+
void Promise.resolve(source).then((result) => {
|
|
3266
|
+
if (generation === sourceGeneration)
|
|
3267
|
+
resolveDeferredSource(result);
|
|
3268
|
+
}, (error) => {
|
|
3269
|
+
if (generation === sourceGeneration)
|
|
3270
|
+
rejectDeferredSource(error);
|
|
3271
|
+
});
|
|
3272
|
+
}
|
|
3273
|
+
};
|
|
3274
|
+
const managedSource = dataSourceManager.subscribe({
|
|
3275
|
+
dataSourceId,
|
|
3276
|
+
consumerId,
|
|
3277
|
+
requestId,
|
|
3278
|
+
onChange: acceptSource
|
|
3279
|
+
});
|
|
3280
|
+
if (managedSource === void 0) {
|
|
3281
|
+
throw new Error(`Federated table source is not registered: ${dataSourceId}`);
|
|
3282
|
+
}
|
|
3283
|
+
acceptSource(managedSource || null);
|
|
3284
|
+
return waitForPromise(deferredSource, signal);
|
|
3285
|
+
}
|
|
3286
|
+
async function waitForPromise(promise, signal) {
|
|
3287
|
+
if (!signal)
|
|
3288
|
+
return await promise;
|
|
3289
|
+
throwIfAborted2(signal);
|
|
3290
|
+
return await new Promise((resolve, reject) => {
|
|
3291
|
+
const abort = () => reject(getAbortReason(signal));
|
|
3292
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
3293
|
+
void promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
|
|
3294
|
+
});
|
|
3295
|
+
}
|
|
3296
|
+
function throwIfAborted2(signal) {
|
|
3297
|
+
if (signal == null ? void 0 : signal.aborted)
|
|
3298
|
+
throw getAbortReason(signal);
|
|
3299
|
+
}
|
|
3300
|
+
function getAbortReason(signal) {
|
|
3301
|
+
return signal.reason || new DOMException("Request aborted", "AbortError");
|
|
3302
|
+
}
|
|
3303
|
+
|
|
3304
|
+
// dist/vector-table-scan-source.js
|
|
3305
|
+
var AddressedVectorTableScanSource = class {
|
|
3306
|
+
/** Stable adapter type reported by `getQueryMetadata()`. */
|
|
3307
|
+
sourceType;
|
|
3308
|
+
/** Optional display name shown by query controls. */
|
|
3309
|
+
name;
|
|
3310
|
+
/** Human-readable description of the bound physical selection. */
|
|
3311
|
+
description;
|
|
3312
|
+
/** Spatial extent of the bound physical selection. */
|
|
3313
|
+
spatial;
|
|
3314
|
+
tablePromise = null;
|
|
3315
|
+
/** Creates a table-scan view over one immutable physical selection. */
|
|
3316
|
+
constructor(options, defaults) {
|
|
3317
|
+
this.sourceType = options.sourceType || defaults.sourceType;
|
|
3318
|
+
this.name = options.name;
|
|
3319
|
+
this.description = options.description || defaults.description;
|
|
3320
|
+
this.spatial = defaults.spatial;
|
|
3321
|
+
}
|
|
3322
|
+
/** Discovers the addressed result schema and portable residual-query capabilities. */
|
|
3323
|
+
async getQueryMetadata(options = {}) {
|
|
3324
|
+
const table = await this.getTable(options.signal);
|
|
3325
|
+
const schema = table.schema || convertArrowToSchema(table.data.schema);
|
|
3326
|
+
return createScanQueryMetadata({
|
|
3327
|
+
sourceType: this.sourceType,
|
|
3328
|
+
queryType: "table",
|
|
3329
|
+
execution: { status: "supported", method: "read" },
|
|
3330
|
+
name: this.name,
|
|
3331
|
+
description: this.description,
|
|
3332
|
+
schema,
|
|
3333
|
+
capabilities: {
|
|
3334
|
+
table: ARROW_TABLE_QUERY_CAPABILITIES,
|
|
3335
|
+
bounds: "unsupported",
|
|
3336
|
+
levelOfDetail: "unsupported"
|
|
3337
|
+
},
|
|
3338
|
+
columnRoles: inferVectorColumnRoles(schema.fields.map((field) => field.name)),
|
|
3339
|
+
spatial: this.spatial,
|
|
3340
|
+
statistics: { rowCount: table.data.numRows }
|
|
3341
|
+
});
|
|
3342
|
+
}
|
|
3343
|
+
/** Applies the portable Arrow query to the addressed feature table. */
|
|
3344
|
+
async query(options = {}) {
|
|
3345
|
+
const table = await this.getTable(options.signal);
|
|
3346
|
+
return queryArrowTable(table, options);
|
|
3347
|
+
}
|
|
3348
|
+
/** Explains the residual Arrow plan after resolving the addressed feature table. */
|
|
3349
|
+
async explain(options = {}) {
|
|
3350
|
+
const table = await this.getTable(options.signal);
|
|
3351
|
+
return explainArrowTableQuery(table, options);
|
|
3352
|
+
}
|
|
3353
|
+
/** Emits the portable query result as one bounded Arrow batch. */
|
|
3354
|
+
async *read(options = {}) {
|
|
3355
|
+
const result = await this.query(options);
|
|
3356
|
+
yield makeTableScanBatch(result);
|
|
3357
|
+
}
|
|
3358
|
+
/** Resolves and caches the addressed table while allowing failed loads to be retried. */
|
|
3359
|
+
async getTable(signal) {
|
|
3360
|
+
throwIfAborted3(signal);
|
|
3361
|
+
if (!this.tablePromise) {
|
|
3362
|
+
this.tablePromise = this.loadTable().then((table) => {
|
|
3363
|
+
if (!table) {
|
|
3364
|
+
throw new Error("The addressed vector selection did not return a feature table.");
|
|
3365
|
+
}
|
|
3366
|
+
if (table.shape !== "arrow-table") {
|
|
3367
|
+
throw new Error('The addressed vector selection must return an Arrow table. Configure the source with shape "arrow-table".');
|
|
3368
|
+
}
|
|
3369
|
+
return table;
|
|
3370
|
+
}).catch((error) => {
|
|
3371
|
+
this.tablePromise = null;
|
|
3372
|
+
throw error;
|
|
3373
|
+
});
|
|
3374
|
+
}
|
|
3375
|
+
return await waitForPromise2(this.tablePromise, signal);
|
|
3376
|
+
}
|
|
3377
|
+
};
|
|
3378
|
+
var VectorTileTableScanSource = class extends AddressedVectorTableScanSource {
|
|
3379
|
+
/** Specialized tile source that owns tile addressing and decoding. */
|
|
3380
|
+
source;
|
|
3381
|
+
/** Immutable tile address bound to this table view. */
|
|
3382
|
+
tile;
|
|
3383
|
+
/** Creates a portable table view over one vector tile. */
|
|
3384
|
+
constructor(source, options) {
|
|
3385
|
+
const tile = Object.freeze({
|
|
3386
|
+
...options.tile,
|
|
3387
|
+
layers: cloneAndFreezeLayers(options.tile.layers)
|
|
3388
|
+
});
|
|
3389
|
+
super(options, {
|
|
3390
|
+
sourceType: "vector-tile-table",
|
|
3391
|
+
description: `Vector tile ${tile.z}/${tile.x}/${tile.y}`,
|
|
3392
|
+
spatial: {
|
|
3393
|
+
bounds: getWGS84TileBounds(tile),
|
|
3394
|
+
coordinateReferenceSystems: ["EPSG:4326"]
|
|
3395
|
+
}
|
|
3396
|
+
});
|
|
3397
|
+
this.source = source;
|
|
3398
|
+
this.tile = tile;
|
|
3399
|
+
}
|
|
3400
|
+
/** Loads the bound vector tile using the source's requested Arrow output shape. */
|
|
3401
|
+
loadTable(signal) {
|
|
3402
|
+
return this.source.getVectorTile({ ...this.tile, signal });
|
|
3403
|
+
}
|
|
3404
|
+
};
|
|
3405
|
+
var VectorFeatureTableScanSource = class extends AddressedVectorTableScanSource {
|
|
3406
|
+
/** Specialized vector source that owns service request construction and parsing. */
|
|
3407
|
+
source;
|
|
3408
|
+
/** Immutable bounded feature request bound to this table view. */
|
|
3409
|
+
request;
|
|
3410
|
+
/** Creates a portable table view over one bounded vector-service request. */
|
|
3411
|
+
constructor(source, options) {
|
|
3412
|
+
const request = Object.freeze({
|
|
3413
|
+
...options.request,
|
|
3414
|
+
layers: cloneAndFreezeLayers(options.request.layers),
|
|
3415
|
+
boundingBox: Object.freeze([
|
|
3416
|
+
Object.freeze([...options.request.boundingBox[0]]),
|
|
3417
|
+
Object.freeze([...options.request.boundingBox[1]])
|
|
3418
|
+
])
|
|
3419
|
+
});
|
|
3420
|
+
const coordinateReferenceSystems = typeof request.crs === "string" ? [request.crs] : void 0;
|
|
3421
|
+
super(options, {
|
|
3422
|
+
sourceType: "vector-feature-table",
|
|
3423
|
+
description: `Vector feature request for ${normalizeLayers(request.layers).join(", ")}`,
|
|
3424
|
+
spatial: {
|
|
3425
|
+
bounds: {
|
|
3426
|
+
minimum: Object.freeze([...request.boundingBox[0]]),
|
|
3427
|
+
maximum: Object.freeze([...request.boundingBox[1]])
|
|
3428
|
+
},
|
|
3429
|
+
coordinateReferenceSystems
|
|
3430
|
+
}
|
|
3431
|
+
});
|
|
3432
|
+
this.source = source;
|
|
3433
|
+
this.request = request;
|
|
3434
|
+
}
|
|
3435
|
+
/** Loads the bound service request through the source's Arrow feature output. */
|
|
3436
|
+
loadTable(signal) {
|
|
3437
|
+
return this.source.getFeatures({
|
|
3438
|
+
...this.request,
|
|
3439
|
+
format: "arrow",
|
|
3440
|
+
signal
|
|
3441
|
+
});
|
|
3442
|
+
}
|
|
3443
|
+
};
|
|
3444
|
+
function getWGS84TileBounds(tile) {
|
|
3445
|
+
const scale = 2 ** tile.z;
|
|
3446
|
+
return {
|
|
3447
|
+
minimum: [tile.x / scale * 360 - 180, tileYToLatitude(tile.y + 1, scale)],
|
|
3448
|
+
maximum: [(tile.x + 1) / scale * 360 - 180, tileYToLatitude(tile.y, scale)]
|
|
3449
|
+
};
|
|
3450
|
+
}
|
|
3451
|
+
function tileYToLatitude(tileY, scale) {
|
|
3452
|
+
return Math.atan(Math.sinh(Math.PI * (1 - 2 * tileY / scale))) * 180 / Math.PI;
|
|
3453
|
+
}
|
|
3454
|
+
function inferVectorColumnRoles(columnNames) {
|
|
3455
|
+
return Object.fromEntries(columnNames.map((columnName) => [
|
|
3456
|
+
columnName,
|
|
3457
|
+
columnName.toLowerCase() === "geometry" ? "geometry" : "attribute"
|
|
3458
|
+
]));
|
|
3459
|
+
}
|
|
3460
|
+
function normalizeLayers(layers) {
|
|
3461
|
+
return Array.isArray(layers) ? layers : [layers];
|
|
3462
|
+
}
|
|
3463
|
+
function cloneAndFreezeLayers(layers) {
|
|
3464
|
+
return Array.isArray(layers) ? Object.freeze([...layers]) : layers;
|
|
3465
|
+
}
|
|
3466
|
+
async function waitForPromise2(promise, signal) {
|
|
3467
|
+
if (!signal) {
|
|
3468
|
+
return await promise;
|
|
3469
|
+
}
|
|
3470
|
+
throwIfAborted3(signal);
|
|
3471
|
+
return await new Promise((resolve, reject) => {
|
|
3472
|
+
const abort = () => reject(getAbortReason2(signal));
|
|
3473
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
3474
|
+
void promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
|
|
3475
|
+
});
|
|
3476
|
+
}
|
|
3477
|
+
function throwIfAborted3(signal) {
|
|
3478
|
+
if (signal == null ? void 0 : signal.aborted) {
|
|
3479
|
+
throw getAbortReason2(signal);
|
|
3480
|
+
}
|
|
3481
|
+
}
|
|
3482
|
+
function getAbortReason2(signal) {
|
|
3483
|
+
return signal.reason || new DOMException("Request aborted", "AbortError");
|
|
3484
|
+
}
|
|
3485
|
+
//# sourceMappingURL=index.cjs.map
|