@loaders.gl/sql 5.0.0-beta.10 → 5.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-table-source.js +1 -1
- package/dist/arrow-table-source.js.map +1 -1
- package/dist/duckdb-sql-source.js +1 -1
- package/dist/duckdb-table-query.d.ts.map +1 -1
- package/dist/duckdb-table-query.js +14 -23
- package/dist/duckdb-table-query.js.map +1 -1
- package/dist/index.cjs +206 -251
- package/dist/index.cjs.map +2 -2
- package/dist/snowflake-sql-source.js +1 -1
- package/package.json +2 -2
- package/src/arrow-table-source.ts +1 -1
- package/src/duckdb-table-query.ts +22 -28
package/dist/index.cjs
CHANGED
|
@@ -793,6 +793,7 @@ function createScanQueryMetadata(options) {
|
|
|
793
793
|
columns: Object.freeze(columns),
|
|
794
794
|
capabilities: queryCapabilities,
|
|
795
795
|
queryCapabilities,
|
|
796
|
+
...options.hasDefaultRowWindow ? { hasDefaultRowWindow: true } : {},
|
|
796
797
|
spatial: options.spatial ? Object.freeze({
|
|
797
798
|
...options.spatial,
|
|
798
799
|
bounds: options.spatial.bounds ? Object.freeze({
|
|
@@ -894,9 +895,6 @@ function validateTableQueryOffset(offset) {
|
|
|
894
895
|
}
|
|
895
896
|
|
|
896
897
|
// ../core/src/loader-utils/lib/scan-utils/table-scan-batch.ts
|
|
897
|
-
function makeTableScanBatch(table) {
|
|
898
|
-
return { ...table, batchType: "data", length: table.data.numRows };
|
|
899
|
-
}
|
|
900
898
|
async function* executeTableScanBatches(readBatches, options = {}, operators = {}) {
|
|
901
899
|
var _a, _b, _c, _d, _e;
|
|
902
900
|
validateTableQueryLimit(options.limit);
|
|
@@ -931,27 +929,28 @@ async function* executeTableScanBatches(readBatches, options = {}, operators = {
|
|
|
931
929
|
}
|
|
932
930
|
(_b = options.signal) == null ? void 0 : _b.throwIfAborted();
|
|
933
931
|
batchesRead++;
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
);
|
|
932
|
+
const rowCount = getScanBatchRowCount(batch);
|
|
933
|
+
rowsRead += rowCount;
|
|
934
|
+
if (options.predicate && !operators.filter && !("shape" in batch)) {
|
|
935
|
+
throw new Error("Native Arrow scans require a filter kernel for residual predicates.");
|
|
936
|
+
}
|
|
937
|
+
const filteredBatch = operators.filter ? await operators.filter(batch, options.predicate, options.signal) : "shape" in batch ? filterTableBatch(batch, options.predicate) : batch;
|
|
940
938
|
(_c = options.signal) == null ? void 0 : _c.throwIfAborted();
|
|
939
|
+
const filteredRowCount = getScanBatchRowCount(filteredBatch);
|
|
941
940
|
if (options.predicate) {
|
|
942
|
-
rowsTested +=
|
|
943
|
-
rowsRetained +=
|
|
941
|
+
rowsTested += rowCount;
|
|
942
|
+
rowsRetained += filteredRowCount;
|
|
944
943
|
}
|
|
945
|
-
if (skip >=
|
|
946
|
-
skip -=
|
|
944
|
+
if (skip >= filteredRowCount && (skip > 0 || options.skipEmptyBatches)) {
|
|
945
|
+
skip -= filteredRowCount;
|
|
947
946
|
continue;
|
|
948
947
|
}
|
|
949
|
-
const end = Math.min(
|
|
950
|
-
const window = skip || end < filteredBatch
|
|
948
|
+
const end = Math.min(filteredRowCount, skip + remaining);
|
|
949
|
+
const window = skip || end < filteredRowCount ? "shape" in filteredBatch ? sliceTableBatch(filteredBatch, skip, end) : filteredBatch.slice(skip, end) : filteredBatch;
|
|
951
950
|
skip = 0;
|
|
952
|
-
const output =
|
|
953
|
-
remaining -= window
|
|
954
|
-
rowsReturned += output
|
|
951
|
+
const output = operators.project ? operators.project(window, options.columns) : projectTableBatch(window, options.columns);
|
|
952
|
+
remaining -= getScanBatchRowCount(window);
|
|
953
|
+
rowsReturned += getScanBatchRowCount(output);
|
|
955
954
|
if (remaining === 0) earlyTerminationReason = "limit";
|
|
956
955
|
yield output;
|
|
957
956
|
if (remaining === 0) return;
|
|
@@ -989,6 +988,9 @@ async function* executeTableScanBatches(readBatches, options = {}, operators = {
|
|
|
989
988
|
);
|
|
990
989
|
}
|
|
991
990
|
}
|
|
991
|
+
function getScanBatchRowCount(batch) {
|
|
992
|
+
return "shape" in batch ? batch.length : batch.numRows;
|
|
993
|
+
}
|
|
992
994
|
function sliceTableBatch(batch, start, end) {
|
|
993
995
|
const length = end - start;
|
|
994
996
|
if (batch.shape === "object-row-table" || batch.shape === "array-row-table") {
|
|
@@ -1006,11 +1008,14 @@ function sliceTableBatch(batch, start, end) {
|
|
|
1006
1008
|
length
|
|
1007
1009
|
};
|
|
1008
1010
|
}
|
|
1009
|
-
if (batch.shape === "arrow-table") return { ...batch, data: batch.data.slice(start, end), length };
|
|
1010
1011
|
return { ...batch, features: batch.features.slice(start, end), length };
|
|
1011
1012
|
}
|
|
1012
1013
|
function projectTableBatch(batch, columns) {
|
|
1013
1014
|
if (columns === void 0) return batch;
|
|
1015
|
+
if (!("shape" in batch)) {
|
|
1016
|
+
const indices = columns.map((column) => batch.schema.fields.findIndex((field) => field.name === column)).filter((index) => index >= 0);
|
|
1017
|
+
return batch.selectAt(indices);
|
|
1018
|
+
}
|
|
1014
1019
|
const schema = projectTableSchema(batch, columns);
|
|
1015
1020
|
if (batch.shape === "object-row-table") {
|
|
1016
1021
|
return {
|
|
@@ -1026,8 +1031,6 @@ function projectTableBatch(batch, columns) {
|
|
|
1026
1031
|
data: Object.fromEntries(columns.map((column) => [column, batch.data[column]]))
|
|
1027
1032
|
};
|
|
1028
1033
|
}
|
|
1029
|
-
if (batch.shape === "arrow-table")
|
|
1030
|
-
return { ...batch, schema, data: batch.data.select([...columns]) };
|
|
1031
1034
|
return batch;
|
|
1032
1035
|
}
|
|
1033
1036
|
function projectTableSchema(batch, columns) {
|
|
@@ -2392,8 +2395,92 @@ function deserializeArrowDictionaryKeyType(keyType) {
|
|
|
2392
2395
|
}
|
|
2393
2396
|
}
|
|
2394
2397
|
|
|
2395
|
-
// ../core/src/schema-utils/lib/
|
|
2396
|
-
|
|
2398
|
+
// ../core/src/schema-utils/lib/schema/deduce-table-schema.ts
|
|
2399
|
+
function deduceTableSchema(table) {
|
|
2400
|
+
if (!("shape" in table)) return convertArrowToSchema(table.schema);
|
|
2401
|
+
switch (table.shape) {
|
|
2402
|
+
case "array-row-table":
|
|
2403
|
+
case "object-row-table":
|
|
2404
|
+
return deduceSchemaFromRows(table.data);
|
|
2405
|
+
case "geojson-table":
|
|
2406
|
+
return deduceSchemaFromGeoJSON(table.features);
|
|
2407
|
+
case "columnar-table":
|
|
2408
|
+
return deduceSchemaFromColumns(table.data);
|
|
2409
|
+
default:
|
|
2410
|
+
throw new Error("Deduce schema");
|
|
2411
|
+
}
|
|
2412
|
+
}
|
|
2413
|
+
function deduceSchemaFromColumns(columnarTable) {
|
|
2414
|
+
const fields = [];
|
|
2415
|
+
for (const [columnName, column] of Object.entries(columnarTable)) {
|
|
2416
|
+
const field = deduceFieldFromColumn(column, columnName);
|
|
2417
|
+
fields.push(field);
|
|
2418
|
+
}
|
|
2419
|
+
return { fields, metadata: {} };
|
|
2420
|
+
}
|
|
2421
|
+
function deduceSchemaFromRows(rowTable) {
|
|
2422
|
+
if (!rowTable.length) {
|
|
2423
|
+
throw new Error("deduce from empty table");
|
|
2424
|
+
}
|
|
2425
|
+
const fields = [];
|
|
2426
|
+
const row0 = rowTable[0];
|
|
2427
|
+
for (const [columnName, value] of Object.entries(row0)) {
|
|
2428
|
+
const name = Array.isArray(row0) ? `column${Number(columnName) + 1}` : columnName;
|
|
2429
|
+
fields.push(deduceFieldFromValue(value, name));
|
|
2430
|
+
}
|
|
2431
|
+
return { fields, metadata: {} };
|
|
2432
|
+
}
|
|
2433
|
+
function deduceSchemaFromGeoJSON(features) {
|
|
2434
|
+
if (!features.length) {
|
|
2435
|
+
throw new Error("deduce from empty table");
|
|
2436
|
+
}
|
|
2437
|
+
const fields = [];
|
|
2438
|
+
const row0 = features[0].properties || {};
|
|
2439
|
+
for (const [columnName, value] of Object.entries(row0)) {
|
|
2440
|
+
fields.push(deduceFieldFromValue(value, columnName));
|
|
2441
|
+
}
|
|
2442
|
+
return { fields, metadata: {} };
|
|
2443
|
+
}
|
|
2444
|
+
function deduceFieldFromColumn(column, name) {
|
|
2445
|
+
if (ArrayBuffer.isView(column)) {
|
|
2446
|
+
const type = getDataTypeFromArray(column);
|
|
2447
|
+
return {
|
|
2448
|
+
name,
|
|
2449
|
+
type: type.type || "null",
|
|
2450
|
+
nullable: type.nullable
|
|
2451
|
+
// metadata: {}
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
if (Array.isArray(column) && column.length > 0) {
|
|
2455
|
+
const value = column[0];
|
|
2456
|
+
const type = getDataTypeFromValue(value);
|
|
2457
|
+
return {
|
|
2458
|
+
name,
|
|
2459
|
+
type,
|
|
2460
|
+
nullable: true
|
|
2461
|
+
// metadata: {},
|
|
2462
|
+
};
|
|
2463
|
+
}
|
|
2464
|
+
throw new Error("empty table");
|
|
2465
|
+
}
|
|
2466
|
+
function deduceFieldFromValue(value, name) {
|
|
2467
|
+
const type = getDataTypeFromValue(value);
|
|
2468
|
+
return {
|
|
2469
|
+
name,
|
|
2470
|
+
type,
|
|
2471
|
+
nullable: true
|
|
2472
|
+
// metadata: {}
|
|
2473
|
+
};
|
|
2474
|
+
}
|
|
2475
|
+
|
|
2476
|
+
// ../core/src/schema-utils/lib/arrow/arrow-metadata.ts
|
|
2477
|
+
var arrow3 = __toESM(require("apache-arrow"), 1);
|
|
2478
|
+
function snapshotArrowSchema(schema) {
|
|
2479
|
+
return deserializeArrowSchema(serializeArrowSchema(schema));
|
|
2480
|
+
}
|
|
2481
|
+
function makeArrowMetadataBatch(schema) {
|
|
2482
|
+
return new arrow3.RecordBatch(snapshotArrowSchema(schema), void 0);
|
|
2483
|
+
}
|
|
2397
2484
|
|
|
2398
2485
|
// ../core/src/schema-utils/lib/table/tables/table-accessors.ts
|
|
2399
2486
|
function getTableLength(table) {
|
|
@@ -2404,8 +2491,6 @@ function getTableLength(table) {
|
|
|
2404
2491
|
return table.data.length;
|
|
2405
2492
|
case "geojson-table":
|
|
2406
2493
|
return table.features.length;
|
|
2407
|
-
case "arrow-table":
|
|
2408
|
-
return getTableLength(table.data);
|
|
2409
2494
|
case "columnar-table":
|
|
2410
2495
|
for (const column of Object.values(table.data)) {
|
|
2411
2496
|
return column.length || 0;
|
|
@@ -2432,8 +2517,6 @@ function getTableNumCols(table) {
|
|
|
2432
2517
|
return Object.keys(table.features[0]).length;
|
|
2433
2518
|
case "columnar-table":
|
|
2434
2519
|
return Object.keys(table.data).length;
|
|
2435
|
-
case "arrow-table":
|
|
2436
|
-
return getTableNumCols(table.data);
|
|
2437
2520
|
default:
|
|
2438
2521
|
throw new Error("table");
|
|
2439
2522
|
}
|
|
@@ -2452,8 +2535,6 @@ function getTableCell(table, rowIndex, columnName) {
|
|
|
2452
2535
|
case "columnar-table":
|
|
2453
2536
|
const column = table.data[columnName];
|
|
2454
2537
|
return column[rowIndex];
|
|
2455
|
-
case "arrow-table":
|
|
2456
|
-
return getTableCell(table.data, rowIndex, columnName);
|
|
2457
2538
|
default:
|
|
2458
2539
|
throw new Error("todo");
|
|
2459
2540
|
}
|
|
@@ -2474,8 +2555,6 @@ function getTableCellAt(table, rowIndex, columnIndex) {
|
|
|
2474
2555
|
const columnName3 = getTableColumnName(table, columnIndex);
|
|
2475
2556
|
const column = table.data[columnName3];
|
|
2476
2557
|
return column[rowIndex];
|
|
2477
|
-
case "arrow-table":
|
|
2478
|
-
return getTableCellAt(table.data, rowIndex, columnIndex);
|
|
2479
2558
|
default:
|
|
2480
2559
|
throw new Error("todo");
|
|
2481
2560
|
}
|
|
@@ -2538,8 +2617,6 @@ function getTableRowAsObject(table, rowIndex, target, copy) {
|
|
|
2538
2617
|
}
|
|
2539
2618
|
return objectRow;
|
|
2540
2619
|
}
|
|
2541
|
-
case "arrow-table":
|
|
2542
|
-
return getTableRowAsObject(table.data, rowIndex, target, copy);
|
|
2543
2620
|
default:
|
|
2544
2621
|
throw new Error("shape");
|
|
2545
2622
|
}
|
|
@@ -2590,196 +2667,34 @@ function getTableRowAsArray(table, rowIndex, target, copy) {
|
|
|
2590
2667
|
}
|
|
2591
2668
|
return arrayRow;
|
|
2592
2669
|
}
|
|
2593
|
-
case "arrow-table":
|
|
2594
|
-
return getTableRowAsArray(table.data, rowIndex, target, copy);
|
|
2595
2670
|
default:
|
|
2596
2671
|
throw new Error("shape");
|
|
2597
2672
|
}
|
|
2598
2673
|
}
|
|
2599
2674
|
|
|
2600
|
-
// ../core/src/schema-utils/lib/schema/deduce-table-schema.ts
|
|
2601
|
-
function deduceTableSchema(table) {
|
|
2602
|
-
switch (table.shape) {
|
|
2603
|
-
case "array-row-table":
|
|
2604
|
-
case "object-row-table":
|
|
2605
|
-
return deduceSchemaFromRows(table.data);
|
|
2606
|
-
case "geojson-table":
|
|
2607
|
-
return deduceSchemaFromGeoJSON(table.features);
|
|
2608
|
-
case "columnar-table":
|
|
2609
|
-
return deduceSchemaFromColumns(table.data);
|
|
2610
|
-
case "arrow-table":
|
|
2611
|
-
default:
|
|
2612
|
-
throw new Error("Deduce schema");
|
|
2613
|
-
}
|
|
2614
|
-
}
|
|
2615
|
-
function deduceSchemaFromColumns(columnarTable) {
|
|
2616
|
-
const fields = [];
|
|
2617
|
-
for (const [columnName, column] of Object.entries(columnarTable)) {
|
|
2618
|
-
const field = deduceFieldFromColumn(column, columnName);
|
|
2619
|
-
fields.push(field);
|
|
2620
|
-
}
|
|
2621
|
-
return { fields, metadata: {} };
|
|
2622
|
-
}
|
|
2623
|
-
function deduceSchemaFromRows(rowTable) {
|
|
2624
|
-
if (!rowTable.length) {
|
|
2625
|
-
throw new Error("deduce from empty table");
|
|
2626
|
-
}
|
|
2627
|
-
const fields = [];
|
|
2628
|
-
const row0 = rowTable[0];
|
|
2629
|
-
for (const [columnName, value] of Object.entries(row0)) {
|
|
2630
|
-
const name = Array.isArray(row0) ? `column${Number(columnName) + 1}` : columnName;
|
|
2631
|
-
fields.push(deduceFieldFromValue(value, name));
|
|
2632
|
-
}
|
|
2633
|
-
return { fields, metadata: {} };
|
|
2634
|
-
}
|
|
2635
|
-
function deduceSchemaFromGeoJSON(features) {
|
|
2636
|
-
if (!features.length) {
|
|
2637
|
-
throw new Error("deduce from empty table");
|
|
2638
|
-
}
|
|
2639
|
-
const fields = [];
|
|
2640
|
-
const row0 = features[0].properties || {};
|
|
2641
|
-
for (const [columnName, value] of Object.entries(row0)) {
|
|
2642
|
-
fields.push(deduceFieldFromValue(value, columnName));
|
|
2643
|
-
}
|
|
2644
|
-
return { fields, metadata: {} };
|
|
2645
|
-
}
|
|
2646
|
-
function deduceFieldFromColumn(column, name) {
|
|
2647
|
-
if (ArrayBuffer.isView(column)) {
|
|
2648
|
-
const type = getDataTypeFromArray(column);
|
|
2649
|
-
return {
|
|
2650
|
-
name,
|
|
2651
|
-
type: type.type || "null",
|
|
2652
|
-
nullable: type.nullable
|
|
2653
|
-
// metadata: {}
|
|
2654
|
-
};
|
|
2655
|
-
}
|
|
2656
|
-
if (Array.isArray(column) && column.length > 0) {
|
|
2657
|
-
const value = column[0];
|
|
2658
|
-
const type = getDataTypeFromValue(value);
|
|
2659
|
-
return {
|
|
2660
|
-
name,
|
|
2661
|
-
type,
|
|
2662
|
-
nullable: true
|
|
2663
|
-
// metadata: {},
|
|
2664
|
-
};
|
|
2665
|
-
}
|
|
2666
|
-
throw new Error("empty table");
|
|
2667
|
-
}
|
|
2668
|
-
function deduceFieldFromValue(value, name) {
|
|
2669
|
-
const type = getDataTypeFromValue(value);
|
|
2670
|
-
return {
|
|
2671
|
-
name,
|
|
2672
|
-
type,
|
|
2673
|
-
nullable: true
|
|
2674
|
-
// metadata: {}
|
|
2675
|
-
};
|
|
2676
|
-
}
|
|
2677
|
-
|
|
2678
2675
|
// ../core/src/schema-utils/lib/table/tables/table-column.ts
|
|
2679
2676
|
function makeColumnFromField(field, length) {
|
|
2680
2677
|
const ArrayType = getArrayTypeFromDataType(field.type, field.nullable);
|
|
2681
2678
|
return new ArrayType(length);
|
|
2682
2679
|
}
|
|
2683
2680
|
|
|
2684
|
-
// ../core/src/schema-utils/lib/table/tables/convert-table.ts
|
|
2685
|
-
|
|
2686
|
-
if (shape === "arrow") return convertTableToArrow(table);
|
|
2687
|
-
if (!("shape" in table)) return convertArrowToTable(table, shape);
|
|
2688
|
-
switch (shape) {
|
|
2689
|
-
case "object-row-table":
|
|
2690
|
-
return convertToObjectRowTable(table);
|
|
2691
|
-
case "array-row-table":
|
|
2692
|
-
return convertToArrayRowTable(table);
|
|
2693
|
-
case "columnar-table":
|
|
2694
|
-
return convertToColumnarTable(table);
|
|
2695
|
-
default:
|
|
2696
|
-
throw new Error(shape);
|
|
2697
|
-
}
|
|
2698
|
-
}
|
|
2699
|
-
function convertToColumnarTable(table) {
|
|
2700
|
-
const schema = table.schema || deduceTableSchema(table);
|
|
2701
|
-
const fields = schema.fields;
|
|
2702
|
-
if (table.shape === "columnar-table") {
|
|
2703
|
-
return { ...table, schema };
|
|
2704
|
-
}
|
|
2705
|
-
const length = getTableLength(table);
|
|
2706
|
-
const columns = {};
|
|
2707
|
-
for (const field of fields) {
|
|
2708
|
-
const column = makeColumnFromField(field, length);
|
|
2709
|
-
columns[field.name] = column;
|
|
2710
|
-
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
2711
|
-
column[rowIndex] = getTableCell(table, rowIndex, field.name);
|
|
2712
|
-
}
|
|
2713
|
-
}
|
|
2714
|
-
return {
|
|
2715
|
-
shape: "columnar-table",
|
|
2716
|
-
schema,
|
|
2717
|
-
data: columns
|
|
2718
|
-
};
|
|
2719
|
-
}
|
|
2720
|
-
function convertToArrayRowTable(table) {
|
|
2721
|
-
if (table.shape === "array-row-table") {
|
|
2722
|
-
return table;
|
|
2723
|
-
}
|
|
2724
|
-
const length = getTableLength(table);
|
|
2725
|
-
const data = new Array(length);
|
|
2726
|
-
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
2727
|
-
data[rowIndex] = getTableRowAsArray(table, rowIndex);
|
|
2728
|
-
}
|
|
2729
|
-
return {
|
|
2730
|
-
shape: "array-row-table",
|
|
2731
|
-
schema: table.schema,
|
|
2732
|
-
data
|
|
2733
|
-
};
|
|
2734
|
-
}
|
|
2735
|
-
function convertToObjectRowTable(table) {
|
|
2736
|
-
if (table.shape === "object-row-table") {
|
|
2737
|
-
return table;
|
|
2738
|
-
}
|
|
2739
|
-
const length = getTableLength(table);
|
|
2740
|
-
const data = new Array(length);
|
|
2741
|
-
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
2742
|
-
data[rowIndex] = getTableRowAsObject(table, rowIndex);
|
|
2743
|
-
}
|
|
2744
|
-
return {
|
|
2745
|
-
shape: "object-row-table",
|
|
2746
|
-
schema: table.schema,
|
|
2747
|
-
data
|
|
2748
|
-
};
|
|
2749
|
-
}
|
|
2681
|
+
// ../core/src/schema-utils/lib/table/tables/convert-arrow-table.ts
|
|
2682
|
+
var arrow5 = __toESM(require("apache-arrow"), 1);
|
|
2750
2683
|
|
|
2751
2684
|
// ../core/src/schema-utils/lib/table/batches/make-arrow-batch-iterator.ts
|
|
2752
|
-
var
|
|
2753
|
-
function*
|
|
2754
|
-
if (batchSize !== void 0 && (!Number.isSafeInteger(batchSize) || batchSize < 1)) {
|
|
2755
|
-
throw new RangeError("batchSize must be a positive integer");
|
|
2756
|
-
}
|
|
2757
|
-
for (const batch of table.batches) {
|
|
2758
|
-
if (batchSize === void 0 || batch.numRows <= batchSize) {
|
|
2759
|
-
yield batch;
|
|
2760
|
-
} else {
|
|
2761
|
-
for (let offset = 0; offset < batch.numRows; offset += batchSize) {
|
|
2762
|
-
yield batch.slice(offset, offset + batchSize);
|
|
2763
|
-
}
|
|
2764
|
-
}
|
|
2765
|
-
}
|
|
2766
|
-
}
|
|
2767
|
-
function* makeWrappedTableRecordBatchIterator(table, options) {
|
|
2768
|
-
if (table.shape === "arrow-table") {
|
|
2769
|
-
yield* sliceNativeArrowBatches(table.data, options == null ? void 0 : options.batchSize);
|
|
2770
|
-
return;
|
|
2771
|
-
}
|
|
2685
|
+
var arrow4 = __toESM(require("apache-arrow"), 1);
|
|
2686
|
+
function* makePortableTableRecordBatchIterator(table, options) {
|
|
2772
2687
|
const arrowSchema = convertSchemaToArrow(table.schema);
|
|
2773
2688
|
const length = getTableLength(table);
|
|
2774
2689
|
const numColumns = getTableNumCols(table);
|
|
2775
2690
|
const batchSize = (options == null ? void 0 : options.batchSize) || length;
|
|
2776
2691
|
const builders = arrowSchema == null ? void 0 : arrowSchema.fields.map(
|
|
2777
|
-
(arrowField) =>
|
|
2692
|
+
(arrowField) => arrow4.makeBuilder({
|
|
2778
2693
|
type: arrowField.type,
|
|
2779
2694
|
nullValues: [null, void 0]
|
|
2780
2695
|
})
|
|
2781
2696
|
);
|
|
2782
|
-
const structField = new
|
|
2697
|
+
const structField = new arrow4.Struct(arrowSchema.fields);
|
|
2783
2698
|
let batchLength = 0;
|
|
2784
2699
|
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
2785
2700
|
for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
|
|
@@ -2790,15 +2705,15 @@ function* makeWrappedTableRecordBatchIterator(table, options) {
|
|
|
2790
2705
|
batchLength++;
|
|
2791
2706
|
if (batchLength >= batchSize) {
|
|
2792
2707
|
const datas = builders.map((builder) => builder.flush());
|
|
2793
|
-
const structData = new
|
|
2794
|
-
yield new
|
|
2708
|
+
const structData = new arrow4.Data(structField, 0, batchLength, 0, void 0, datas);
|
|
2709
|
+
yield new arrow4.RecordBatch(arrowSchema, structData);
|
|
2795
2710
|
batchLength = 0;
|
|
2796
2711
|
}
|
|
2797
2712
|
}
|
|
2798
2713
|
if (batchLength > 0) {
|
|
2799
2714
|
const datas = builders.map((builder) => builder.flush());
|
|
2800
|
-
const structData = new
|
|
2801
|
-
yield new
|
|
2715
|
+
const structData = new arrow4.Data(structField, 0, batchLength, 0, void 0, datas);
|
|
2716
|
+
yield new arrow4.RecordBatch(arrowSchema, structData);
|
|
2802
2717
|
batchLength = 0;
|
|
2803
2718
|
}
|
|
2804
2719
|
builders.map((builder) => builder.finish());
|
|
@@ -2807,19 +2722,10 @@ function* makeWrappedTableRecordBatchIterator(table, options) {
|
|
|
2807
2722
|
// ../core/src/schema-utils/lib/table/tables/convert-arrow-table.ts
|
|
2808
2723
|
function convertTableToArrow(table, options) {
|
|
2809
2724
|
if (!("shape" in table)) return table;
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
// TODO - optimized implementation is possible
|
|
2815
|
-
// return convertColumnarTableToArrow(table, options);
|
|
2816
|
-
// fall through
|
|
2817
|
-
default:
|
|
2818
|
-
const schema = table.schema ?? deduceTableSchema(table);
|
|
2819
|
-
return new arrow4.Table(convertSchemaToArrow(schema), [
|
|
2820
|
-
...makeWrappedTableRecordBatchIterator({ ...table, schema }, options)
|
|
2821
|
-
]);
|
|
2822
|
-
}
|
|
2725
|
+
const schema = table.schema ?? deduceTableSchema(table);
|
|
2726
|
+
return new arrow5.Table(convertSchemaToArrow(schema), [
|
|
2727
|
+
...makePortableTableRecordBatchIterator({ ...table, schema }, options)
|
|
2728
|
+
]);
|
|
2823
2729
|
}
|
|
2824
2730
|
function convertArrowToTable(arrowTable, shape) {
|
|
2825
2731
|
switch (shape) {
|
|
@@ -2875,13 +2781,71 @@ function convertArrowToGeoJSONTable(arrowTable) {
|
|
|
2875
2781
|
};
|
|
2876
2782
|
}
|
|
2877
2783
|
|
|
2878
|
-
// ../core/src/schema-utils/lib/
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2784
|
+
// ../core/src/schema-utils/lib/table/tables/convert-table.ts
|
|
2785
|
+
function convertTable(table, shape) {
|
|
2786
|
+
if (shape === "arrow") return convertTableToArrow(table);
|
|
2787
|
+
if (!("shape" in table)) return convertArrowToTable(table, shape);
|
|
2788
|
+
switch (shape) {
|
|
2789
|
+
case "object-row-table":
|
|
2790
|
+
return convertToObjectRowTable(table);
|
|
2791
|
+
case "array-row-table":
|
|
2792
|
+
return convertToArrayRowTable(table);
|
|
2793
|
+
case "columnar-table":
|
|
2794
|
+
return convertToColumnarTable(table);
|
|
2795
|
+
default:
|
|
2796
|
+
throw new Error(shape);
|
|
2797
|
+
}
|
|
2882
2798
|
}
|
|
2883
|
-
function
|
|
2884
|
-
|
|
2799
|
+
function convertToColumnarTable(table) {
|
|
2800
|
+
const schema = table.schema || deduceTableSchema(table);
|
|
2801
|
+
const fields = schema.fields;
|
|
2802
|
+
if (table.shape === "columnar-table") {
|
|
2803
|
+
return { ...table, schema };
|
|
2804
|
+
}
|
|
2805
|
+
const length = getTableLength(table);
|
|
2806
|
+
const columns = {};
|
|
2807
|
+
for (const field of fields) {
|
|
2808
|
+
const column = makeColumnFromField(field, length);
|
|
2809
|
+
columns[field.name] = column;
|
|
2810
|
+
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
2811
|
+
column[rowIndex] = getTableCell(table, rowIndex, field.name);
|
|
2812
|
+
}
|
|
2813
|
+
}
|
|
2814
|
+
return {
|
|
2815
|
+
shape: "columnar-table",
|
|
2816
|
+
schema,
|
|
2817
|
+
data: columns
|
|
2818
|
+
};
|
|
2819
|
+
}
|
|
2820
|
+
function convertToArrayRowTable(table) {
|
|
2821
|
+
if (table.shape === "array-row-table") {
|
|
2822
|
+
return table;
|
|
2823
|
+
}
|
|
2824
|
+
const length = getTableLength(table);
|
|
2825
|
+
const data = new Array(length);
|
|
2826
|
+
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
2827
|
+
data[rowIndex] = getTableRowAsArray(table, rowIndex);
|
|
2828
|
+
}
|
|
2829
|
+
return {
|
|
2830
|
+
shape: "array-row-table",
|
|
2831
|
+
schema: table.schema,
|
|
2832
|
+
data
|
|
2833
|
+
};
|
|
2834
|
+
}
|
|
2835
|
+
function convertToObjectRowTable(table) {
|
|
2836
|
+
if (table.shape === "object-row-table") {
|
|
2837
|
+
return table;
|
|
2838
|
+
}
|
|
2839
|
+
const length = getTableLength(table);
|
|
2840
|
+
const data = new Array(length);
|
|
2841
|
+
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
2842
|
+
data[rowIndex] = getTableRowAsObject(table, rowIndex);
|
|
2843
|
+
}
|
|
2844
|
+
return {
|
|
2845
|
+
shape: "object-row-table",
|
|
2846
|
+
schema: table.schema,
|
|
2847
|
+
data
|
|
2848
|
+
};
|
|
2885
2849
|
}
|
|
2886
2850
|
|
|
2887
2851
|
// ../core/src/schema-utils/deprecated.ts
|
|
@@ -3623,7 +3587,7 @@ var ArrowTableSource = class extends DataSource {
|
|
|
3623
3587
|
throwIfAborted2(options.signal);
|
|
3624
3588
|
const schema = convertArrowToSchema(this.data.schema);
|
|
3625
3589
|
return createScanQueryMetadata({
|
|
3626
|
-
sourceType: "arrow
|
|
3590
|
+
sourceType: "arrow",
|
|
3627
3591
|
queryType: "table",
|
|
3628
3592
|
execution: { status: "supported", method: "read" },
|
|
3629
3593
|
schema,
|
|
@@ -3746,17 +3710,11 @@ function makeDuckDBTableQuerySource(source, table, sourceSchema) {
|
|
|
3746
3710
|
return Number(value);
|
|
3747
3711
|
return value;
|
|
3748
3712
|
}), field.type));
|
|
3749
|
-
yield
|
|
3750
|
-
|
|
3751
|
-
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
type: new import_apache_arrow.Struct(arrowSchema.fields),
|
|
3755
|
-
length: rows.length,
|
|
3756
|
-
children: vectors.map((vector) => vector.data[0])
|
|
3757
|
-
}))
|
|
3758
|
-
])
|
|
3759
|
-
});
|
|
3713
|
+
yield new import_apache_arrow.RecordBatch(arrowSchema, (0, import_apache_arrow.makeData)({
|
|
3714
|
+
type: new import_apache_arrow.Struct(arrowSchema.fields),
|
|
3715
|
+
length: rows.length,
|
|
3716
|
+
children: vectors.map((vector) => vector.data[0])
|
|
3717
|
+
}));
|
|
3760
3718
|
}
|
|
3761
3719
|
function plan(query) {
|
|
3762
3720
|
var _a;
|
|
@@ -3804,9 +3762,7 @@ function makeDuckDBTableQuerySource(source, table, sourceSchema) {
|
|
|
3804
3762
|
schema: outputSchema,
|
|
3805
3763
|
/** Executes the planned SQL when the consumer starts reading. */
|
|
3806
3764
|
async *read(options) {
|
|
3807
|
-
|
|
3808
|
-
yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
|
|
3809
|
-
}
|
|
3765
|
+
yield* readStatement(sql, outputSchema, options == null ? void 0 : options.signal);
|
|
3810
3766
|
}
|
|
3811
3767
|
};
|
|
3812
3768
|
}
|
|
@@ -3833,19 +3789,18 @@ function makeDuckDBTableQuerySource(source, table, sourceSchema) {
|
|
|
3833
3789
|
yield* accepted.read({ signal });
|
|
3834
3790
|
return;
|
|
3835
3791
|
}
|
|
3836
|
-
for await (const batch of executeTableScanBatches(
|
|
3792
|
+
for await (const batch of executeTableScanBatches(async function* (readSignal) {
|
|
3793
|
+
for await (const batch2 of readStatement(`SELECT * FROM ${tableName}${order}`, schema, readSignal)) {
|
|
3794
|
+
yield new import_apache_arrow.Table(batch2.schema, [batch2]);
|
|
3795
|
+
}
|
|
3796
|
+
}, options, {
|
|
3837
3797
|
filter: (batch2, predicate2, filterSignal) => {
|
|
3838
3798
|
if (!predicate2)
|
|
3839
3799
|
return batch2;
|
|
3840
|
-
|
|
3841
|
-
return makeTableScanBatch({
|
|
3842
|
-
...arrowBatch,
|
|
3843
|
-
data: filterArrowTable(arrowBatch.data, predicate2, { signal: filterSignal })
|
|
3844
|
-
});
|
|
3800
|
+
return filterArrowTable(batch2, predicate2, { signal: filterSignal });
|
|
3845
3801
|
}
|
|
3846
3802
|
})) {
|
|
3847
|
-
|
|
3848
|
-
yield* data.batches.length ? data.batches : [makeArrowMetadataBatch(data.schema)];
|
|
3803
|
+
yield* batch.batches.length ? batch.batches : [makeArrowMetadataBatch(batch.schema)];
|
|
3849
3804
|
}
|
|
3850
3805
|
}
|
|
3851
3806
|
};
|
|
@@ -3884,7 +3839,7 @@ function compilePredicate(predicate, schema) {
|
|
|
3884
3839
|
}
|
|
3885
3840
|
|
|
3886
3841
|
// dist/duckdb-sql-source.js
|
|
3887
|
-
var VERSION2 = true ? "5.0.0-beta.
|
|
3842
|
+
var VERSION2 = true ? "5.0.0-beta.11" : "latest";
|
|
3888
3843
|
var DuckDBSQLSource = {
|
|
3889
3844
|
dataType: null,
|
|
3890
3845
|
batchType: null,
|
|
@@ -4221,7 +4176,7 @@ function getDuckDBDatabasePath(url, options) {
|
|
|
4221
4176
|
}
|
|
4222
4177
|
|
|
4223
4178
|
// dist/snowflake-sql-source.js
|
|
4224
|
-
var VERSION3 = true ? "5.0.0-beta.
|
|
4179
|
+
var VERSION3 = true ? "5.0.0-beta.11" : "latest";
|
|
4225
4180
|
var SnowflakeSQLSource = {
|
|
4226
4181
|
dataType: null,
|
|
4227
4182
|
batchType: null,
|
|
@@ -4427,9 +4382,9 @@ function getSnowflakeBaseUrl(url, options) {
|
|
|
4427
4382
|
}
|
|
4428
4383
|
|
|
4429
4384
|
// ../core/src/scan/parse-sql-predicate.ts
|
|
4430
|
-
var
|
|
4385
|
+
var import_parser = require("@math.gl/sql/parser");
|
|
4431
4386
|
function parseSQLPredicate(source, options) {
|
|
4432
|
-
const predicate = options ? (0,
|
|
4387
|
+
const predicate = options ? (0, import_parser.parseSQLPredicate)(source, options) : (0, import_parser.parseSQLPredicate)(source);
|
|
4433
4388
|
validateSQLPredicate(predicate);
|
|
4434
4389
|
return predicate;
|
|
4435
4390
|
}
|