@loaders.gl/sql 5.0.0-beta.1 → 5.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/arrow-table-source.d.ts +5 -5
- package/dist/arrow-table-source.d.ts.map +1 -1
- package/dist/arrow-table-source.js +10 -6
- package/dist/arrow-table-source.js.map +1 -1
- package/dist/duckdb-sql-source.d.ts +21 -26
- package/dist/duckdb-sql-source.d.ts.map +1 -1
- package/dist/duckdb-sql-source.js +24 -4
- package/dist/duckdb-sql-source.js.map +1 -1
- package/dist/duckdb-table-query.d.ts +9 -0
- package/dist/duckdb-table-query.d.ts.map +1 -0
- package/dist/duckdb-table-query.js +233 -0
- package/dist/duckdb-table-query.js.map +1 -0
- package/dist/index.cjs +1480 -1204
- package/dist/index.cjs.map +3 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/snowflake-sql-source.d.ts +6 -23
- package/dist/snowflake-sql-source.d.ts.map +1 -1
- package/dist/snowflake-sql-source.js +2 -2
- package/dist/snowflake-sql-source.js.map +1 -1
- package/dist/sql-source.d.ts +17 -4
- package/dist/sql-source.d.ts.map +1 -1
- package/dist/sql-source.js +33 -1
- package/dist/sql-source.js.map +1 -1
- package/dist/sql-types.d.ts +3 -2
- package/dist/sql-types.d.ts.map +1 -1
- package/package.json +5 -4
- package/src/arrow-table-source.ts +12 -11
- package/src/duckdb-sql-source.ts +40 -7
- package/src/duckdb-table-query.ts +271 -0
- package/src/index.ts +3 -0
- package/src/snowflake-sql-source.ts +1 -1
- package/src/sql-source.ts +43 -4
- package/src/sql-types.ts +3 -2
package/dist/index.cjs
CHANGED
|
@@ -69,6 +69,36 @@ function createLog() {
|
|
|
69
69
|
return log2;
|
|
70
70
|
}
|
|
71
71
|
var log = createLog();
|
|
72
|
+
var NullLog = class {
|
|
73
|
+
/** Discards debug messages. */
|
|
74
|
+
log() {
|
|
75
|
+
return ignoreMessage;
|
|
76
|
+
}
|
|
77
|
+
/** Discards informational messages. */
|
|
78
|
+
info() {
|
|
79
|
+
return ignoreMessage;
|
|
80
|
+
}
|
|
81
|
+
/** Discards messages marked to be emitted once. */
|
|
82
|
+
once() {
|
|
83
|
+
return ignoreMessage;
|
|
84
|
+
}
|
|
85
|
+
/** Discards warnings. */
|
|
86
|
+
warn() {
|
|
87
|
+
return ignoreMessage;
|
|
88
|
+
}
|
|
89
|
+
/** Discards errors. */
|
|
90
|
+
error() {
|
|
91
|
+
return ignoreMessage;
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
function ignoreMessage() {
|
|
95
|
+
}
|
|
96
|
+
var nullLog = new NullLog();
|
|
97
|
+
function getLoaderLog(options) {
|
|
98
|
+
var _a;
|
|
99
|
+
const logger = (_a = options == null ? void 0 : options.core) == null ? void 0 : _a.log;
|
|
100
|
+
return logger === null ? nullLog : logger || log;
|
|
101
|
+
}
|
|
72
102
|
|
|
73
103
|
// ../core/src/loader-utils/lib/javascript-utils/is-type.ts
|
|
74
104
|
var isObject = (value) => value !== null && typeof value === "object";
|
|
@@ -226,7 +256,7 @@ function createAuthenticatedFetch(options) {
|
|
|
226
256
|
}
|
|
227
257
|
function getAuthenticatedFetch(options = {}) {
|
|
228
258
|
var _a, _b, _c;
|
|
229
|
-
const fetchOption =
|
|
259
|
+
const fetchOption = (_a = options.core) == null ? void 0 : _a.fetch;
|
|
230
260
|
let fetchFunction;
|
|
231
261
|
if (typeof fetchOption === "function") {
|
|
232
262
|
fetchFunction = fetchOption;
|
|
@@ -402,6 +432,8 @@ var DataSource = class _DataSource {
|
|
|
402
432
|
options;
|
|
403
433
|
data;
|
|
404
434
|
url;
|
|
435
|
+
/** Stable query-visible name, resolved without fetching source data. */
|
|
436
|
+
name;
|
|
405
437
|
/** Shared parser options derived from the source options, excluding source-only core controls. */
|
|
406
438
|
loadOptions;
|
|
407
439
|
/** A resolved fetch function extracted from the shared core options. */
|
|
@@ -418,6 +450,7 @@ var DataSource = class _DataSource {
|
|
|
418
450
|
);
|
|
419
451
|
this.data = data;
|
|
420
452
|
this.url = typeof data === "string" ? resolvePath(data) : "";
|
|
453
|
+
this.name = getDataSourceName(data, this.options);
|
|
421
454
|
const loadOptions = resolveAuthenticationOptions(getSourceLoaderOptions(this.options));
|
|
422
455
|
this.loadOptions = loadOptions;
|
|
423
456
|
const fetch2 = getFetchFunction(loadOptions);
|
|
@@ -426,6 +459,9 @@ var DataSource = class _DataSource {
|
|
|
426
459
|
this.fetch = fetch2;
|
|
427
460
|
}
|
|
428
461
|
setProps(options) {
|
|
462
|
+
var _a;
|
|
463
|
+
if (((_a = options.core) == null ? void 0 : _a.name) !== void 0 && options.core.name !== this.name)
|
|
464
|
+
throw new Error("Data source names are fixed at creation.");
|
|
429
465
|
const mergedOptions = mergeOptions(this.options, options);
|
|
430
466
|
const loadOptions = resolveAuthenticationOptions(getSourceLoaderOptions(mergedOptions));
|
|
431
467
|
this.options = mergedOptions;
|
|
@@ -456,7 +492,7 @@ var DataSource = class _DataSource {
|
|
|
456
492
|
if (callback) {
|
|
457
493
|
callback(normalizedError, this);
|
|
458
494
|
} else {
|
|
459
|
-
|
|
495
|
+
getLoaderLog(this.options).warn(`${this.constructor.name}: ${normalizedError.message}`)();
|
|
460
496
|
}
|
|
461
497
|
return normalizedError;
|
|
462
498
|
}
|
|
@@ -475,23 +511,13 @@ function getSourceLoaderOptions(options) {
|
|
|
475
511
|
if (options == null ? void 0 : options.core) {
|
|
476
512
|
loadOptions.core = { ...options.core };
|
|
477
513
|
const parserCore = loadOptions.core;
|
|
478
|
-
for (const key of ["type", "attributions", "loaders", "onError", "loadOptions"]) {
|
|
514
|
+
for (const key of ["type", "name", "attributions", "loaders", "onError", "loadOptions"]) {
|
|
479
515
|
delete parserCore[key];
|
|
480
516
|
}
|
|
481
517
|
if (Object.keys(parserCore).length === 0) {
|
|
482
518
|
delete loadOptions.core;
|
|
483
519
|
}
|
|
484
520
|
}
|
|
485
|
-
const topLevelBaseUri = typeof loadOptions.baseUri === "string" ? loadOptions.baseUri : void 0;
|
|
486
|
-
const topLevelBaseUrl = typeof loadOptions.baseUrl === "string" ? loadOptions.baseUrl : void 0;
|
|
487
|
-
if (topLevelBaseUri !== void 0 || topLevelBaseUrl !== void 0) {
|
|
488
|
-
loadOptions.core ||= {};
|
|
489
|
-
if (loadOptions.core.baseUrl === void 0) {
|
|
490
|
-
loadOptions.core.baseUrl = topLevelBaseUrl ?? topLevelBaseUri;
|
|
491
|
-
}
|
|
492
|
-
delete loadOptions.baseUri;
|
|
493
|
-
delete loadOptions.baseUrl;
|
|
494
|
-
}
|
|
495
521
|
return loadOptions;
|
|
496
522
|
}
|
|
497
523
|
function normalizeError(error, message) {
|
|
@@ -506,10 +532,48 @@ function normalizeError(error, message) {
|
|
|
506
532
|
function unavailableCoreApiMethod(methodName) {
|
|
507
533
|
return () => {
|
|
508
534
|
throw new Error(
|
|
509
|
-
`CoreAPI.${methodName} is unavailable. Use @loaders.gl/core.
|
|
535
|
+
`CoreAPI.${methodName} is unavailable. Use @loaders.gl/core.createDataSourceSync().`
|
|
510
536
|
);
|
|
511
537
|
};
|
|
512
538
|
}
|
|
539
|
+
function getDataSourceName(data, options) {
|
|
540
|
+
var _a, _b, _c;
|
|
541
|
+
const explicitName = (_a = options.core) == null ? void 0 : _a.name;
|
|
542
|
+
if (explicitName !== void 0) {
|
|
543
|
+
if (typeof explicitName !== "string" || !explicitName.trim())
|
|
544
|
+
throw new Error("Data source name must be a non-empty string.");
|
|
545
|
+
return explicitName;
|
|
546
|
+
}
|
|
547
|
+
if (data && typeof data === "object") {
|
|
548
|
+
const named = data;
|
|
549
|
+
if (typeof named.name === "string" && named.name.trim())
|
|
550
|
+
return data instanceof Blob ? removeFileExtension(named.name) : named.name;
|
|
551
|
+
const metadata = (_b = named.schema) == null ? void 0 : _b.metadata;
|
|
552
|
+
const embeddedName = metadata instanceof Map ? metadata.get("name") : metadata == null ? void 0 : metadata.name;
|
|
553
|
+
if (typeof embeddedName === "string" && embeddedName.trim()) return embeddedName;
|
|
554
|
+
}
|
|
555
|
+
if (typeof data === "string") {
|
|
556
|
+
try {
|
|
557
|
+
const url = new URL(data.replace(/\\/g, "/"), "https://source.invalid/");
|
|
558
|
+
if (!["data:", "blob:"].includes(url.protocol)) {
|
|
559
|
+
let basename = url.pathname.split("/").filter(Boolean).pop();
|
|
560
|
+
if (basename) {
|
|
561
|
+
try {
|
|
562
|
+
basename = decodeURIComponent(basename);
|
|
563
|
+
} catch {
|
|
564
|
+
}
|
|
565
|
+
return removeFileExtension(basename);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
} catch {
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
return ((_c = options.core) == null ? void 0 : _c.type) && options.core.type !== "auto" ? options.core.type : "source";
|
|
572
|
+
}
|
|
573
|
+
function removeFileExtension(name) {
|
|
574
|
+
const extension = name.lastIndexOf(".");
|
|
575
|
+
return extension > 0 ? name.slice(0, extension) : name;
|
|
576
|
+
}
|
|
513
577
|
|
|
514
578
|
// ../core/src/loader-utils/lib/scan-utils/columnar-predicate.ts
|
|
515
579
|
function getColumnarPredicateColumns(predicate) {
|
|
@@ -588,6 +652,34 @@ function bindColumnarPredicateParameters(predicate, parameters) {
|
|
|
588
652
|
};
|
|
589
653
|
}
|
|
590
654
|
}
|
|
655
|
+
function filterColumnarRowIndices(predicate, columns, rowCount) {
|
|
656
|
+
if (!predicate) return Array.from({ length: rowCount }, (_, index) => index);
|
|
657
|
+
const result = [];
|
|
658
|
+
for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
|
|
659
|
+
if (evaluateColumnarPredicate(predicate, columns, rowIndex) === true) result.push(rowIndex);
|
|
660
|
+
}
|
|
661
|
+
return result;
|
|
662
|
+
}
|
|
663
|
+
function evaluateColumnarPredicate(predicate, columns, rowIndex) {
|
|
664
|
+
if (predicate.op === "and" || predicate.op === "or") {
|
|
665
|
+
const results = predicate.args.map(
|
|
666
|
+
(child) => evaluateColumnarPredicate(child, columns, rowIndex)
|
|
667
|
+
);
|
|
668
|
+
return predicate.op === "and" ? results.includes(false) ? false : results.includes(null) ? null : true : results.includes(true) ? true : results.includes(null) ? null : false;
|
|
669
|
+
}
|
|
670
|
+
if (predicate.op === "not") {
|
|
671
|
+
const value2 = evaluateColumnarPredicate(predicate.args[0], columns, rowIndex);
|
|
672
|
+
return value2 === null ? null : !value2;
|
|
673
|
+
}
|
|
674
|
+
const leaf = predicate;
|
|
675
|
+
const value = getColumnarValue(columns, getColumnarPredicatePath(leaf.args[0]), rowIndex);
|
|
676
|
+
if (leaf.op === "isNull") return value === null || value === void 0;
|
|
677
|
+
if (value === null || value === void 0) return null;
|
|
678
|
+
if (leaf.op === "in")
|
|
679
|
+
return leaf.args[1].some((candidate) => compareColumnarValues(value, candidate) === 0);
|
|
680
|
+
const comparison = compareColumnarValues(value, leaf.args[1]);
|
|
681
|
+
return leaf.op === "=" ? comparison === 0 : leaf.op === "<>" ? comparison !== 0 : leaf.op === "<" ? comparison < 0 : leaf.op === "<=" ? comparison <= 0 : leaf.op === ">" ? comparison > 0 : comparison >= 0;
|
|
682
|
+
}
|
|
591
683
|
function visitColumnarPredicate(predicate, visitor) {
|
|
592
684
|
visitor(predicate);
|
|
593
685
|
if (predicate.op === "and" || predicate.op === "or")
|
|
@@ -616,14 +708,56 @@ function copyColumnarPredicateProperty(property) {
|
|
|
616
708
|
property: typeof property.property === "string" ? property.property : [...property.property]
|
|
617
709
|
};
|
|
618
710
|
}
|
|
711
|
+
function getColumnarValue(columns, path, rowIndex) {
|
|
712
|
+
var _a;
|
|
713
|
+
let value = (_a = columns[path[0]]) == null ? void 0 : _a[rowIndex];
|
|
714
|
+
for (let index = 1; index < path.length && value != null; index++) value = value[path[index]];
|
|
715
|
+
return value;
|
|
716
|
+
}
|
|
619
717
|
function copyColumnarPredicateValue(value) {
|
|
620
718
|
if (value instanceof Uint8Array) return value.slice();
|
|
621
719
|
if (value instanceof Date) return new Date(value.getTime());
|
|
622
720
|
return value;
|
|
623
721
|
}
|
|
722
|
+
function compareColumnarValues(left, right) {
|
|
723
|
+
if (left instanceof Uint8Array && right instanceof Uint8Array) return compareBytes(left, right);
|
|
724
|
+
if (left instanceof Uint8Array || right instanceof Uint8Array)
|
|
725
|
+
throw new Error("Columnar binary predicates require binary values on both sides");
|
|
726
|
+
if (left instanceof Date || right instanceof Date) {
|
|
727
|
+
if (!(left instanceof Date) || !(right instanceof Date))
|
|
728
|
+
throw new Error("Columnar date predicates require date values on both sides");
|
|
729
|
+
return compareNumbers(left.getTime(), right.getTime());
|
|
730
|
+
}
|
|
731
|
+
const leftType = typeof left;
|
|
732
|
+
const rightType = typeof right;
|
|
733
|
+
const numeric = (leftType === "number" || leftType === "bigint") && (rightType === "number" || rightType === "bigint");
|
|
734
|
+
if (!numeric && leftType !== rightType)
|
|
735
|
+
throw new Error(`Columnar predicate cannot compare ${leftType} with ${rightType}`);
|
|
736
|
+
if (!numeric && leftType !== "string" && leftType !== "boolean")
|
|
737
|
+
throw new Error(`Columnar predicate does not support ${leftType} values`);
|
|
738
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
739
|
+
}
|
|
740
|
+
function compareNumbers(left, right) {
|
|
741
|
+
if (!Number.isFinite(left) || !Number.isFinite(right))
|
|
742
|
+
throw new Error("Non-finite values cannot be ordered by a columnar predicate");
|
|
743
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
744
|
+
}
|
|
745
|
+
function compareBytes(left, right) {
|
|
746
|
+
const length = Math.min(left.length, right.length);
|
|
747
|
+
for (let index = 0; index < length; index++)
|
|
748
|
+
if (left[index] !== right[index]) return left[index] < right[index] ? -1 : 1;
|
|
749
|
+
return left.length - right.length;
|
|
750
|
+
}
|
|
624
751
|
|
|
625
752
|
// ../core/src/loader-utils/lib/scan-utils/scan-query-metadata.ts
|
|
626
753
|
var import_crs = require("@math.gl/geospatial/crs");
|
|
754
|
+
function emitQueryComplete(callback, summary) {
|
|
755
|
+
if (!callback) return;
|
|
756
|
+
try {
|
|
757
|
+
callback(summary);
|
|
758
|
+
} catch {
|
|
759
|
+
}
|
|
760
|
+
}
|
|
627
761
|
function createScanQueryMetadata(options) {
|
|
628
762
|
validateScanExecutionSupport(options.queryType, options.execution);
|
|
629
763
|
const schema = Object.freeze({
|
|
@@ -648,6 +782,7 @@ function createScanQueryMetadata(options) {
|
|
|
648
782
|
});
|
|
649
783
|
}
|
|
650
784
|
);
|
|
785
|
+
const queryCapabilities = Object.freeze({ ...options.capabilities });
|
|
651
786
|
return Object.freeze({
|
|
652
787
|
sourceType: options.sourceType,
|
|
653
788
|
queryType: options.queryType,
|
|
@@ -656,7 +791,9 @@ function createScanQueryMetadata(options) {
|
|
|
656
791
|
description: options.description,
|
|
657
792
|
schema,
|
|
658
793
|
columns: Object.freeze(columns),
|
|
659
|
-
capabilities:
|
|
794
|
+
capabilities: queryCapabilities,
|
|
795
|
+
queryCapabilities,
|
|
796
|
+
...options.hasDefaultRowWindow ? { hasDefaultRowWindow: true } : {},
|
|
660
797
|
spatial: options.spatial ? Object.freeze({
|
|
661
798
|
...options.spatial,
|
|
662
799
|
bounds: options.spatial.bounds ? Object.freeze({
|
|
@@ -699,6 +836,13 @@ function getNonEmptyMetadataValue(metadata, key) {
|
|
|
699
836
|
const value = metadata == null ? void 0 : metadata[key];
|
|
700
837
|
return value || void 0;
|
|
701
838
|
}
|
|
839
|
+
function createQueryMetadata(options) {
|
|
840
|
+
const { queryCapabilities, ...metadataOptions } = options;
|
|
841
|
+
return createScanQueryMetadata({
|
|
842
|
+
...metadataOptions,
|
|
843
|
+
capabilities: queryCapabilities
|
|
844
|
+
});
|
|
845
|
+
}
|
|
702
846
|
|
|
703
847
|
// ../core/src/loader-utils/lib/scan-utils/table-query.ts
|
|
704
848
|
function planTableQuery(sourceColumnNames, options = {}) {
|
|
@@ -744,10 +888,173 @@ function validateTableQueryLimit(limit) {
|
|
|
744
888
|
throw new Error("Table query limit must be a non-negative safe integer.");
|
|
745
889
|
}
|
|
746
890
|
}
|
|
891
|
+
function validateTableQueryOffset(offset) {
|
|
892
|
+
if (offset !== void 0 && (!Number.isSafeInteger(offset) || offset < 0)) {
|
|
893
|
+
throw new Error("Table query offset must be a non-negative safe integer.");
|
|
894
|
+
}
|
|
895
|
+
}
|
|
747
896
|
|
|
748
897
|
// ../core/src/loader-utils/lib/scan-utils/table-scan-batch.ts
|
|
749
|
-
function
|
|
750
|
-
|
|
898
|
+
async function* executeTableScanBatches(readBatches, options = {}, operators = {}) {
|
|
899
|
+
var _a, _b, _c, _d, _e;
|
|
900
|
+
validateTableQueryLimit(options.limit);
|
|
901
|
+
validateTableQueryOffset(options.offset);
|
|
902
|
+
let skip = options.offset ?? 0;
|
|
903
|
+
const startedAt = Date.now();
|
|
904
|
+
let sourcesRead = 0;
|
|
905
|
+
let batchesRead = 0;
|
|
906
|
+
let rowsRead = 0;
|
|
907
|
+
let rowsTested = 0;
|
|
908
|
+
let rowsRetained = 0;
|
|
909
|
+
let rowsReturned = 0;
|
|
910
|
+
let bytesFetched = 0;
|
|
911
|
+
let status = "early-terminated";
|
|
912
|
+
let earlyTerminationReason;
|
|
913
|
+
let executionError;
|
|
914
|
+
let remaining = options.limit === void 0 ? Number.POSITIVE_INFINITY : Math.max(0, options.limit);
|
|
915
|
+
try {
|
|
916
|
+
(_a = options.signal) == null ? void 0 : _a.throwIfAborted();
|
|
917
|
+
if (remaining <= 0) {
|
|
918
|
+
earlyTerminationReason = "limit";
|
|
919
|
+
return;
|
|
920
|
+
}
|
|
921
|
+
sourcesRead = 1;
|
|
922
|
+
for await (const batch of readBatches(
|
|
923
|
+
options.signal,
|
|
924
|
+
(byteLength) => bytesFetched += byteLength
|
|
925
|
+
)) {
|
|
926
|
+
if (remaining <= 0) {
|
|
927
|
+
earlyTerminationReason = "limit";
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
930
|
+
(_b = options.signal) == null ? void 0 : _b.throwIfAborted();
|
|
931
|
+
batchesRead++;
|
|
932
|
+
const rowCount = getScanBatchRowCount(batch);
|
|
933
|
+
rowsRead += rowCount;
|
|
934
|
+
if (options.predicate && !operators.filter && !("shape" in batch)) {
|
|
935
|
+
throw new Error("Native Arrow scans require a filter kernel for residual predicates.");
|
|
936
|
+
}
|
|
937
|
+
const filteredBatch = operators.filter ? await operators.filter(batch, options.predicate, options.signal) : "shape" in batch ? filterTableBatch(batch, options.predicate) : batch;
|
|
938
|
+
(_c = options.signal) == null ? void 0 : _c.throwIfAborted();
|
|
939
|
+
const filteredRowCount = getScanBatchRowCount(filteredBatch);
|
|
940
|
+
if (options.predicate) {
|
|
941
|
+
rowsTested += rowCount;
|
|
942
|
+
rowsRetained += filteredRowCount;
|
|
943
|
+
}
|
|
944
|
+
if (skip >= filteredRowCount && (skip > 0 || options.skipEmptyBatches)) {
|
|
945
|
+
skip -= filteredRowCount;
|
|
946
|
+
continue;
|
|
947
|
+
}
|
|
948
|
+
const end = Math.min(filteredRowCount, skip + remaining);
|
|
949
|
+
const window = skip || end < filteredRowCount ? "shape" in filteredBatch ? sliceTableBatch(filteredBatch, skip, end) : filteredBatch.slice(skip, end) : filteredBatch;
|
|
950
|
+
skip = 0;
|
|
951
|
+
const output = operators.project ? operators.project(window, options.columns) : projectTableBatch(window, options.columns);
|
|
952
|
+
remaining -= getScanBatchRowCount(window);
|
|
953
|
+
rowsReturned += getScanBatchRowCount(output);
|
|
954
|
+
if (remaining === 0) earlyTerminationReason = "limit";
|
|
955
|
+
yield output;
|
|
956
|
+
if (remaining === 0) return;
|
|
957
|
+
(_d = options.signal) == null ? void 0 : _d.throwIfAborted();
|
|
958
|
+
}
|
|
959
|
+
status = "completed";
|
|
960
|
+
} catch (error) {
|
|
961
|
+
status = ((_e = options.signal) == null ? void 0 : _e.aborted) ? "cancelled" : "failed";
|
|
962
|
+
executionError = error;
|
|
963
|
+
throw error;
|
|
964
|
+
} finally {
|
|
965
|
+
if (status === "early-terminated" && !earlyTerminationReason) {
|
|
966
|
+
earlyTerminationReason = "consumer-return";
|
|
967
|
+
}
|
|
968
|
+
emitQueryComplete(
|
|
969
|
+
options.onQueryComplete,
|
|
970
|
+
Object.freeze({
|
|
971
|
+
status,
|
|
972
|
+
sourcesPlanned: 1,
|
|
973
|
+
sourcesRead,
|
|
974
|
+
batchesRead,
|
|
975
|
+
batchesDecoded: batchesRead,
|
|
976
|
+
rowsRead,
|
|
977
|
+
rowsTested: rowsTested || void 0,
|
|
978
|
+
rowsRetained: rowsRetained || void 0,
|
|
979
|
+
rowsReturned,
|
|
980
|
+
bytesRead: bytesFetched,
|
|
981
|
+
bytesFetched,
|
|
982
|
+
filesOpened: sourcesRead,
|
|
983
|
+
tasksOpened: sourcesRead,
|
|
984
|
+
durationMilliseconds: Date.now() - startedAt,
|
|
985
|
+
earlyTerminationReason,
|
|
986
|
+
...executionError === void 0 ? {} : { error: executionError }
|
|
987
|
+
})
|
|
988
|
+
);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
function getScanBatchRowCount(batch) {
|
|
992
|
+
return "shape" in batch ? batch.length : batch.numRows;
|
|
993
|
+
}
|
|
994
|
+
function sliceTableBatch(batch, start, end) {
|
|
995
|
+
const length = end - start;
|
|
996
|
+
if (batch.shape === "object-row-table" || batch.shape === "array-row-table") {
|
|
997
|
+
return { ...batch, data: batch.data.slice(start, end), length };
|
|
998
|
+
}
|
|
999
|
+
if (batch.shape === "columnar-table") {
|
|
1000
|
+
return {
|
|
1001
|
+
...batch,
|
|
1002
|
+
data: Object.fromEntries(
|
|
1003
|
+
Object.entries(batch.data).map(([name, values]) => [
|
|
1004
|
+
name,
|
|
1005
|
+
Array.from(values).slice(start, end)
|
|
1006
|
+
])
|
|
1007
|
+
),
|
|
1008
|
+
length
|
|
1009
|
+
};
|
|
1010
|
+
}
|
|
1011
|
+
return { ...batch, features: batch.features.slice(start, end), length };
|
|
1012
|
+
}
|
|
1013
|
+
function projectTableBatch(batch, columns) {
|
|
1014
|
+
if (columns === void 0) return batch;
|
|
1015
|
+
if (!("shape" in batch)) {
|
|
1016
|
+
const indices = columns.map((column) => batch.schema.fields.findIndex((field) => field.name === column)).filter((index) => index >= 0);
|
|
1017
|
+
return batch.selectAt(indices);
|
|
1018
|
+
}
|
|
1019
|
+
const schema = projectTableSchema(batch, columns);
|
|
1020
|
+
if (batch.shape === "object-row-table") {
|
|
1021
|
+
return {
|
|
1022
|
+
...batch,
|
|
1023
|
+
schema,
|
|
1024
|
+
data: batch.data.map((row) => Object.fromEntries(columns.map((column) => [column, row[column]])))
|
|
1025
|
+
};
|
|
1026
|
+
}
|
|
1027
|
+
if (batch.shape === "columnar-table") {
|
|
1028
|
+
return {
|
|
1029
|
+
...batch,
|
|
1030
|
+
schema,
|
|
1031
|
+
data: Object.fromEntries(columns.map((column) => [column, batch.data[column]]))
|
|
1032
|
+
};
|
|
1033
|
+
}
|
|
1034
|
+
return batch;
|
|
1035
|
+
}
|
|
1036
|
+
function projectTableSchema(batch, columns) {
|
|
1037
|
+
if (!batch.schema) return void 0;
|
|
1038
|
+
const fieldsByName = new Map(batch.schema.fields.map((field) => [field.name, field]));
|
|
1039
|
+
return {
|
|
1040
|
+
...batch.schema,
|
|
1041
|
+
fields: columns.flatMap((column) => {
|
|
1042
|
+
const field = fieldsByName.get(column);
|
|
1043
|
+
return field ? [field] : [];
|
|
1044
|
+
})
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
function filterTableBatch(batch, predicate) {
|
|
1048
|
+
var _a;
|
|
1049
|
+
if (!predicate || batch.shape !== "object-row-table") return batch;
|
|
1050
|
+
const rows = batch.data;
|
|
1051
|
+
const columnNames = new Set(((_a = batch.schema) == null ? void 0 : _a.fields.map((field) => field.name)) || []);
|
|
1052
|
+
validateColumnarPredicate(predicate, columnNames);
|
|
1053
|
+
const columns = Object.fromEntries(
|
|
1054
|
+
[...columnNames].map((name) => [name, rows.map((row) => row[name])])
|
|
1055
|
+
);
|
|
1056
|
+
const rowIndices = filterColumnarRowIndices(predicate, columns, rows.length);
|
|
1057
|
+
return { ...batch, data: rowIndices.map((rowIndex) => rows[rowIndex]), length: rowIndices.length };
|
|
751
1058
|
}
|
|
752
1059
|
|
|
753
1060
|
// ../core/src/loader-utils/lib/scan-utils/relational-query.ts
|
|
@@ -1287,115 +1594,389 @@ function validateSQLIdentifier(identifier) {
|
|
|
1287
1594
|
}
|
|
1288
1595
|
|
|
1289
1596
|
// ../core/src/scan/rows-to-arrow.ts
|
|
1290
|
-
var
|
|
1597
|
+
var arrow = __toESM(require("apache-arrow"), 1);
|
|
1598
|
+
function convertRowsToArrowTable(rows) {
|
|
1599
|
+
return arrow.tableFromJSON(rows);
|
|
1600
|
+
}
|
|
1291
1601
|
|
|
1292
|
-
//
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1602
|
+
// dist/sql-utils.js
|
|
1603
|
+
function isNodeRuntime() {
|
|
1604
|
+
var _a;
|
|
1605
|
+
return typeof process !== "undefined" && Boolean((_a = process.versions) == null ? void 0 : _a.node);
|
|
1606
|
+
}
|
|
1607
|
+
function escapeSqlString(value) {
|
|
1608
|
+
return value.replace(/'/g, "''");
|
|
1609
|
+
}
|
|
1610
|
+
function quoteSqlIdentifier(identifier) {
|
|
1611
|
+
return `"${identifier.replace(/"/g, '""')}"`;
|
|
1612
|
+
}
|
|
1613
|
+
function convertSQLColumnsToSchema(columns) {
|
|
1614
|
+
return {
|
|
1615
|
+
metadata: {},
|
|
1616
|
+
fields: columns.sort((left, right) => (left.ordinalPosition || 0) - (right.ordinalPosition || 0)).map((column) => ({
|
|
1617
|
+
name: column.columnName,
|
|
1618
|
+
type: mapSQLTypeToDataType(column.sqlType),
|
|
1619
|
+
nullable: column.nullable,
|
|
1620
|
+
metadata: {
|
|
1621
|
+
sqlType: column.sqlType
|
|
1622
|
+
}
|
|
1623
|
+
}))
|
|
1624
|
+
};
|
|
1625
|
+
}
|
|
1626
|
+
function mapSQLTypeToDataType(sqlType) {
|
|
1627
|
+
const normalizedType = sqlType.toUpperCase();
|
|
1628
|
+
if (normalizedType.includes("BIGINT") || normalizedType === "HUGEINT" || normalizedType === "LONG") {
|
|
1629
|
+
return "int64";
|
|
1300
1630
|
}
|
|
1301
|
-
if (
|
|
1302
|
-
return "
|
|
1631
|
+
if (normalizedType.includes("INT") || normalizedType === "INTEGER" || normalizedType === "SMALLINT" || normalizedType === "TINYINT") {
|
|
1632
|
+
return "int32";
|
|
1303
1633
|
}
|
|
1304
|
-
if (
|
|
1305
|
-
return "
|
|
1634
|
+
if (normalizedType.includes("DOUBLE") || normalizedType.includes("FLOAT8") || normalizedType.includes("REAL")) {
|
|
1635
|
+
return "float64";
|
|
1306
1636
|
}
|
|
1307
|
-
if (
|
|
1308
|
-
return "
|
|
1637
|
+
if (normalizedType.includes("FLOAT") || normalizedType.includes("DECIMAL") || normalizedType.includes("NUMERIC")) {
|
|
1638
|
+
return "float64";
|
|
1309
1639
|
}
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
function getDataTypeFromArray(array) {
|
|
1313
|
-
let type = getDataTypeFromTypedArray(array);
|
|
1314
|
-
if (type !== "null") {
|
|
1315
|
-
return { type, nullable: false };
|
|
1640
|
+
if (normalizedType.includes("BOOL")) {
|
|
1641
|
+
return "bool";
|
|
1316
1642
|
}
|
|
1317
|
-
if (
|
|
1318
|
-
|
|
1319
|
-
return { type, nullable: true };
|
|
1643
|
+
if (normalizedType.includes("DATE")) {
|
|
1644
|
+
return "date-day";
|
|
1320
1645
|
}
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
function getDataTypeFromTypedArray(array) {
|
|
1324
|
-
if (typeof Float16Array !== "undefined" && array instanceof Float16Array) {
|
|
1325
|
-
return "float16";
|
|
1646
|
+
if (normalizedType.includes("TIME") || normalizedType.includes("TIMESTAMP")) {
|
|
1647
|
+
return "timestamp-millisecond";
|
|
1326
1648
|
}
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
return "int8";
|
|
1330
|
-
case Uint8Array:
|
|
1331
|
-
case Uint8ClampedArray:
|
|
1332
|
-
return "uint8";
|
|
1333
|
-
case Int16Array:
|
|
1334
|
-
return "int16";
|
|
1335
|
-
case Uint16Array:
|
|
1336
|
-
return "uint16";
|
|
1337
|
-
case Int32Array:
|
|
1338
|
-
return "int32";
|
|
1339
|
-
case Uint32Array:
|
|
1340
|
-
return "uint32";
|
|
1341
|
-
case BigInt64Array:
|
|
1342
|
-
return "int64";
|
|
1343
|
-
case BigUint64Array:
|
|
1344
|
-
return "uint64";
|
|
1345
|
-
case Float32Array:
|
|
1346
|
-
return "float32";
|
|
1347
|
-
case Float64Array:
|
|
1348
|
-
return "float64";
|
|
1349
|
-
default:
|
|
1350
|
-
return "null";
|
|
1649
|
+
if (normalizedType.includes("BLOB") || normalizedType.includes("BINARY")) {
|
|
1650
|
+
return "binary";
|
|
1351
1651
|
}
|
|
1652
|
+
return "utf8";
|
|
1352
1653
|
}
|
|
1353
|
-
function
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
case "int8":
|
|
1357
|
-
return Int8Array;
|
|
1358
|
-
case "uint8":
|
|
1359
|
-
return Uint8Array;
|
|
1360
|
-
case "int16":
|
|
1361
|
-
return Int16Array;
|
|
1362
|
-
case "uint16":
|
|
1363
|
-
return Uint16Array;
|
|
1364
|
-
case "int32":
|
|
1365
|
-
return Int32Array;
|
|
1366
|
-
case "uint32":
|
|
1367
|
-
return Uint32Array;
|
|
1368
|
-
case "float32":
|
|
1369
|
-
return Float32Array;
|
|
1370
|
-
case "float64":
|
|
1371
|
-
return Float64Array;
|
|
1372
|
-
case "float16":
|
|
1373
|
-
return (0, import_core.getFloat16ArrayConstructor)();
|
|
1374
|
-
default:
|
|
1375
|
-
break;
|
|
1376
|
-
}
|
|
1377
|
-
}
|
|
1378
|
-
return Array;
|
|
1654
|
+
function getQualifiedTableName(options) {
|
|
1655
|
+
const identifiers = [options.catalogName, options.schemaName, options.tableName].filter((value) => Boolean(value));
|
|
1656
|
+
return identifiers.map((identifier) => quoteSqlIdentifier(identifier)).join(".");
|
|
1379
1657
|
}
|
|
1380
1658
|
|
|
1381
|
-
// ../core/src/
|
|
1382
|
-
var
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1659
|
+
// ../core/src/scan/table-query-capabilities.ts
|
|
1660
|
+
var ARROW_TABLE_QUERY_CAPABILITIES = Object.freeze({
|
|
1661
|
+
projection: "residual",
|
|
1662
|
+
predicate: "residual",
|
|
1663
|
+
limit: "residual",
|
|
1664
|
+
streaming: false,
|
|
1665
|
+
cancellation: true,
|
|
1666
|
+
expressions: "residual",
|
|
1667
|
+
orderBy: "residual",
|
|
1668
|
+
aggregates: "residual"
|
|
1669
|
+
});
|
|
1670
|
+
var SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES = Object.freeze({
|
|
1671
|
+
projection: "pushdown",
|
|
1672
|
+
predicate: "pushdown",
|
|
1673
|
+
limit: "pushdown",
|
|
1674
|
+
streaming: false,
|
|
1675
|
+
cancellation: false,
|
|
1676
|
+
expressions: "pushdown",
|
|
1677
|
+
orderBy: "pushdown",
|
|
1678
|
+
aggregates: "pushdown"
|
|
1679
|
+
});
|
|
1680
|
+
|
|
1681
|
+
// dist/sql-source.js
|
|
1682
|
+
var SQL_ADAPTER_FACTORIES = /* @__PURE__ */ new Map();
|
|
1683
|
+
function registerSQLAdapter(type, factory) {
|
|
1684
|
+
SQL_ADAPTER_FACTORIES.set(type, factory);
|
|
1388
1685
|
}
|
|
1389
|
-
function
|
|
1390
|
-
return
|
|
1391
|
-
fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
|
|
1392
|
-
metadata: serializeArrowMetadata(arrowSchema.metadata)
|
|
1393
|
-
};
|
|
1686
|
+
function getSQLAdapterFactory(type) {
|
|
1687
|
+
return SQL_ADAPTER_FACTORIES.get(type);
|
|
1394
1688
|
}
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1689
|
+
var SQL_SOURCE_DEFAULT_OPTIONS = {
|
|
1690
|
+
sql: {
|
|
1691
|
+
resultFormat: "auto",
|
|
1692
|
+
adapterOptions: {},
|
|
1693
|
+
browserAdapterFactory: void 0,
|
|
1694
|
+
nodeAdapterFactory: void 0
|
|
1695
|
+
},
|
|
1696
|
+
duckdb: {
|
|
1697
|
+
accessMode: "auto",
|
|
1698
|
+
bundles: void 0,
|
|
1699
|
+
databasePath: void 0,
|
|
1700
|
+
remoteUrl: void 0,
|
|
1701
|
+
workerUrl: void 0
|
|
1702
|
+
},
|
|
1703
|
+
snowflake: {
|
|
1704
|
+
account: void 0,
|
|
1705
|
+
database: void 0,
|
|
1706
|
+
schema: void 0,
|
|
1707
|
+
token: void 0,
|
|
1708
|
+
warehouse: void 0,
|
|
1709
|
+
role: void 0,
|
|
1710
|
+
authenticator: void 0
|
|
1711
|
+
}
|
|
1712
|
+
};
|
|
1713
|
+
var SQLDataSource = class extends DataSource {
|
|
1714
|
+
/** Optimization and execution capabilities of portable table-bound queries. */
|
|
1715
|
+
tableQueryCapabilities = SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES;
|
|
1716
|
+
sourceType;
|
|
1717
|
+
adapterFactory;
|
|
1718
|
+
adapterPromise = null;
|
|
1719
|
+
metadataPromise = null;
|
|
1720
|
+
constructor(data, options, sourceType, defaultOptions = SQL_SOURCE_DEFAULT_OPTIONS, adapterFactory, coreApi) {
|
|
1721
|
+
super(data, options, defaultOptions, coreApi);
|
|
1722
|
+
this.sourceType = sourceType;
|
|
1723
|
+
this.adapterFactory = adapterFactory || getSQLAdapterFactory(sourceType);
|
|
1724
|
+
}
|
|
1725
|
+
/** Returns cached high-level metadata for the SQL source. */
|
|
1726
|
+
async getMetadata() {
|
|
1727
|
+
if (!this.metadataPromise) {
|
|
1728
|
+
this.metadataPromise = this.loadMetadata();
|
|
1729
|
+
}
|
|
1730
|
+
return this.metadataPromise;
|
|
1731
|
+
}
|
|
1732
|
+
/** Lists available catalogs from the connected database. */
|
|
1733
|
+
async listCatalogs() {
|
|
1734
|
+
const adapter = await this.getAdapter();
|
|
1735
|
+
return await adapter.listCatalogs();
|
|
1736
|
+
}
|
|
1737
|
+
/** Lists available schemas from the connected database. */
|
|
1738
|
+
async listSchemas(catalogName) {
|
|
1739
|
+
const adapter = await this.getAdapter();
|
|
1740
|
+
return await adapter.listSchemas(catalogName);
|
|
1741
|
+
}
|
|
1742
|
+
/** Lists available tables from the connected database. */
|
|
1743
|
+
async listTables(options) {
|
|
1744
|
+
const adapter = await this.getAdapter();
|
|
1745
|
+
return await adapter.listTables(options);
|
|
1746
|
+
}
|
|
1747
|
+
/** Lists stable native table identifiers without fetching any column schemas or data rows. */
|
|
1748
|
+
async getTablesMetadata(options = {}) {
|
|
1749
|
+
var _a, _b;
|
|
1750
|
+
(_a = options.signal) == null ? void 0 : _a.throwIfAborted();
|
|
1751
|
+
const tables = await this.listTables();
|
|
1752
|
+
(_b = options.signal) == null ? void 0 : _b.throwIfAborted();
|
|
1753
|
+
return tables.map((table) => ({ name: getTableMetadataName(table), queryable: false }));
|
|
1754
|
+
}
|
|
1755
|
+
/** Discovers one native table schema; ambiguous unqualified names are rejected. */
|
|
1756
|
+
async getTableMetadata(name, options = {}) {
|
|
1757
|
+
var _a;
|
|
1758
|
+
const table = await this.resolveTable(name, options.signal);
|
|
1759
|
+
const schema = await this.getTableSchema(table);
|
|
1760
|
+
(_a = options.signal) == null ? void 0 : _a.throwIfAborted();
|
|
1761
|
+
return { name: getTableMetadataName(table), schema, queryable: false };
|
|
1762
|
+
}
|
|
1763
|
+
/** Resolves a qualified or unique unqualified table name. */
|
|
1764
|
+
async resolveTable(name, signal) {
|
|
1765
|
+
signal == null ? void 0 : signal.throwIfAborted();
|
|
1766
|
+
const tables = await this.listTables();
|
|
1767
|
+
signal == null ? void 0 : signal.throwIfAborted();
|
|
1768
|
+
const exact = tables.filter((table) => getTableMetadataName(table) === name);
|
|
1769
|
+
const matches = exact.length ? exact : tables.filter((table) => table.tableName === name);
|
|
1770
|
+
if (matches.length !== 1)
|
|
1771
|
+
throw new Error(`Unknown or ambiguous SQL table: ${name}`);
|
|
1772
|
+
return matches[0];
|
|
1773
|
+
}
|
|
1774
|
+
/** Returns the schema for a specific SQL table. */
|
|
1775
|
+
async getTableSchema(options) {
|
|
1776
|
+
const adapter = await this.getAdapter();
|
|
1777
|
+
return await adapter.getTableSchema(options);
|
|
1778
|
+
}
|
|
1779
|
+
/** Executes a SQL query and returns object rows. */
|
|
1780
|
+
async queryRows(query, options = {}) {
|
|
1781
|
+
const adapter = await this.getAdapter();
|
|
1782
|
+
const compiledQuery = this.compileQuery(query, options);
|
|
1783
|
+
return await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
|
|
1784
|
+
}
|
|
1785
|
+
/** Executes a SQL query and returns a native Arrow table. */
|
|
1786
|
+
async queryArrow(query, options = {}) {
|
|
1787
|
+
const adapter = await this.getAdapter();
|
|
1788
|
+
const compiledQuery = this.compileQuery(query, options);
|
|
1789
|
+
if (adapter.executeArrow) {
|
|
1790
|
+
return await adapter.executeArrow(compiledQuery.sql, compiledQuery.options);
|
|
1791
|
+
}
|
|
1792
|
+
const rows = await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
|
|
1793
|
+
return convertRowsToArrowTable(rows);
|
|
1794
|
+
}
|
|
1795
|
+
/** Closes any underlying adapter connection. */
|
|
1796
|
+
async close() {
|
|
1797
|
+
if (!this.adapterPromise) {
|
|
1798
|
+
return;
|
|
1799
|
+
}
|
|
1800
|
+
const adapter = await this.adapterPromise;
|
|
1801
|
+
await adapter.close();
|
|
1802
|
+
this.adapterPromise = null;
|
|
1803
|
+
this.metadataPromise = null;
|
|
1804
|
+
}
|
|
1805
|
+
async getAdapter() {
|
|
1806
|
+
if (!this.adapterPromise) {
|
|
1807
|
+
this.adapterPromise = this.loadAdapter();
|
|
1808
|
+
}
|
|
1809
|
+
return await this.adapterPromise;
|
|
1810
|
+
}
|
|
1811
|
+
/** Converts a portable query object to SQL while leaving raw SQL text unchanged. */
|
|
1812
|
+
compileQuery(query, options) {
|
|
1813
|
+
if (typeof query === "string") {
|
|
1814
|
+
return { sql: query, options };
|
|
1815
|
+
}
|
|
1816
|
+
const namedParameters = options.parameters && !Array.isArray(options.parameters) ? options.parameters : void 0;
|
|
1817
|
+
const compiledQuery = compileSQLTableQuery(query, {
|
|
1818
|
+
dialect: getSQLTableQueryDialect(this.sourceType),
|
|
1819
|
+
parameters: namedParameters
|
|
1820
|
+
});
|
|
1821
|
+
return {
|
|
1822
|
+
sql: compiledQuery.sql,
|
|
1823
|
+
options: { signal: options.signal, parameters: compiledQuery.parameters }
|
|
1824
|
+
};
|
|
1825
|
+
}
|
|
1826
|
+
async loadAdapter() {
|
|
1827
|
+
var _a, _b;
|
|
1828
|
+
try {
|
|
1829
|
+
const runtimeFactory = isNodeRuntime() ? (_a = this.options.sql) == null ? void 0 : _a.nodeAdapterFactory : (_b = this.options.sql) == null ? void 0 : _b.browserAdapterFactory;
|
|
1830
|
+
const factory = runtimeFactory || this.adapterFactory;
|
|
1831
|
+
if (!factory) {
|
|
1832
|
+
throw new Error(`No SQL adapter is registered for source type "${this.sourceType}".`);
|
|
1833
|
+
}
|
|
1834
|
+
const adapter = await factory({
|
|
1835
|
+
coreApi: this.coreApi,
|
|
1836
|
+
options: this.options,
|
|
1837
|
+
sourceType: this.sourceType,
|
|
1838
|
+
url: this.url
|
|
1839
|
+
});
|
|
1840
|
+
await adapter.connect();
|
|
1841
|
+
return adapter;
|
|
1842
|
+
} catch (error) {
|
|
1843
|
+
throw this.reportError(error, `Failed to create SQL adapter for ${this.sourceType} from ${this.url}`);
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
async loadMetadata() {
|
|
1847
|
+
const adapter = await this.getAdapter();
|
|
1848
|
+
const [catalogs, schemas, tables] = await Promise.all([
|
|
1849
|
+
adapter.listCatalogs(),
|
|
1850
|
+
adapter.listSchemas(),
|
|
1851
|
+
adapter.listTables()
|
|
1852
|
+
]);
|
|
1853
|
+
return {
|
|
1854
|
+
type: this.sourceType,
|
|
1855
|
+
capabilities: adapter.capabilities,
|
|
1856
|
+
catalogs,
|
|
1857
|
+
schemas,
|
|
1858
|
+
tables
|
|
1859
|
+
};
|
|
1860
|
+
}
|
|
1861
|
+
};
|
|
1862
|
+
function getSQLTableQueryDialect(sourceType) {
|
|
1863
|
+
if (sourceType === "duckdb-sql")
|
|
1864
|
+
return "duckdb";
|
|
1865
|
+
if (sourceType === "snowflake-sql")
|
|
1866
|
+
return "snowflake";
|
|
1867
|
+
throw new Error(`Portable table queries are not supported for SQL source type "${sourceType}".`);
|
|
1868
|
+
}
|
|
1869
|
+
function getTableMetadataName(table) {
|
|
1870
|
+
return [table.catalogName, table.schemaName, table.tableName].filter((name) => typeof name === "string" && name.length > 0).map((name) => `"${name.replaceAll('"', '""')}"`).join(".");
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
// ../core/src/schema-utils/lib/schema/data-type.ts
|
|
1874
|
+
var import_core = require("@math.gl/core");
|
|
1875
|
+
function getDataTypeFromValue(value, defaultNumberType = "float32") {
|
|
1876
|
+
if (value instanceof Date) {
|
|
1877
|
+
return "date-millisecond";
|
|
1878
|
+
}
|
|
1879
|
+
if (typeof value === "number") {
|
|
1880
|
+
return defaultNumberType;
|
|
1881
|
+
}
|
|
1882
|
+
if (typeof value === "boolean") {
|
|
1883
|
+
return "bool";
|
|
1884
|
+
}
|
|
1885
|
+
if (typeof value === "string") {
|
|
1886
|
+
return "utf8";
|
|
1887
|
+
}
|
|
1888
|
+
if (value === null || value === "undefined") {
|
|
1889
|
+
return "null";
|
|
1890
|
+
}
|
|
1891
|
+
return "null";
|
|
1892
|
+
}
|
|
1893
|
+
function getDataTypeFromArray(array) {
|
|
1894
|
+
let type = getDataTypeFromTypedArray(array);
|
|
1895
|
+
if (type !== "null") {
|
|
1896
|
+
return { type, nullable: false };
|
|
1897
|
+
}
|
|
1898
|
+
if (array.length > 0) {
|
|
1899
|
+
type = getDataTypeFromValue(array[0]);
|
|
1900
|
+
return { type, nullable: true };
|
|
1901
|
+
}
|
|
1902
|
+
return { type: "null", nullable: true };
|
|
1903
|
+
}
|
|
1904
|
+
function getDataTypeFromTypedArray(array) {
|
|
1905
|
+
if (typeof Float16Array !== "undefined" && array instanceof Float16Array) {
|
|
1906
|
+
return "float16";
|
|
1907
|
+
}
|
|
1908
|
+
switch (array.constructor) {
|
|
1909
|
+
case Int8Array:
|
|
1910
|
+
return "int8";
|
|
1911
|
+
case Uint8Array:
|
|
1912
|
+
case Uint8ClampedArray:
|
|
1913
|
+
return "uint8";
|
|
1914
|
+
case Int16Array:
|
|
1915
|
+
return "int16";
|
|
1916
|
+
case Uint16Array:
|
|
1917
|
+
return "uint16";
|
|
1918
|
+
case Int32Array:
|
|
1919
|
+
return "int32";
|
|
1920
|
+
case Uint32Array:
|
|
1921
|
+
return "uint32";
|
|
1922
|
+
case BigInt64Array:
|
|
1923
|
+
return "int64";
|
|
1924
|
+
case BigUint64Array:
|
|
1925
|
+
return "uint64";
|
|
1926
|
+
case Float32Array:
|
|
1927
|
+
return "float32";
|
|
1928
|
+
case Float64Array:
|
|
1929
|
+
return "float64";
|
|
1930
|
+
default:
|
|
1931
|
+
return "null";
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
function getArrayTypeFromDataType(type, nullable) {
|
|
1935
|
+
if (!nullable) {
|
|
1936
|
+
switch (type) {
|
|
1937
|
+
case "int8":
|
|
1938
|
+
return Int8Array;
|
|
1939
|
+
case "uint8":
|
|
1940
|
+
return Uint8Array;
|
|
1941
|
+
case "int16":
|
|
1942
|
+
return Int16Array;
|
|
1943
|
+
case "uint16":
|
|
1944
|
+
return Uint16Array;
|
|
1945
|
+
case "int32":
|
|
1946
|
+
return Int32Array;
|
|
1947
|
+
case "uint32":
|
|
1948
|
+
return Uint32Array;
|
|
1949
|
+
case "float32":
|
|
1950
|
+
return Float32Array;
|
|
1951
|
+
case "float64":
|
|
1952
|
+
return Float64Array;
|
|
1953
|
+
case "float16":
|
|
1954
|
+
return (0, import_core.getFloat16ArrayConstructor)();
|
|
1955
|
+
default:
|
|
1956
|
+
break;
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
return Array;
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
// ../core/src/schema-utils/lib/schema/convert-arrow-schema.ts
|
|
1963
|
+
var arrow2 = __toESM(require("apache-arrow"), 1);
|
|
1964
|
+
function convertArrowToSchema(arrowSchema) {
|
|
1965
|
+
return serializeArrowSchema(arrowSchema);
|
|
1966
|
+
}
|
|
1967
|
+
function convertSchemaToArrow(schema, options) {
|
|
1968
|
+
return deserializeArrowSchema(schema, options);
|
|
1969
|
+
}
|
|
1970
|
+
function serializeArrowSchema(arrowSchema) {
|
|
1971
|
+
return {
|
|
1972
|
+
fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
|
|
1973
|
+
metadata: serializeArrowMetadata(arrowSchema.metadata)
|
|
1974
|
+
};
|
|
1975
|
+
}
|
|
1976
|
+
function deserializeArrowSchema(schema, options) {
|
|
1977
|
+
return new arrow2.Schema(
|
|
1978
|
+
schema.fields.map((field) => deserializeArrowField(field, options)),
|
|
1979
|
+
deserializeArrowMetadata(schema.metadata)
|
|
1399
1980
|
);
|
|
1400
1981
|
}
|
|
1401
1982
|
function serializeArrowMetadata(arrowMetadata) {
|
|
@@ -1413,7 +1994,7 @@ function serializeArrowField(field) {
|
|
|
1413
1994
|
};
|
|
1414
1995
|
}
|
|
1415
1996
|
function deserializeArrowField(field, options) {
|
|
1416
|
-
return new
|
|
1997
|
+
return new arrow2.Field(
|
|
1417
1998
|
field.name,
|
|
1418
1999
|
deserializeArrowType(field.type, options),
|
|
1419
2000
|
field.nullable,
|
|
@@ -1435,57 +2016,57 @@ function serializeArrowType(arrowType) {
|
|
|
1435
2016
|
};
|
|
1436
2017
|
}
|
|
1437
2018
|
switch (arrowType.constructor) {
|
|
1438
|
-
case
|
|
2019
|
+
case arrow2.Null:
|
|
1439
2020
|
return "null";
|
|
1440
|
-
case
|
|
2021
|
+
case arrow2.Binary:
|
|
1441
2022
|
return "binary";
|
|
1442
|
-
case
|
|
2023
|
+
case arrow2.FixedSizeBinary:
|
|
1443
2024
|
return {
|
|
1444
2025
|
type: "fixed-size-binary",
|
|
1445
2026
|
byteWidth: arrowType.byteWidth
|
|
1446
2027
|
};
|
|
1447
|
-
case
|
|
2028
|
+
case arrow2.Bool:
|
|
1448
2029
|
return "bool";
|
|
1449
|
-
case
|
|
2030
|
+
case arrow2.Int:
|
|
1450
2031
|
const intType = arrowType;
|
|
1451
2032
|
return `${intType.isSigned ? "" : "u"}int${intType.bitWidth}`;
|
|
1452
|
-
case
|
|
2033
|
+
case arrow2.Int8:
|
|
1453
2034
|
return "int8";
|
|
1454
|
-
case
|
|
2035
|
+
case arrow2.Int16:
|
|
1455
2036
|
return "int16";
|
|
1456
|
-
case
|
|
2037
|
+
case arrow2.Int32:
|
|
1457
2038
|
return "int32";
|
|
1458
|
-
case
|
|
2039
|
+
case arrow2.Int64:
|
|
1459
2040
|
return "int64";
|
|
1460
|
-
case
|
|
2041
|
+
case arrow2.Uint8:
|
|
1461
2042
|
return "uint8";
|
|
1462
|
-
case
|
|
2043
|
+
case arrow2.Uint16:
|
|
1463
2044
|
return "uint16";
|
|
1464
|
-
case
|
|
2045
|
+
case arrow2.Uint32:
|
|
1465
2046
|
return "uint32";
|
|
1466
|
-
case
|
|
2047
|
+
case arrow2.Uint64:
|
|
1467
2048
|
return "uint64";
|
|
1468
|
-
case
|
|
2049
|
+
case arrow2.Float:
|
|
1469
2050
|
const precision = arrowType.precision;
|
|
1470
2051
|
switch (precision) {
|
|
1471
|
-
case
|
|
2052
|
+
case arrow2.Precision.HALF:
|
|
1472
2053
|
return "float16";
|
|
1473
|
-
case
|
|
2054
|
+
case arrow2.Precision.SINGLE:
|
|
1474
2055
|
return "float32";
|
|
1475
|
-
case
|
|
2056
|
+
case arrow2.Precision.DOUBLE:
|
|
1476
2057
|
return "float64";
|
|
1477
2058
|
default:
|
|
1478
2059
|
return "float16";
|
|
1479
2060
|
}
|
|
1480
|
-
case
|
|
2061
|
+
case arrow2.Float16:
|
|
1481
2062
|
return "float16";
|
|
1482
|
-
case
|
|
2063
|
+
case arrow2.Float32:
|
|
1483
2064
|
return "float32";
|
|
1484
|
-
case
|
|
2065
|
+
case arrow2.Float64:
|
|
1485
2066
|
return "float64";
|
|
1486
|
-
case
|
|
2067
|
+
case arrow2.Utf8:
|
|
1487
2068
|
return "utf8";
|
|
1488
|
-
case
|
|
2069
|
+
case arrow2.Decimal:
|
|
1489
2070
|
const decimal = arrowType;
|
|
1490
2071
|
return {
|
|
1491
2072
|
type: "decimal",
|
|
@@ -1493,108 +2074,108 @@ function serializeArrowType(arrowType) {
|
|
|
1493
2074
|
precision: decimal.precision,
|
|
1494
2075
|
scale: decimal.scale
|
|
1495
2076
|
};
|
|
1496
|
-
case
|
|
2077
|
+
case arrow2.Date_:
|
|
1497
2078
|
const dateUnit = arrowType.unit;
|
|
1498
|
-
return dateUnit ===
|
|
1499
|
-
case
|
|
2079
|
+
return dateUnit === arrow2.DateUnit.DAY ? "date-day" : "date-millisecond";
|
|
2080
|
+
case arrow2.DateDay:
|
|
1500
2081
|
return "date-day";
|
|
1501
|
-
case
|
|
2082
|
+
case arrow2.DateMillisecond:
|
|
1502
2083
|
return "date-millisecond";
|
|
1503
|
-
case
|
|
2084
|
+
case arrow2.Time:
|
|
1504
2085
|
const timeUnit = arrowType.unit;
|
|
1505
2086
|
switch (timeUnit) {
|
|
1506
|
-
case
|
|
2087
|
+
case arrow2.TimeUnit.SECOND:
|
|
1507
2088
|
return "time-second";
|
|
1508
|
-
case
|
|
2089
|
+
case arrow2.TimeUnit.MILLISECOND:
|
|
1509
2090
|
return "time-millisecond";
|
|
1510
|
-
case
|
|
2091
|
+
case arrow2.TimeUnit.MICROSECOND:
|
|
1511
2092
|
return "time-microsecond";
|
|
1512
|
-
case
|
|
2093
|
+
case arrow2.TimeUnit.NANOSECOND:
|
|
1513
2094
|
return "time-nanosecond";
|
|
1514
2095
|
default:
|
|
1515
2096
|
return "time-second";
|
|
1516
2097
|
}
|
|
1517
|
-
case
|
|
2098
|
+
case arrow2.TimeMillisecond:
|
|
1518
2099
|
return "time-millisecond";
|
|
1519
|
-
case
|
|
2100
|
+
case arrow2.TimeSecond:
|
|
1520
2101
|
return "time-second";
|
|
1521
|
-
case
|
|
2102
|
+
case arrow2.TimeMicrosecond:
|
|
1522
2103
|
return "time-microsecond";
|
|
1523
|
-
case
|
|
2104
|
+
case arrow2.TimeNanosecond:
|
|
1524
2105
|
return "time-nanosecond";
|
|
1525
|
-
case
|
|
2106
|
+
case arrow2.Timestamp:
|
|
1526
2107
|
const timeStampUnit = arrowType.unit;
|
|
1527
2108
|
switch (timeStampUnit) {
|
|
1528
|
-
case
|
|
2109
|
+
case arrow2.TimeUnit.SECOND:
|
|
1529
2110
|
return "timestamp-second";
|
|
1530
|
-
case
|
|
2111
|
+
case arrow2.TimeUnit.MILLISECOND:
|
|
1531
2112
|
return "timestamp-millisecond";
|
|
1532
|
-
case
|
|
2113
|
+
case arrow2.TimeUnit.MICROSECOND:
|
|
1533
2114
|
return "timestamp-microsecond";
|
|
1534
|
-
case
|
|
2115
|
+
case arrow2.TimeUnit.NANOSECOND:
|
|
1535
2116
|
return "timestamp-nanosecond";
|
|
1536
2117
|
default:
|
|
1537
2118
|
return "timestamp-second";
|
|
1538
2119
|
}
|
|
1539
|
-
case
|
|
2120
|
+
case arrow2.TimestampSecond:
|
|
1540
2121
|
return "timestamp-second";
|
|
1541
|
-
case
|
|
2122
|
+
case arrow2.TimestampMillisecond:
|
|
1542
2123
|
return "timestamp-millisecond";
|
|
1543
|
-
case
|
|
2124
|
+
case arrow2.TimestampMicrosecond:
|
|
1544
2125
|
return "timestamp-microsecond";
|
|
1545
|
-
case
|
|
2126
|
+
case arrow2.TimestampNanosecond:
|
|
1546
2127
|
return "timestamp-nanosecond";
|
|
1547
|
-
case
|
|
2128
|
+
case arrow2.Interval:
|
|
1548
2129
|
const intervalUnit = arrowType.unit;
|
|
1549
2130
|
switch (intervalUnit) {
|
|
1550
|
-
case
|
|
2131
|
+
case arrow2.IntervalUnit.DAY_TIME:
|
|
1551
2132
|
return "interval-daytime";
|
|
1552
|
-
case
|
|
2133
|
+
case arrow2.IntervalUnit.YEAR_MONTH:
|
|
1553
2134
|
return "interval-yearmonth";
|
|
1554
2135
|
default:
|
|
1555
2136
|
return "interval-daytime";
|
|
1556
2137
|
}
|
|
1557
|
-
case
|
|
2138
|
+
case arrow2.IntervalDayTime:
|
|
1558
2139
|
return "interval-daytime";
|
|
1559
|
-
case
|
|
2140
|
+
case arrow2.IntervalYearMonth:
|
|
1560
2141
|
return "interval-yearmonth";
|
|
1561
|
-
case
|
|
2142
|
+
case arrow2.Map_:
|
|
1562
2143
|
const mapType = arrowType;
|
|
1563
2144
|
return {
|
|
1564
2145
|
type: "map",
|
|
1565
2146
|
keysSorted: mapType.keysSorted,
|
|
1566
2147
|
children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
1567
2148
|
};
|
|
1568
|
-
case
|
|
2149
|
+
case arrow2.List:
|
|
1569
2150
|
const listType = arrowType;
|
|
1570
2151
|
const listField = listType.valueField;
|
|
1571
2152
|
return {
|
|
1572
2153
|
type: "list",
|
|
1573
2154
|
children: [serializeArrowField(listField)]
|
|
1574
2155
|
};
|
|
1575
|
-
case
|
|
2156
|
+
case arrow2.FixedSizeList:
|
|
1576
2157
|
const fixedSizeList = arrowType;
|
|
1577
2158
|
return {
|
|
1578
2159
|
type: "fixed-size-list",
|
|
1579
2160
|
listSize: fixedSizeList.listSize,
|
|
1580
2161
|
children: [serializeArrowField(fixedSizeList.children[0])]
|
|
1581
2162
|
};
|
|
1582
|
-
case
|
|
2163
|
+
case arrow2.Struct:
|
|
1583
2164
|
const structType = arrowType;
|
|
1584
2165
|
return {
|
|
1585
2166
|
type: "struct",
|
|
1586
2167
|
children: structType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
1587
2168
|
};
|
|
1588
|
-
case
|
|
1589
|
-
case
|
|
2169
|
+
case arrow2.SparseUnion:
|
|
2170
|
+
case arrow2.DenseUnion:
|
|
1590
2171
|
const unionType = arrowType;
|
|
1591
2172
|
return {
|
|
1592
|
-
type: unionType instanceof
|
|
2173
|
+
type: unionType instanceof arrow2.DenseUnion ? "dense-union" : "sparse-union",
|
|
1593
2174
|
typeIds: unionType.typeIds,
|
|
1594
2175
|
children: unionType.children.map((arrowField) => serializeArrowField(arrowField)),
|
|
1595
2176
|
typeIdToChildIndex: { ...unionType.typeIdToChildIndex }
|
|
1596
2177
|
};
|
|
1597
|
-
case
|
|
2178
|
+
case arrow2.Dictionary:
|
|
1598
2179
|
const dictionaryType = arrowType;
|
|
1599
2180
|
return {
|
|
1600
2181
|
type: "dictionary",
|
|
@@ -1611,16 +2192,16 @@ function deserializeArrowType(dataType, options) {
|
|
|
1611
2192
|
if (typeof dataType === "object") {
|
|
1612
2193
|
switch (dataType.type) {
|
|
1613
2194
|
case "decimal":
|
|
1614
|
-
return new
|
|
2195
|
+
return new arrow2.Decimal(dataType.scale, dataType.precision, dataType.bitWidth);
|
|
1615
2196
|
case "map": {
|
|
1616
2197
|
const children = dataType.children.map(
|
|
1617
2198
|
(arrowField) => deserializeArrowField(arrowField, options)
|
|
1618
2199
|
);
|
|
1619
|
-
return new
|
|
2200
|
+
return new arrow2.Map_(children[0], dataType.keysSorted);
|
|
1620
2201
|
}
|
|
1621
2202
|
case "list": {
|
|
1622
2203
|
const field = deserializeArrowField(dataType.children[0], options);
|
|
1623
|
-
return new
|
|
2204
|
+
return new arrow2.List(field);
|
|
1624
2205
|
}
|
|
1625
2206
|
case "large-list": {
|
|
1626
2207
|
const field = deserializeArrowField(dataType.children[0], options);
|
|
@@ -1634,25 +2215,25 @@ function deserializeArrowType(dataType, options) {
|
|
|
1634
2215
|
}
|
|
1635
2216
|
case "fixed-size-list": {
|
|
1636
2217
|
const child = deserializeArrowField(dataType.children[0], options);
|
|
1637
|
-
return new
|
|
2218
|
+
return new arrow2.FixedSizeList(dataType.listSize, child);
|
|
1638
2219
|
}
|
|
1639
2220
|
case "fixed-size-binary":
|
|
1640
|
-
return new
|
|
2221
|
+
return new arrow2.FixedSizeBinary(dataType.byteWidth);
|
|
1641
2222
|
case "struct": {
|
|
1642
2223
|
const children = dataType.children.map(
|
|
1643
2224
|
(arrowField) => deserializeArrowField(arrowField, options)
|
|
1644
2225
|
);
|
|
1645
|
-
return new
|
|
2226
|
+
return new arrow2.Struct(children);
|
|
1646
2227
|
}
|
|
1647
2228
|
case "sparse-union":
|
|
1648
2229
|
case "dense-union": {
|
|
1649
2230
|
const children = dataType.children.map(
|
|
1650
2231
|
(arrowField) => deserializeArrowField(arrowField, options)
|
|
1651
2232
|
);
|
|
1652
|
-
return dataType.type === "dense-union" ? new
|
|
2233
|
+
return dataType.type === "dense-union" ? new arrow2.DenseUnion(dataType.typeIds, children) : new arrow2.SparseUnion(dataType.typeIds, children);
|
|
1653
2234
|
}
|
|
1654
2235
|
case "dictionary": {
|
|
1655
|
-
return new
|
|
2236
|
+
return new arrow2.Dictionary(
|
|
1656
2237
|
deserializeArrowType(dataType.dictionary, options),
|
|
1657
2238
|
deserializeArrowDictionaryKeyType(dataType.indices),
|
|
1658
2239
|
dataType.id,
|
|
@@ -1665,76 +2246,76 @@ function deserializeArrowType(dataType, options) {
|
|
|
1665
2246
|
}
|
|
1666
2247
|
switch (dataType) {
|
|
1667
2248
|
case "null":
|
|
1668
|
-
return new
|
|
2249
|
+
return new arrow2.Null();
|
|
1669
2250
|
case "binary":
|
|
1670
|
-
return makeVariableWidthArrowType("BinaryView", new
|
|
2251
|
+
return makeVariableWidthArrowType("BinaryView", new arrow2.Binary(), options);
|
|
1671
2252
|
case "binary-view":
|
|
1672
2253
|
return makeRequiredArrowViewType("BinaryView");
|
|
1673
2254
|
case "bool":
|
|
1674
|
-
return new
|
|
2255
|
+
return new arrow2.Bool();
|
|
1675
2256
|
case "int8":
|
|
1676
|
-
return new
|
|
2257
|
+
return new arrow2.Int8();
|
|
1677
2258
|
case "int16":
|
|
1678
|
-
return new
|
|
2259
|
+
return new arrow2.Int16();
|
|
1679
2260
|
case "int32":
|
|
1680
|
-
return new
|
|
2261
|
+
return new arrow2.Int32();
|
|
1681
2262
|
case "int64":
|
|
1682
|
-
return new
|
|
2263
|
+
return new arrow2.Int64();
|
|
1683
2264
|
case "uint8":
|
|
1684
|
-
return new
|
|
2265
|
+
return new arrow2.Uint8();
|
|
1685
2266
|
case "uint16":
|
|
1686
|
-
return new
|
|
2267
|
+
return new arrow2.Uint16();
|
|
1687
2268
|
case "uint32":
|
|
1688
|
-
return new
|
|
2269
|
+
return new arrow2.Uint32();
|
|
1689
2270
|
case "uint64":
|
|
1690
|
-
return new
|
|
2271
|
+
return new arrow2.Uint64();
|
|
1691
2272
|
case "float16":
|
|
1692
|
-
return new
|
|
2273
|
+
return new arrow2.Float16();
|
|
1693
2274
|
case "float32":
|
|
1694
|
-
return new
|
|
2275
|
+
return new arrow2.Float32();
|
|
1695
2276
|
case "float64":
|
|
1696
|
-
return new
|
|
2277
|
+
return new arrow2.Float64();
|
|
1697
2278
|
case "utf8":
|
|
1698
|
-
return makeVariableWidthArrowType("Utf8View", new
|
|
2279
|
+
return makeVariableWidthArrowType("Utf8View", new arrow2.Utf8(), options);
|
|
1699
2280
|
case "utf8-view":
|
|
1700
2281
|
return makeRequiredArrowViewType("Utf8View");
|
|
1701
2282
|
case "date-day":
|
|
1702
|
-
return new
|
|
2283
|
+
return new arrow2.DateDay();
|
|
1703
2284
|
case "date-millisecond":
|
|
1704
|
-
return new
|
|
2285
|
+
return new arrow2.DateMillisecond();
|
|
1705
2286
|
case "time-second":
|
|
1706
|
-
return new
|
|
2287
|
+
return new arrow2.TimeSecond();
|
|
1707
2288
|
case "time-millisecond":
|
|
1708
|
-
return new
|
|
2289
|
+
return new arrow2.TimeMillisecond();
|
|
1709
2290
|
case "time-microsecond":
|
|
1710
|
-
return new
|
|
2291
|
+
return new arrow2.TimeMicrosecond();
|
|
1711
2292
|
case "time-nanosecond":
|
|
1712
|
-
return new
|
|
2293
|
+
return new arrow2.TimeNanosecond();
|
|
1713
2294
|
case "timestamp-second":
|
|
1714
|
-
return new
|
|
2295
|
+
return new arrow2.TimestampSecond();
|
|
1715
2296
|
case "timestamp-millisecond":
|
|
1716
|
-
return new
|
|
2297
|
+
return new arrow2.TimestampMillisecond();
|
|
1717
2298
|
case "timestamp-microsecond":
|
|
1718
|
-
return new
|
|
2299
|
+
return new arrow2.TimestampMicrosecond();
|
|
1719
2300
|
case "timestamp-nanosecond":
|
|
1720
|
-
return new
|
|
2301
|
+
return new arrow2.TimestampNanosecond();
|
|
1721
2302
|
case "interval-daytime":
|
|
1722
|
-
return new
|
|
2303
|
+
return new arrow2.IntervalDayTime();
|
|
1723
2304
|
case "interval-yearmonth":
|
|
1724
|
-
return new
|
|
2305
|
+
return new arrow2.IntervalYearMonth();
|
|
1725
2306
|
default:
|
|
1726
2307
|
throw new Error("array type not supported");
|
|
1727
2308
|
}
|
|
1728
2309
|
}
|
|
1729
2310
|
function getOptionalArrowConstructor(typeName) {
|
|
1730
|
-
return
|
|
2311
|
+
return arrow2[typeName];
|
|
1731
2312
|
}
|
|
1732
2313
|
function isArrowViewType(arrowType, viewTypeName) {
|
|
1733
2314
|
const constructor = getOptionalArrowConstructor(viewTypeName);
|
|
1734
2315
|
if (constructor && arrowType instanceof constructor) {
|
|
1735
2316
|
return true;
|
|
1736
2317
|
}
|
|
1737
|
-
const guards =
|
|
2318
|
+
const guards = arrow2.DataType;
|
|
1738
2319
|
const guard = viewTypeName === "BinaryView" ? guards.isBinaryView : guards.isUtf8View;
|
|
1739
2320
|
return Boolean(guard == null ? void 0 : guard(arrowType));
|
|
1740
2321
|
}
|
|
@@ -1768,7 +2349,7 @@ function makeRequiredArrowViewType(viewTypeName) {
|
|
|
1768
2349
|
return new constructor();
|
|
1769
2350
|
}
|
|
1770
2351
|
function serializeArrowDictionaryKeyType(arrowType) {
|
|
1771
|
-
if (arrowType instanceof
|
|
2352
|
+
if (arrowType instanceof arrow2.Int) {
|
|
1772
2353
|
const prefix = arrowType.isSigned ? "int" : "uint";
|
|
1773
2354
|
const keyType = `${prefix}${arrowType.bitWidth}`;
|
|
1774
2355
|
if (isSchemaDictionaryKeyType(keyType)) {
|
|
@@ -1776,17 +2357,17 @@ function serializeArrowDictionaryKeyType(arrowType) {
|
|
|
1776
2357
|
}
|
|
1777
2358
|
}
|
|
1778
2359
|
switch (arrowType.constructor) {
|
|
1779
|
-
case
|
|
2360
|
+
case arrow2.Int8:
|
|
1780
2361
|
return "int8";
|
|
1781
|
-
case
|
|
2362
|
+
case arrow2.Int16:
|
|
1782
2363
|
return "int16";
|
|
1783
|
-
case
|
|
2364
|
+
case arrow2.Int32:
|
|
1784
2365
|
return "int32";
|
|
1785
|
-
case
|
|
2366
|
+
case arrow2.Uint8:
|
|
1786
2367
|
return "uint8";
|
|
1787
|
-
case
|
|
2368
|
+
case arrow2.Uint16:
|
|
1788
2369
|
return "uint16";
|
|
1789
|
-
case
|
|
2370
|
+
case arrow2.Uint32:
|
|
1790
2371
|
return "uint32";
|
|
1791
2372
|
default:
|
|
1792
2373
|
throw new Error(`arrow dictionary index type not supported: ${arrowType.constructor.name}`);
|
|
@@ -1798,17 +2379,17 @@ function isSchemaDictionaryKeyType(keyType) {
|
|
|
1798
2379
|
function deserializeArrowDictionaryKeyType(keyType) {
|
|
1799
2380
|
switch (keyType) {
|
|
1800
2381
|
case "int8":
|
|
1801
|
-
return new
|
|
2382
|
+
return new arrow2.Int8();
|
|
1802
2383
|
case "int16":
|
|
1803
|
-
return new
|
|
2384
|
+
return new arrow2.Int16();
|
|
1804
2385
|
case "int32":
|
|
1805
|
-
return new
|
|
2386
|
+
return new arrow2.Int32();
|
|
1806
2387
|
case "uint8":
|
|
1807
|
-
return new
|
|
2388
|
+
return new arrow2.Uint8();
|
|
1808
2389
|
case "uint16":
|
|
1809
|
-
return new
|
|
2390
|
+
return new arrow2.Uint16();
|
|
1810
2391
|
case "uint32":
|
|
1811
|
-
return new
|
|
2392
|
+
return new arrow2.Uint32();
|
|
1812
2393
|
default:
|
|
1813
2394
|
throw new Error(`schema dictionary index type not supported: ${keyType}`);
|
|
1814
2395
|
}
|
|
@@ -1816,6 +2397,7 @@ function deserializeArrowDictionaryKeyType(keyType) {
|
|
|
1816
2397
|
|
|
1817
2398
|
// ../core/src/schema-utils/lib/schema/deduce-table-schema.ts
|
|
1818
2399
|
function deduceTableSchema(table) {
|
|
2400
|
+
if (!("shape" in table)) return convertArrowToSchema(table.schema);
|
|
1819
2401
|
switch (table.shape) {
|
|
1820
2402
|
case "array-row-table":
|
|
1821
2403
|
case "object-row-table":
|
|
@@ -1824,7 +2406,6 @@ function deduceTableSchema(table) {
|
|
|
1824
2406
|
return deduceSchemaFromGeoJSON(table.features);
|
|
1825
2407
|
case "columnar-table":
|
|
1826
2408
|
return deduceSchemaFromColumns(table.data);
|
|
1827
|
-
case "arrow-table":
|
|
1828
2409
|
default:
|
|
1829
2410
|
throw new Error("Deduce schema");
|
|
1830
2411
|
}
|
|
@@ -1844,7 +2425,8 @@ function deduceSchemaFromRows(rowTable) {
|
|
|
1844
2425
|
const fields = [];
|
|
1845
2426
|
const row0 = rowTable[0];
|
|
1846
2427
|
for (const [columnName, value] of Object.entries(row0)) {
|
|
1847
|
-
|
|
2428
|
+
const name = Array.isArray(row0) ? `column${Number(columnName) + 1}` : columnName;
|
|
2429
|
+
fields.push(deduceFieldFromValue(value, name));
|
|
1848
2430
|
}
|
|
1849
2431
|
return { fields, metadata: {} };
|
|
1850
2432
|
}
|
|
@@ -1891,62 +2473,24 @@ function deduceFieldFromValue(value, name) {
|
|
|
1891
2473
|
};
|
|
1892
2474
|
}
|
|
1893
2475
|
|
|
1894
|
-
// ../core/src/schema-utils/lib/
|
|
1895
|
-
var
|
|
1896
|
-
function
|
|
1897
|
-
|
|
1898
|
-
switch (getTableShapeFromData(data)) {
|
|
1899
|
-
case "array-row-table":
|
|
1900
|
-
table = { shape: "array-row-table", data };
|
|
1901
|
-
break;
|
|
1902
|
-
case "object-row-table":
|
|
1903
|
-
table = { shape: "object-row-table", data };
|
|
1904
|
-
break;
|
|
1905
|
-
case "columnar-table":
|
|
1906
|
-
table = { shape: "columnar-table", data };
|
|
1907
|
-
break;
|
|
1908
|
-
case "arrow-table":
|
|
1909
|
-
table = { shape: "arrow-table", data };
|
|
1910
|
-
break;
|
|
1911
|
-
default:
|
|
1912
|
-
throw new Error("table");
|
|
1913
|
-
}
|
|
1914
|
-
const schema = deduceTableSchema(table);
|
|
1915
|
-
return { ...table, schema };
|
|
2476
|
+
// ../core/src/schema-utils/lib/arrow/arrow-metadata.ts
|
|
2477
|
+
var arrow3 = __toESM(require("apache-arrow"), 1);
|
|
2478
|
+
function snapshotArrowSchema(schema) {
|
|
2479
|
+
return deserializeArrowSchema(serializeArrowSchema(schema));
|
|
1916
2480
|
}
|
|
1917
|
-
function
|
|
1918
|
-
|
|
1919
|
-
return "arrow-table";
|
|
1920
|
-
}
|
|
1921
|
-
if (Array.isArray(data)) {
|
|
1922
|
-
if (data.length === 0) {
|
|
1923
|
-
throw new Error("cannot deduce type of empty table");
|
|
1924
|
-
}
|
|
1925
|
-
const firstRow = data[0];
|
|
1926
|
-
if (Array.isArray(firstRow)) {
|
|
1927
|
-
return "array-row-table";
|
|
1928
|
-
}
|
|
1929
|
-
if (firstRow && typeof firstRow === "object") {
|
|
1930
|
-
return "object-row-table";
|
|
1931
|
-
}
|
|
1932
|
-
}
|
|
1933
|
-
if (data && typeof data === "object") {
|
|
1934
|
-
return "columnar-table";
|
|
1935
|
-
}
|
|
1936
|
-
throw new Error("invalid table");
|
|
2481
|
+
function makeArrowMetadataBatch(schema) {
|
|
2482
|
+
return new arrow3.RecordBatch(snapshotArrowSchema(schema), void 0);
|
|
1937
2483
|
}
|
|
1938
2484
|
|
|
1939
2485
|
// ../core/src/schema-utils/lib/table/tables/table-accessors.ts
|
|
1940
2486
|
function getTableLength(table) {
|
|
2487
|
+
if (!("shape" in table)) return table.numRows;
|
|
1941
2488
|
switch (table.shape) {
|
|
1942
2489
|
case "array-row-table":
|
|
1943
2490
|
case "object-row-table":
|
|
1944
2491
|
return table.data.length;
|
|
1945
2492
|
case "geojson-table":
|
|
1946
2493
|
return table.features.length;
|
|
1947
|
-
case "arrow-table":
|
|
1948
|
-
const arrowTable = table.data;
|
|
1949
|
-
return arrowTable.numRows;
|
|
1950
2494
|
case "columnar-table":
|
|
1951
2495
|
for (const column of Object.values(table.data)) {
|
|
1952
2496
|
return column.length || 0;
|
|
@@ -1957,6 +2501,7 @@ function getTableLength(table) {
|
|
|
1957
2501
|
}
|
|
1958
2502
|
}
|
|
1959
2503
|
function getTableNumCols(table) {
|
|
2504
|
+
if (!("shape" in table)) return table.numCols;
|
|
1960
2505
|
if (table.schema) {
|
|
1961
2506
|
return table.schema.fields.length;
|
|
1962
2507
|
}
|
|
@@ -1972,15 +2517,13 @@ function getTableNumCols(table) {
|
|
|
1972
2517
|
return Object.keys(table.features[0]).length;
|
|
1973
2518
|
case "columnar-table":
|
|
1974
2519
|
return Object.keys(table.data).length;
|
|
1975
|
-
case "arrow-table":
|
|
1976
|
-
const arrowTable = table.data;
|
|
1977
|
-
return arrowTable.numCols;
|
|
1978
2520
|
default:
|
|
1979
2521
|
throw new Error("table");
|
|
1980
2522
|
}
|
|
1981
2523
|
}
|
|
1982
2524
|
function getTableCell(table, rowIndex, columnName) {
|
|
1983
2525
|
var _a;
|
|
2526
|
+
if (!("shape" in table)) return (_a = table.getChild(columnName)) == null ? void 0 : _a.get(rowIndex);
|
|
1984
2527
|
switch (table.shape) {
|
|
1985
2528
|
case "array-row-table":
|
|
1986
2529
|
const columnIndex = getTableColumnIndex(table, columnName);
|
|
@@ -1992,18 +2535,13 @@ function getTableCell(table, rowIndex, columnName) {
|
|
|
1992
2535
|
case "columnar-table":
|
|
1993
2536
|
const column = table.data[columnName];
|
|
1994
2537
|
return column[rowIndex];
|
|
1995
|
-
case "arrow-table":
|
|
1996
|
-
const arrowTable = table.data;
|
|
1997
|
-
const arrowColumnIndex = arrowTable.schema.fields.findIndex(
|
|
1998
|
-
(field) => field.name === columnName
|
|
1999
|
-
);
|
|
2000
|
-
return (_a = arrowTable.getChildAt(arrowColumnIndex)) == null ? void 0 : _a.get(rowIndex);
|
|
2001
2538
|
default:
|
|
2002
2539
|
throw new Error("todo");
|
|
2003
2540
|
}
|
|
2004
2541
|
}
|
|
2005
2542
|
function getTableCellAt(table, rowIndex, columnIndex) {
|
|
2006
2543
|
var _a;
|
|
2544
|
+
if (!("shape" in table)) return (_a = table.getChildAt(columnIndex)) == null ? void 0 : _a.get(rowIndex);
|
|
2007
2545
|
switch (table.shape) {
|
|
2008
2546
|
case "array-row-table":
|
|
2009
2547
|
return table.data[rowIndex][columnIndex];
|
|
@@ -2017,9 +2555,6 @@ function getTableCellAt(table, rowIndex, columnIndex) {
|
|
|
2017
2555
|
const columnName3 = getTableColumnName(table, columnIndex);
|
|
2018
2556
|
const column = table.data[columnName3];
|
|
2019
2557
|
return column[rowIndex];
|
|
2020
|
-
case "arrow-table":
|
|
2021
|
-
const arrowTable = table.data;
|
|
2022
|
-
return (_a = arrowTable.getChildAt(columnIndex)) == null ? void 0 : _a.get(rowIndex);
|
|
2023
2558
|
default:
|
|
2024
2559
|
throw new Error("todo");
|
|
2025
2560
|
}
|
|
@@ -2041,101 +2576,97 @@ function getTableColumnName(table, columnIndex) {
|
|
|
2041
2576
|
return columnName;
|
|
2042
2577
|
}
|
|
2043
2578
|
function getTableRowAsObject(table, rowIndex, target, copy) {
|
|
2579
|
+
if (!("shape" in table)) {
|
|
2580
|
+
const objectRow = target || {};
|
|
2581
|
+
const row = table.get(rowIndex);
|
|
2582
|
+
for (const field of table.schema.fields) objectRow[field.name] = row == null ? void 0 : row[field.name];
|
|
2583
|
+
return objectRow;
|
|
2584
|
+
}
|
|
2044
2585
|
switch (table.shape) {
|
|
2045
2586
|
case "object-row-table":
|
|
2046
2587
|
return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
|
|
2047
2588
|
case "array-row-table":
|
|
2048
2589
|
if (table.schema) {
|
|
2049
|
-
const
|
|
2590
|
+
const objectRow = target || {};
|
|
2050
2591
|
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
2051
|
-
|
|
2592
|
+
objectRow[table.schema.fields[i].name] = table.data[rowIndex][i];
|
|
2052
2593
|
}
|
|
2053
|
-
return
|
|
2594
|
+
return objectRow;
|
|
2054
2595
|
}
|
|
2055
2596
|
throw new Error("no schema");
|
|
2056
2597
|
case "geojson-table":
|
|
2057
2598
|
if (table.schema) {
|
|
2058
|
-
const
|
|
2599
|
+
const objectRow = target || {};
|
|
2059
2600
|
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
2060
|
-
|
|
2601
|
+
objectRow[table.schema.fields[i].name] = table.features[rowIndex][table.schema.fields[i].name];
|
|
2061
2602
|
}
|
|
2062
|
-
return
|
|
2603
|
+
return objectRow;
|
|
2063
2604
|
}
|
|
2064
2605
|
throw new Error("no schema");
|
|
2065
2606
|
case "columnar-table":
|
|
2066
2607
|
if (table.schema) {
|
|
2067
|
-
const
|
|
2608
|
+
const objectRow = target || {};
|
|
2068
2609
|
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
2069
|
-
|
|
2610
|
+
objectRow[table.schema.fields[i].name] = table.data[table.schema.fields[i].name][rowIndex];
|
|
2070
2611
|
}
|
|
2071
|
-
return
|
|
2612
|
+
return objectRow;
|
|
2072
2613
|
} else {
|
|
2073
|
-
const
|
|
2614
|
+
const objectRow = target || {};
|
|
2074
2615
|
for (const [name, column] of Object.entries(table.data)) {
|
|
2075
|
-
|
|
2616
|
+
objectRow[name] = column[rowIndex];
|
|
2076
2617
|
}
|
|
2077
|
-
return
|
|
2078
|
-
}
|
|
2079
|
-
case "arrow-table":
|
|
2080
|
-
const arrowTable = table.data;
|
|
2081
|
-
const objectRow = target || {};
|
|
2082
|
-
const row = arrowTable.get(rowIndex);
|
|
2083
|
-
const schema = arrowTable.schema;
|
|
2084
|
-
for (let i = 0; i < schema.fields.length; i++) {
|
|
2085
|
-
objectRow[schema.fields[i].name] = row == null ? void 0 : row[schema.fields[i].name];
|
|
2618
|
+
return objectRow;
|
|
2086
2619
|
}
|
|
2087
|
-
return objectRow;
|
|
2088
2620
|
default:
|
|
2089
2621
|
throw new Error("shape");
|
|
2090
2622
|
}
|
|
2091
2623
|
}
|
|
2092
2624
|
function getTableRowAsArray(table, rowIndex, target, copy) {
|
|
2625
|
+
if (!("shape" in table)) {
|
|
2626
|
+
const arrayRow = target || [];
|
|
2627
|
+
const row = table.get(rowIndex);
|
|
2628
|
+
table.schema.fields.forEach((field, columnIndex) => {
|
|
2629
|
+
arrayRow[columnIndex] = row == null ? void 0 : row[field.name];
|
|
2630
|
+
});
|
|
2631
|
+
return arrayRow;
|
|
2632
|
+
}
|
|
2093
2633
|
switch (table.shape) {
|
|
2094
2634
|
case "array-row-table":
|
|
2095
2635
|
return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
|
|
2096
2636
|
case "object-row-table":
|
|
2097
2637
|
if (table.schema) {
|
|
2098
|
-
const
|
|
2638
|
+
const arrayRow = target || [];
|
|
2099
2639
|
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
2100
|
-
|
|
2640
|
+
arrayRow[i] = table.data[rowIndex][table.schema.fields[i].name];
|
|
2101
2641
|
}
|
|
2102
|
-
return
|
|
2642
|
+
return arrayRow;
|
|
2103
2643
|
}
|
|
2104
2644
|
return Object.values(table.data[rowIndex]);
|
|
2105
2645
|
case "geojson-table":
|
|
2106
2646
|
if (table.schema) {
|
|
2107
|
-
const
|
|
2647
|
+
const arrayRow = target || [];
|
|
2108
2648
|
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
2109
|
-
|
|
2649
|
+
arrayRow[i] = table.features[rowIndex][table.schema.fields[i].name];
|
|
2110
2650
|
}
|
|
2111
|
-
return
|
|
2651
|
+
return arrayRow;
|
|
2112
2652
|
}
|
|
2113
2653
|
return Object.values(table.features[rowIndex]);
|
|
2114
2654
|
case "columnar-table":
|
|
2115
2655
|
if (table.schema) {
|
|
2116
|
-
const
|
|
2656
|
+
const arrayRow = target || [];
|
|
2117
2657
|
for (let i = 0; i < table.schema.fields.length; i++) {
|
|
2118
|
-
|
|
2658
|
+
arrayRow[i] = table.data[table.schema.fields[i].name][rowIndex];
|
|
2119
2659
|
}
|
|
2120
|
-
return
|
|
2660
|
+
return arrayRow;
|
|
2121
2661
|
} else {
|
|
2122
|
-
const
|
|
2662
|
+
const arrayRow = target || [];
|
|
2123
2663
|
let i = 0;
|
|
2124
2664
|
for (const column of Object.values(table.data)) {
|
|
2125
|
-
|
|
2665
|
+
arrayRow[i] = column[rowIndex];
|
|
2126
2666
|
i++;
|
|
2127
2667
|
}
|
|
2128
|
-
return
|
|
2129
|
-
}
|
|
2130
|
-
case "arrow-table":
|
|
2131
|
-
const arrowTable = table.data;
|
|
2132
|
-
const arrayRow = target || [];
|
|
2133
|
-
const row = arrowTable.get(rowIndex);
|
|
2134
|
-
const schema = arrowTable.schema;
|
|
2135
|
-
for (let i = 0; i < schema.fields.length; i++) {
|
|
2136
|
-
arrayRow[i] = row == null ? void 0 : row[schema.fields[i].name];
|
|
2668
|
+
return arrayRow;
|
|
2137
2669
|
}
|
|
2138
|
-
return arrayRow;
|
|
2139
2670
|
default:
|
|
2140
2671
|
throw new Error("shape");
|
|
2141
2672
|
}
|
|
@@ -2148,22 +2679,22 @@ function makeColumnFromField(field, length) {
|
|
|
2148
2679
|
}
|
|
2149
2680
|
|
|
2150
2681
|
// ../core/src/schema-utils/lib/table/tables/convert-arrow-table.ts
|
|
2151
|
-
var
|
|
2682
|
+
var arrow5 = __toESM(require("apache-arrow"), 1);
|
|
2152
2683
|
|
|
2153
2684
|
// ../core/src/schema-utils/lib/table/batches/make-arrow-batch-iterator.ts
|
|
2154
|
-
var
|
|
2155
|
-
function*
|
|
2685
|
+
var arrow4 = __toESM(require("apache-arrow"), 1);
|
|
2686
|
+
function* makePortableTableRecordBatchIterator(table, options) {
|
|
2156
2687
|
const arrowSchema = convertSchemaToArrow(table.schema);
|
|
2157
2688
|
const length = getTableLength(table);
|
|
2158
2689
|
const numColumns = getTableNumCols(table);
|
|
2159
2690
|
const batchSize = (options == null ? void 0 : options.batchSize) || length;
|
|
2160
2691
|
const builders = arrowSchema == null ? void 0 : arrowSchema.fields.map(
|
|
2161
|
-
(arrowField) =>
|
|
2692
|
+
(arrowField) => arrow4.makeBuilder({
|
|
2162
2693
|
type: arrowField.type,
|
|
2163
2694
|
nullValues: [null, void 0]
|
|
2164
2695
|
})
|
|
2165
2696
|
);
|
|
2166
|
-
const structField = new
|
|
2697
|
+
const structField = new arrow4.Struct(arrowSchema.fields);
|
|
2167
2698
|
let batchLength = 0;
|
|
2168
2699
|
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
|
|
2169
2700
|
for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
|
|
@@ -2174,15 +2705,15 @@ function* makeArrowRecordBatchIterator(table, options) {
|
|
|
2174
2705
|
batchLength++;
|
|
2175
2706
|
if (batchLength >= batchSize) {
|
|
2176
2707
|
const datas = builders.map((builder) => builder.flush());
|
|
2177
|
-
const structData = new
|
|
2178
|
-
yield new
|
|
2708
|
+
const structData = new arrow4.Data(structField, 0, batchLength, 0, void 0, datas);
|
|
2709
|
+
yield new arrow4.RecordBatch(arrowSchema, structData);
|
|
2179
2710
|
batchLength = 0;
|
|
2180
2711
|
}
|
|
2181
2712
|
}
|
|
2182
2713
|
if (batchLength > 0) {
|
|
2183
2714
|
const datas = builders.map((builder) => builder.flush());
|
|
2184
|
-
const structData = new
|
|
2185
|
-
yield new
|
|
2715
|
+
const structData = new arrow4.Data(structField, 0, batchLength, 0, void 0, datas);
|
|
2716
|
+
yield new arrow4.RecordBatch(arrowSchema, structData);
|
|
2186
2717
|
batchLength = 0;
|
|
2187
2718
|
}
|
|
2188
2719
|
builders.map((builder) => builder.finish());
|
|
@@ -2190,22 +2721,16 @@ function* makeArrowRecordBatchIterator(table, options) {
|
|
|
2190
2721
|
|
|
2191
2722
|
// ../core/src/schema-utils/lib/table/tables/convert-arrow-table.ts
|
|
2192
2723
|
function convertTableToArrow(table, options) {
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
// return convertColumnarTableToArrow(table, options);
|
|
2199
|
-
// fall through
|
|
2200
|
-
default:
|
|
2201
|
-
const arrowBatchIterator = makeArrowRecordBatchIterator(table, options);
|
|
2202
|
-
return new arrow4.Table(arrowBatchIterator);
|
|
2203
|
-
}
|
|
2724
|
+
if (!("shape" in table)) return table;
|
|
2725
|
+
const schema = table.schema ?? deduceTableSchema(table);
|
|
2726
|
+
return new arrow5.Table(convertSchemaToArrow(schema), [
|
|
2727
|
+
...makePortableTableRecordBatchIterator({ ...table, schema }, options)
|
|
2728
|
+
]);
|
|
2204
2729
|
}
|
|
2205
2730
|
function convertArrowToTable(arrowTable, shape) {
|
|
2206
2731
|
switch (shape) {
|
|
2207
|
-
case "arrow
|
|
2208
|
-
return
|
|
2732
|
+
case "arrow":
|
|
2733
|
+
return arrowTable;
|
|
2209
2734
|
case "array-row-table":
|
|
2210
2735
|
return convertArrowToArrayRowTable(arrowTable);
|
|
2211
2736
|
case "object-row-table":
|
|
@@ -2218,13 +2743,6 @@ function convertArrowToTable(arrowTable, shape) {
|
|
|
2218
2743
|
throw new Error(shape);
|
|
2219
2744
|
}
|
|
2220
2745
|
}
|
|
2221
|
-
function convertArrowToArrowTable(arrowTable) {
|
|
2222
|
-
return {
|
|
2223
|
-
shape: "arrow-table",
|
|
2224
|
-
schema: convertArrowToSchema(arrowTable.schema),
|
|
2225
|
-
data: arrowTable
|
|
2226
|
-
};
|
|
2227
|
-
}
|
|
2228
2746
|
function convertArrowToArrayRowTable(arrowTable) {
|
|
2229
2747
|
const columnarTable = convertArrowToColumnarTable(arrowTable);
|
|
2230
2748
|
return convertTable(columnarTable, "array-row-table");
|
|
@@ -2265,6 +2783,8 @@ function convertArrowToGeoJSONTable(arrowTable) {
|
|
|
2265
2783
|
|
|
2266
2784
|
// ../core/src/schema-utils/lib/table/tables/convert-table.ts
|
|
2267
2785
|
function convertTable(table, shape) {
|
|
2786
|
+
if (shape === "arrow") return convertTableToArrow(table);
|
|
2787
|
+
if (!("shape" in table)) return convertArrowToTable(table, shape);
|
|
2268
2788
|
switch (shape) {
|
|
2269
2789
|
case "object-row-table":
|
|
2270
2790
|
return convertToObjectRowTable(table);
|
|
@@ -2272,8 +2792,6 @@ function convertTable(table, shape) {
|
|
|
2272
2792
|
return convertToArrayRowTable(table);
|
|
2273
2793
|
case "columnar-table":
|
|
2274
2794
|
return convertToColumnarTable(table);
|
|
2275
|
-
case "arrow-table":
|
|
2276
|
-
return convertToArrowTable(table);
|
|
2277
2795
|
default:
|
|
2278
2796
|
throw new Error(shape);
|
|
2279
2797
|
}
|
|
@@ -2329,295 +2847,257 @@ function convertToObjectRowTable(table) {
|
|
|
2329
2847
|
data
|
|
2330
2848
|
};
|
|
2331
2849
|
}
|
|
2332
|
-
function convertToArrowTable(table) {
|
|
2333
|
-
const schema = table.schema || deduceTableSchema(table);
|
|
2334
|
-
const arrowTable = convertTableToArrow({ ...table, schema });
|
|
2335
|
-
return {
|
|
2336
|
-
shape: "arrow-table",
|
|
2337
|
-
schema,
|
|
2338
|
-
data: arrowTable
|
|
2339
|
-
};
|
|
2340
|
-
}
|
|
2341
2850
|
|
|
2342
2851
|
// ../core/src/schema-utils/deprecated.ts
|
|
2343
2852
|
var convertArrowToTable2 = ((...args) => convertArrowToTable(
|
|
2344
2853
|
...args
|
|
2345
2854
|
));
|
|
2346
2855
|
|
|
2347
|
-
// ../core/src/scan/
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2856
|
+
// ../core/src/scan/query-arrow-table.ts
|
|
2857
|
+
var arrow8 = __toESM(require("apache-arrow"), 1);
|
|
2858
|
+
|
|
2859
|
+
// ../core/src/scan/arrow-execution/project-arrow-table.ts
|
|
2860
|
+
var arrow6 = __toESM(require("apache-arrow"), 1);
|
|
2861
|
+
function projectArrowTable(source, selections) {
|
|
2862
|
+
const outputNames = /* @__PURE__ */ new Set();
|
|
2863
|
+
const indices = [];
|
|
2864
|
+
const fields = selections.map((selection) => {
|
|
2865
|
+
const column = typeof selection === "string" ? selection : selection.column;
|
|
2866
|
+
const name = typeof selection === "string" ? selection : selection.as === void 0 ? column : selection.as;
|
|
2867
|
+
const index = source.schema.fields.findIndex((field) => field.name === column);
|
|
2868
|
+
if (index < 0) throw new Error(`Arrow projection column not found: ${column}`);
|
|
2869
|
+
if (typeof name !== "string" || typeof selection !== "string" && selection.as !== void 0 && !name.length || outputNames.has(name)) {
|
|
2870
|
+
throw new Error(`Invalid or duplicate Arrow projection output name: ${name}`);
|
|
2871
|
+
}
|
|
2872
|
+
outputNames.add(name);
|
|
2873
|
+
indices.push(index);
|
|
2874
|
+
return source.schema.fields[index].clone({ name });
|
|
2875
|
+
});
|
|
2876
|
+
const schema = new arrow6.Schema(fields, source.schema.metadata);
|
|
2877
|
+
const batches = source.batches.map(
|
|
2878
|
+
(batch) => new arrow6.RecordBatch(
|
|
2354
2879
|
schema,
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
function deducePrimitiveAwareSchema(table) {
|
|
2365
|
-
return {
|
|
2366
|
-
...table.schema,
|
|
2367
|
-
fields: table.schema.fields.map((field) => {
|
|
2368
|
-
if (field.type !== "null") {
|
|
2369
|
-
return field;
|
|
2370
|
-
}
|
|
2371
|
-
const columnValues = table.data.map((row) => row[field.name]);
|
|
2372
|
-
const inferredType = getDataTypeFromArray(columnValues);
|
|
2373
|
-
return { ...field, type: inferredType.type, nullable: inferredType.nullable };
|
|
2374
|
-
})
|
|
2375
|
-
};
|
|
2880
|
+
arrow6.makeData({
|
|
2881
|
+
type: new arrow6.Struct(fields),
|
|
2882
|
+
length: batch.numRows,
|
|
2883
|
+
children: indices.map((index) => batch.data.children[index])
|
|
2884
|
+
}),
|
|
2885
|
+
batch.metadata
|
|
2886
|
+
)
|
|
2887
|
+
);
|
|
2888
|
+
return new arrow6.Table(schema, batches);
|
|
2376
2889
|
}
|
|
2377
2890
|
|
|
2378
|
-
//
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
return
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
fields: columns.sort((left, right) => (left.ordinalPosition || 0) - (right.ordinalPosition || 0)).map((column) => ({
|
|
2393
|
-
name: column.columnName,
|
|
2394
|
-
type: mapSQLTypeToDataType(column.sqlType),
|
|
2395
|
-
nullable: column.nullable,
|
|
2396
|
-
metadata: {
|
|
2397
|
-
sqlType: column.sqlType
|
|
2398
|
-
}
|
|
2399
|
-
}))
|
|
2400
|
-
};
|
|
2891
|
+
// ../core/src/scan/arrow-execution/filter-arrow-table.ts
|
|
2892
|
+
var arrow7 = __toESM(require("apache-arrow"), 1);
|
|
2893
|
+
var import_arrow = require("@math.gl/arrow");
|
|
2894
|
+
function filterArrowTable(source, predicate, options = {}) {
|
|
2895
|
+
validateColumnarPredicate(predicate, new Set(source.schema.fields.map((field) => field.name)));
|
|
2896
|
+
const columnNames = getColumnarPredicateColumns(predicate);
|
|
2897
|
+
return filterArrowRows(
|
|
2898
|
+
source,
|
|
2899
|
+
(batch) => {
|
|
2900
|
+
const columns = getPredicateColumns(new arrow7.Table(source.schema, [batch]), columnNames);
|
|
2901
|
+
return (rowIndex) => evaluatePredicate(predicate, columns, rowIndex) === true;
|
|
2902
|
+
},
|
|
2903
|
+
options
|
|
2904
|
+
);
|
|
2401
2905
|
}
|
|
2402
|
-
function
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2906
|
+
function filterArrowRows(source, selectBatch, options = {}) {
|
|
2907
|
+
var _a, _b, _c;
|
|
2908
|
+
let remaining = options.limit ?? Infinity;
|
|
2909
|
+
if (remaining !== Infinity && (!Number.isSafeInteger(remaining) || remaining < 0)) {
|
|
2910
|
+
throw new Error("Arrow filter limit must be a non-negative safe integer.");
|
|
2911
|
+
}
|
|
2912
|
+
(_a = options.signal) == null ? void 0 : _a.throwIfAborted();
|
|
2913
|
+
const batches = [];
|
|
2914
|
+
for (const batch of source.batches) {
|
|
2915
|
+
if (remaining === 0) break;
|
|
2916
|
+
(_b = options.signal) == null ? void 0 : _b.throwIfAborted();
|
|
2917
|
+
const data = new arrow7.Table(source.schema, [batch]);
|
|
2918
|
+
const selectRow = selectBatch(batch);
|
|
2919
|
+
const indices = [];
|
|
2920
|
+
for (let rowIndex = 0; rowIndex < batch.numRows && indices.length < remaining; rowIndex++) {
|
|
2921
|
+
(_c = options.signal) == null ? void 0 : _c.throwIfAborted();
|
|
2922
|
+
if (selectRow(rowIndex)) indices.push(rowIndex);
|
|
2923
|
+
}
|
|
2924
|
+
remaining -= indices.length;
|
|
2925
|
+
if (indices.length === 0) continue;
|
|
2926
|
+
if (indices.length === batch.numRows) {
|
|
2927
|
+
batches.push(batch);
|
|
2928
|
+
continue;
|
|
2929
|
+
}
|
|
2930
|
+
if (indices[indices.length - 1] - indices[0] + 1 === indices.length) {
|
|
2931
|
+
batches.push(batch.slice(indices[0], indices[indices.length - 1] + 1));
|
|
2932
|
+
continue;
|
|
2933
|
+
}
|
|
2934
|
+
const gathered = (0, import_arrow.takeArrowTableRows)(data, Int32Array.from(indices));
|
|
2935
|
+
batches.push(
|
|
2936
|
+
...gathered.batches.map(
|
|
2937
|
+
(output) => new arrow7.RecordBatch(source.schema, output.data, batch.metadata)
|
|
2938
|
+
)
|
|
2939
|
+
);
|
|
2406
2940
|
}
|
|
2407
|
-
|
|
2408
|
-
|
|
2941
|
+
return new arrow7.Table(source.schema, batches);
|
|
2942
|
+
}
|
|
2943
|
+
function getPredicateColumns(sourceTable, columnNames) {
|
|
2944
|
+
const columns = {};
|
|
2945
|
+
for (const columnName of columnNames) {
|
|
2946
|
+
const column = sourceTable.getChild(columnName);
|
|
2947
|
+
if (!column) {
|
|
2948
|
+
throw new Error(`Arrow query column not found: ${columnName}`);
|
|
2949
|
+
}
|
|
2950
|
+
columns[columnName] = column;
|
|
2409
2951
|
}
|
|
2410
|
-
|
|
2411
|
-
|
|
2952
|
+
return columns;
|
|
2953
|
+
}
|
|
2954
|
+
function evaluatePredicate(predicate, columns, rowIndex) {
|
|
2955
|
+
if (isLogicalPredicate(predicate)) {
|
|
2956
|
+
const values = predicate.args.map((child) => evaluatePredicate(child, columns, rowIndex));
|
|
2957
|
+
return predicate.op === "and" ? evaluateAnd(values) : evaluateOr(values);
|
|
2412
2958
|
}
|
|
2413
|
-
if (
|
|
2414
|
-
|
|
2959
|
+
if (isNotPredicate(predicate)) {
|
|
2960
|
+
const value2 = evaluatePredicate(predicate.args[0], columns, rowIndex);
|
|
2961
|
+
return value2 === null ? null : !value2;
|
|
2415
2962
|
}
|
|
2416
|
-
|
|
2417
|
-
|
|
2963
|
+
const value = getColumnValue(columns, predicate.args[0].property, rowIndex);
|
|
2964
|
+
if (isNullPredicate(predicate)) {
|
|
2965
|
+
return value === null || value === void 0;
|
|
2418
2966
|
}
|
|
2419
|
-
if (
|
|
2420
|
-
return
|
|
2967
|
+
if (isInPredicate(predicate)) {
|
|
2968
|
+
return evaluateIn(value, predicate.args[1]);
|
|
2421
2969
|
}
|
|
2422
|
-
|
|
2423
|
-
|
|
2970
|
+
return evaluateComparison(value, predicate);
|
|
2971
|
+
}
|
|
2972
|
+
function getColumnValue(columns, columnName, rowIndex) {
|
|
2973
|
+
const path = typeof columnName === "string" ? [columnName] : columnName;
|
|
2974
|
+
const column = columns[path[0]];
|
|
2975
|
+
if (!column) {
|
|
2976
|
+
throw new Error(`Arrow query could not read predicate column "${columnName}".`);
|
|
2424
2977
|
}
|
|
2425
|
-
|
|
2426
|
-
|
|
2978
|
+
let value = column.get(rowIndex);
|
|
2979
|
+
for (const property of path.slice(1)) {
|
|
2980
|
+
if (value === null || value === void 0) return null;
|
|
2981
|
+
value = value[property];
|
|
2427
2982
|
}
|
|
2428
|
-
return
|
|
2429
|
-
}
|
|
2430
|
-
function getQualifiedTableName(options) {
|
|
2431
|
-
const identifiers = [options.catalogName, options.schemaName, options.tableName].filter((value) => Boolean(value));
|
|
2432
|
-
return identifiers.map((identifier) => quoteSqlIdentifier(identifier)).join(".");
|
|
2433
|
-
}
|
|
2434
|
-
|
|
2435
|
-
// ../core/src/scan/table-query-capabilities.ts
|
|
2436
|
-
var ARROW_TABLE_QUERY_CAPABILITIES = Object.freeze({
|
|
2437
|
-
projection: "residual",
|
|
2438
|
-
predicate: "residual",
|
|
2439
|
-
limit: "residual",
|
|
2440
|
-
streaming: false,
|
|
2441
|
-
cancellation: true,
|
|
2442
|
-
expressions: "residual",
|
|
2443
|
-
orderBy: "residual",
|
|
2444
|
-
aggregates: "residual"
|
|
2445
|
-
});
|
|
2446
|
-
var SQL_DATA_SOURCE_TABLE_QUERY_CAPABILITIES = Object.freeze({
|
|
2447
|
-
projection: "pushdown",
|
|
2448
|
-
predicate: "pushdown",
|
|
2449
|
-
limit: "pushdown",
|
|
2450
|
-
streaming: false,
|
|
2451
|
-
cancellation: false,
|
|
2452
|
-
expressions: "pushdown",
|
|
2453
|
-
orderBy: "pushdown",
|
|
2454
|
-
aggregates: "pushdown"
|
|
2455
|
-
});
|
|
2456
|
-
|
|
2457
|
-
// dist/sql-source.js
|
|
2458
|
-
var SQL_ADAPTER_FACTORIES = /* @__PURE__ */ new Map();
|
|
2459
|
-
function registerSQLAdapter(type, factory) {
|
|
2460
|
-
SQL_ADAPTER_FACTORIES.set(type, factory);
|
|
2983
|
+
return value;
|
|
2461
2984
|
}
|
|
2462
|
-
function
|
|
2463
|
-
|
|
2985
|
+
function evaluateAnd(values) {
|
|
2986
|
+
if (values.includes(false)) {
|
|
2987
|
+
return false;
|
|
2988
|
+
}
|
|
2989
|
+
return values.includes(null) ? null : true;
|
|
2464
2990
|
}
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
adapterOptions: {},
|
|
2469
|
-
browserAdapterFactory: void 0,
|
|
2470
|
-
nodeAdapterFactory: void 0
|
|
2471
|
-
},
|
|
2472
|
-
duckdb: {
|
|
2473
|
-
accessMode: "auto",
|
|
2474
|
-
bundles: void 0,
|
|
2475
|
-
databasePath: void 0,
|
|
2476
|
-
remoteUrl: void 0,
|
|
2477
|
-
workerUrl: void 0
|
|
2478
|
-
},
|
|
2479
|
-
snowflake: {
|
|
2480
|
-
account: void 0,
|
|
2481
|
-
database: void 0,
|
|
2482
|
-
schema: void 0,
|
|
2483
|
-
token: void 0,
|
|
2484
|
-
warehouse: void 0,
|
|
2485
|
-
role: void 0,
|
|
2486
|
-
authenticator: void 0
|
|
2991
|
+
function evaluateOr(values) {
|
|
2992
|
+
if (values.includes(true)) {
|
|
2993
|
+
return true;
|
|
2487
2994
|
}
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
adapterFactory;
|
|
2494
|
-
adapterPromise = null;
|
|
2495
|
-
metadataPromise = null;
|
|
2496
|
-
constructor(data, options, sourceType, defaultOptions = SQL_SOURCE_DEFAULT_OPTIONS, adapterFactory, coreApi) {
|
|
2497
|
-
super(data, options, defaultOptions, coreApi);
|
|
2498
|
-
this.sourceType = sourceType;
|
|
2499
|
-
this.adapterFactory = adapterFactory || getSQLAdapterFactory(sourceType);
|
|
2995
|
+
return values.includes(null) ? null : false;
|
|
2996
|
+
}
|
|
2997
|
+
function evaluateIn(value, candidates) {
|
|
2998
|
+
if (value === null || value === void 0) {
|
|
2999
|
+
return null;
|
|
2500
3000
|
}
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
this.metadataPromise = this.loadMetadata();
|
|
3001
|
+
for (const candidate of candidates) {
|
|
3002
|
+
if (compareValues(value, candidate) === 0) {
|
|
3003
|
+
return true;
|
|
2505
3004
|
}
|
|
2506
|
-
return this.metadataPromise;
|
|
2507
3005
|
}
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
3006
|
+
return false;
|
|
3007
|
+
}
|
|
3008
|
+
function evaluateComparison(value, predicate) {
|
|
3009
|
+
if (value === null || value === void 0) {
|
|
3010
|
+
return null;
|
|
2512
3011
|
}
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
3012
|
+
const comparison = compareValues(value, predicate.args[1]);
|
|
3013
|
+
switch (predicate.op) {
|
|
3014
|
+
case "=":
|
|
3015
|
+
return comparison === 0;
|
|
3016
|
+
case "<>":
|
|
3017
|
+
return comparison !== 0;
|
|
3018
|
+
case "<":
|
|
3019
|
+
return comparison < 0;
|
|
3020
|
+
case "<=":
|
|
3021
|
+
return comparison <= 0;
|
|
3022
|
+
case ">":
|
|
3023
|
+
return comparison > 0;
|
|
3024
|
+
case ">=":
|
|
3025
|
+
return comparison >= 0;
|
|
2517
3026
|
}
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
3027
|
+
}
|
|
3028
|
+
function compareValues(left, right) {
|
|
3029
|
+
if (isColumnarPredicateParameter(right)) {
|
|
3030
|
+
throw new Error(
|
|
3031
|
+
`Arrow query parameter ":${right.parameter}" must be bound before query execution.`
|
|
3032
|
+
);
|
|
2522
3033
|
}
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
const adapter = await this.getAdapter();
|
|
2526
|
-
return await adapter.getTableSchema(options);
|
|
3034
|
+
if (left instanceof Date && right instanceof Date) {
|
|
3035
|
+
return compareArrowQueryNumbers(left.getTime(), right.getTime());
|
|
2527
3036
|
}
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
const adapter = await this.getAdapter();
|
|
2531
|
-
const compiledQuery = this.compileQuery(query, options);
|
|
2532
|
-
return await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
|
|
3037
|
+
if (typeof left === "number" && right instanceof Date) {
|
|
3038
|
+
return compareArrowQueryNumbers(left, right.getTime());
|
|
2533
3039
|
}
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
const adapter = await this.getAdapter();
|
|
2537
|
-
const compiledQuery = this.compileQuery(query, options);
|
|
2538
|
-
if (adapter.executeArrow) {
|
|
2539
|
-
return await adapter.executeArrow(compiledQuery.sql, compiledQuery.options);
|
|
2540
|
-
}
|
|
2541
|
-
const rows = await adapter.executeRows(compiledQuery.sql, compiledQuery.options);
|
|
2542
|
-
return convertRowsToArrowTable(rows);
|
|
3040
|
+
if (left instanceof Uint8Array && right instanceof Uint8Array) {
|
|
3041
|
+
return compareBinary(left, right);
|
|
2543
3042
|
}
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
if (!this.adapterPromise) {
|
|
2547
|
-
return;
|
|
2548
|
-
}
|
|
2549
|
-
const adapter = await this.adapterPromise;
|
|
2550
|
-
await adapter.close();
|
|
2551
|
-
this.adapterPromise = null;
|
|
2552
|
-
this.metadataPromise = null;
|
|
3043
|
+
if (typeof left === "bigint" && typeof right === "bigint") {
|
|
3044
|
+
return left === right ? 0 : left < right ? -1 : 1;
|
|
2553
3045
|
}
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
return
|
|
3046
|
+
if (typeof left === "bigint" && typeof right === "number" || typeof left === "number" && typeof right === "bigint") {
|
|
3047
|
+
const numericValue = typeof left === "number" ? left : right;
|
|
3048
|
+
if (!Number.isFinite(numericValue))
|
|
3049
|
+
throw new Error("Arrow query cannot compare non-finite numbers.");
|
|
3050
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
2559
3051
|
}
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
if (typeof query === "string") {
|
|
2563
|
-
return { sql: query, options };
|
|
2564
|
-
}
|
|
2565
|
-
const namedParameters = options.parameters && !Array.isArray(options.parameters) ? options.parameters : void 0;
|
|
2566
|
-
const compiledQuery = compileSQLTableQuery(query, {
|
|
2567
|
-
dialect: getSQLTableQueryDialect(this.sourceType),
|
|
2568
|
-
parameters: namedParameters
|
|
2569
|
-
});
|
|
2570
|
-
return {
|
|
2571
|
-
sql: compiledQuery.sql,
|
|
2572
|
-
options: { signal: options.signal, parameters: compiledQuery.parameters }
|
|
2573
|
-
};
|
|
3052
|
+
if (typeof left === "number" && typeof right === "number") {
|
|
3053
|
+
return compareArrowQueryNumbers(left, right);
|
|
2574
3054
|
}
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
3055
|
+
if (typeof left === "string" && typeof right === "string") {
|
|
3056
|
+
return left === right ? 0 : left < right ? -1 : 1;
|
|
3057
|
+
}
|
|
3058
|
+
if (typeof left === "boolean" && typeof right === "boolean") {
|
|
3059
|
+
return Number(left) - Number(right);
|
|
3060
|
+
}
|
|
3061
|
+
throw new Error(
|
|
3062
|
+
`Arrow query cannot compare ${getArrowQueryValueType(left)} with ${getArrowQueryValueType(right)} values.`
|
|
3063
|
+
);
|
|
3064
|
+
}
|
|
3065
|
+
function compareArrowQueryNumbers(left, right) {
|
|
3066
|
+
if (!Number.isFinite(left) || !Number.isFinite(right)) {
|
|
3067
|
+
throw new Error("Arrow query cannot compare non-finite numbers.");
|
|
3068
|
+
}
|
|
3069
|
+
return left === right ? 0 : left < right ? -1 : 1;
|
|
3070
|
+
}
|
|
3071
|
+
function compareBinary(left, right) {
|
|
3072
|
+
const commonLength = Math.min(left.length, right.length);
|
|
3073
|
+
for (let index = 0; index < commonLength; index++) {
|
|
3074
|
+
if (left[index] !== right[index]) {
|
|
3075
|
+
return left[index] < right[index] ? -1 : 1;
|
|
2593
3076
|
}
|
|
2594
3077
|
}
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
adapter.listTables()
|
|
2601
|
-
]);
|
|
2602
|
-
return {
|
|
2603
|
-
type: this.sourceType,
|
|
2604
|
-
capabilities: adapter.capabilities,
|
|
2605
|
-
catalogs,
|
|
2606
|
-
schemas,
|
|
2607
|
-
tables
|
|
2608
|
-
};
|
|
3078
|
+
return left.length === right.length ? 0 : left.length < right.length ? -1 : 1;
|
|
3079
|
+
}
|
|
3080
|
+
function getArrowQueryValueType(value) {
|
|
3081
|
+
if (value instanceof Date) {
|
|
3082
|
+
return "Date";
|
|
2609
3083
|
}
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
3084
|
+
if (value instanceof Uint8Array) {
|
|
3085
|
+
return "Uint8Array";
|
|
3086
|
+
}
|
|
3087
|
+
return typeof value;
|
|
3088
|
+
}
|
|
3089
|
+
function isLogicalPredicate(predicate) {
|
|
3090
|
+
return predicate.op === "and" || predicate.op === "or";
|
|
3091
|
+
}
|
|
3092
|
+
function isNotPredicate(predicate) {
|
|
3093
|
+
return predicate.op === "not";
|
|
3094
|
+
}
|
|
3095
|
+
function isNullPredicate(predicate) {
|
|
3096
|
+
return predicate.op === "isNull";
|
|
3097
|
+
}
|
|
3098
|
+
function isInPredicate(predicate) {
|
|
3099
|
+
return predicate.op === "in";
|
|
2617
3100
|
}
|
|
2618
|
-
|
|
2619
|
-
// ../core/src/scan/query-arrow-table.ts
|
|
2620
|
-
var arrow6 = __toESM(require("apache-arrow"), 1);
|
|
2621
3101
|
|
|
2622
3102
|
// ../core/src/scan/table-query.ts
|
|
2623
3103
|
function planTableQuery2(sourceColumnNames, options = {}) {
|
|
@@ -2642,11 +3122,36 @@ function getSQLPredicateColumnNames(predicate) {
|
|
|
2642
3122
|
|
|
2643
3123
|
// ../core/src/scan/query-arrow-table.ts
|
|
2644
3124
|
function explainArrowTableQuery(sourceTable, options = {}) {
|
|
3125
|
+
if (options.select !== void 0) {
|
|
3126
|
+
if (options.columns !== void 0 || hasRelationalOperators(options) || options.join) {
|
|
3127
|
+
throw new Error("Arrow select cannot be combined with columns or relational operators.");
|
|
3128
|
+
}
|
|
3129
|
+
const projected = projectArrowTable(sourceTable, options.select);
|
|
3130
|
+
const columns = [
|
|
3131
|
+
...new Set(
|
|
3132
|
+
options.select.map(
|
|
3133
|
+
(selection) => typeof selection === "string" ? selection : selection.column
|
|
3134
|
+
)
|
|
3135
|
+
)
|
|
3136
|
+
];
|
|
3137
|
+
const { select, ...remainingOptions } = options;
|
|
3138
|
+
const explanation = explainArrowTableQuery(sourceTable, { ...remainingOptions, columns });
|
|
3139
|
+
const outputColumns = Object.freeze(projected.schema.fields.map((field) => field.name));
|
|
3140
|
+
return Object.freeze({
|
|
3141
|
+
...explanation,
|
|
3142
|
+
outputColumns,
|
|
3143
|
+
plan: Object.freeze(
|
|
3144
|
+
explanation.plan.map(
|
|
3145
|
+
(step) => step.kind === "project" ? Object.freeze({ ...step, columns: outputColumns, select }) : step
|
|
3146
|
+
)
|
|
3147
|
+
)
|
|
3148
|
+
});
|
|
3149
|
+
}
|
|
2645
3150
|
if (options.predicate) {
|
|
2646
3151
|
validateSQLPredicate(options.predicate);
|
|
2647
3152
|
}
|
|
2648
3153
|
return explainTableQuery(
|
|
2649
|
-
sourceTable.
|
|
3154
|
+
sourceTable.schema.fields.map((field) => field.name),
|
|
2650
3155
|
options,
|
|
2651
3156
|
ARROW_TABLE_QUERY_CAPABILITIES
|
|
2652
3157
|
);
|
|
@@ -2654,6 +3159,22 @@ function explainArrowTableQuery(sourceTable, options = {}) {
|
|
|
2654
3159
|
function queryArrowTable(sourceTable, options = {}) {
|
|
2655
3160
|
var _a, _b;
|
|
2656
3161
|
throwIfAborted(options.signal);
|
|
3162
|
+
if (options.select !== void 0) {
|
|
3163
|
+
if (options.columns !== void 0 || hasRelationalOperators(options) || options.join) {
|
|
3164
|
+
throw new Error("Arrow select cannot be combined with columns or relational operators.");
|
|
3165
|
+
}
|
|
3166
|
+
projectArrowTable(sourceTable, options.select);
|
|
3167
|
+
const columns = [
|
|
3168
|
+
...new Set(
|
|
3169
|
+
options.select.map(
|
|
3170
|
+
(selection) => typeof selection === "string" ? selection : selection.column
|
|
3171
|
+
)
|
|
3172
|
+
)
|
|
3173
|
+
];
|
|
3174
|
+
const { select, ...remainingOptions } = options;
|
|
3175
|
+
const result = queryArrowTable(sourceTable, { ...remainingOptions, columns });
|
|
3176
|
+
return projectArrowTable(result, select);
|
|
3177
|
+
}
|
|
2657
3178
|
if (options.join) {
|
|
2658
3179
|
validateArrowJoinQuery(sourceTable, options);
|
|
2659
3180
|
}
|
|
@@ -2666,56 +3187,37 @@ function queryArrowTable(sourceTable, options = {}) {
|
|
|
2666
3187
|
if (hasRelationalOperators(options)) {
|
|
2667
3188
|
return queryArrowTableRelational(sourceTable, options);
|
|
2668
3189
|
}
|
|
2669
|
-
const sourceData = sourceTable
|
|
3190
|
+
const sourceData = sourceTable;
|
|
2670
3191
|
const plan = planTableQuery2(
|
|
2671
3192
|
sourceData.schema.fields.map((field) => field.name),
|
|
2672
3193
|
options
|
|
2673
3194
|
);
|
|
2674
3195
|
const scannedData = sourceData.select([...getScanStep(plan).columns]);
|
|
2675
|
-
const projectedData = scannedData.select([...getProjectStep(plan).columns]);
|
|
2676
3196
|
const predicate = (_a = getFilterStep(plan)) == null ? void 0 : _a.predicate;
|
|
2677
3197
|
const limit = ((_b = getLimitStep(plan)) == null ? void 0 : _b.limit) ?? Number.POSITIVE_INFINITY;
|
|
2678
3198
|
if (!predicate) {
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
const predicateColumns = getPredicateColumns(scannedData, getSQLPredicateColumnNames(predicate));
|
|
2682
|
-
const matchingRowIndices = [];
|
|
2683
|
-
for (let rowIndex = 0; rowIndex < scannedData.numRows && matchingRowIndices.length < limit; rowIndex++) {
|
|
2684
|
-
throwIfAborted(options.signal);
|
|
2685
|
-
if (evaluatePredicate(predicate, predicateColumns, rowIndex) === true) {
|
|
2686
|
-
matchingRowIndices.push(rowIndex);
|
|
2687
|
-
}
|
|
2688
|
-
}
|
|
2689
|
-
if (matchingRowIndices.length === 0) {
|
|
2690
|
-
return wrapArrowTable(projectedData.slice(0, 0));
|
|
2691
|
-
}
|
|
2692
|
-
const outputColumns = {};
|
|
2693
|
-
for (const field of projectedData.schema.fields) {
|
|
2694
|
-
const sourceColumn = projectedData.getChild(field.name);
|
|
2695
|
-
if (!sourceColumn) {
|
|
2696
|
-
throw new Error(`Arrow query could not read column "${field.name}".`);
|
|
2697
|
-
}
|
|
2698
|
-
const values = matchingRowIndices.map((rowIndex) => sourceColumn.get(rowIndex));
|
|
2699
|
-
outputColumns[field.name] = arrow6.vectorFromArray(values, sourceColumn.type);
|
|
3199
|
+
const projectedData = projectArrowTable(scannedData, getProjectStep(plan).columns);
|
|
3200
|
+
return projectedData.numRows === 0 ? projectedData : projectedData.slice(0, limit);
|
|
2700
3201
|
}
|
|
2701
|
-
|
|
3202
|
+
const filteredData = filterArrowTable(scannedData, predicate, { limit, signal: options.signal });
|
|
3203
|
+
return projectArrowTable(filteredData, getProjectStep(plan).columns);
|
|
2702
3204
|
}
|
|
2703
3205
|
function queryArrowTableRelational(sourceTable, options) {
|
|
2704
3206
|
var _a, _b, _c, _d, _e, _f;
|
|
2705
|
-
const sourceData = sourceTable
|
|
3207
|
+
const sourceData = sourceTable;
|
|
2706
3208
|
const sourceColumnNames = sourceData.schema.fields.map((field) => field.name);
|
|
2707
3209
|
const relationalPlan = planRelationalQuery(sourceColumnNames, options);
|
|
2708
3210
|
const scanStep = relationalPlan.tablePlan.find((step) => step.kind === "scan");
|
|
2709
3211
|
if (!scanStep || scanStep.kind !== "scan") {
|
|
2710
3212
|
throw new Error("Arrow relational query plan is missing a scan step.");
|
|
2711
3213
|
}
|
|
2712
|
-
const scannedTable =
|
|
3214
|
+
const scannedTable = sourceData.select([...scanStep.columns]);
|
|
2713
3215
|
const filteredTable = queryArrowTable(scannedTable, {
|
|
2714
3216
|
predicate: options.predicate,
|
|
2715
3217
|
columns: scanStep.columns,
|
|
2716
3218
|
signal: options.signal
|
|
2717
3219
|
});
|
|
2718
|
-
let rows = arrowTableToRows(filteredTable
|
|
3220
|
+
let rows = arrowTableToRows(filteredTable);
|
|
2719
3221
|
for (const child of options.union || []) {
|
|
2720
3222
|
const childTable = (_a = options.tables) == null ? void 0 : _a[child.source];
|
|
2721
3223
|
if (!childTable) throw new Error(`Arrow union source not found: ${child.source}`);
|
|
@@ -2724,7 +3226,7 @@ function queryArrowTableRelational(sourceTable, options) {
|
|
|
2724
3226
|
columns: getUnionChildColumns(sourceColumnNames, options),
|
|
2725
3227
|
signal: options.signal
|
|
2726
3228
|
});
|
|
2727
|
-
rows = rows.concat(arrowTableToRows(childResult
|
|
3229
|
+
rows = rows.concat(arrowTableToRows(childResult));
|
|
2728
3230
|
}
|
|
2729
3231
|
for (const expression of options.expressions || []) {
|
|
2730
3232
|
for (const row of rows) {
|
|
@@ -2807,12 +3309,12 @@ function queryArrowJoin(sourceTable, options) {
|
|
|
2807
3309
|
...join.child.query || {},
|
|
2808
3310
|
signal: options.signal
|
|
2809
3311
|
});
|
|
2810
|
-
const leftRows = arrowTableToRows(leftResult
|
|
2811
|
-
const rightRows = arrowTableToRows(rightResult
|
|
3312
|
+
const leftRows = arrowTableToRows(leftResult);
|
|
3313
|
+
const rightRows = arrowTableToRows(rightResult);
|
|
2812
3314
|
const rightPrefix = `${join.child.source}.`;
|
|
2813
3315
|
const defaultOutputColumns = [
|
|
2814
|
-
...sourceTable.
|
|
2815
|
-
...rightResult.
|
|
3316
|
+
...sourceTable.schema.fields.map((field) => field.name),
|
|
3317
|
+
...rightResult.schema.fields.map((field) => `${rightPrefix}${field.name}`)
|
|
2816
3318
|
];
|
|
2817
3319
|
const rows = [];
|
|
2818
3320
|
for (const leftRow of leftRows) {
|
|
@@ -2833,13 +3335,13 @@ function queryArrowJoin(sourceTable, options) {
|
|
|
2833
3335
|
);
|
|
2834
3336
|
const limitedRows = outputRows.slice(0, options.limit);
|
|
2835
3337
|
return limitedRows.length ? convertRowsToArrowTable(limitedRows) : createEmptyRelationalResult(
|
|
2836
|
-
sourceTable
|
|
3338
|
+
sourceTable,
|
|
2837
3339
|
outputColumns,
|
|
2838
3340
|
options,
|
|
2839
3341
|
new Map(
|
|
2840
|
-
rightResult.
|
|
3342
|
+
rightResult.schema.fields.map((field) => [
|
|
2841
3343
|
`${rightPrefix}${field.name}`,
|
|
2842
|
-
new
|
|
3344
|
+
new arrow8.Field(
|
|
2843
3345
|
`${rightPrefix}${field.name}`,
|
|
2844
3346
|
field.type,
|
|
2845
3347
|
field.nullable,
|
|
@@ -2853,10 +3355,10 @@ function validateArrowJoinQuery(sourceTable, options) {
|
|
|
2853
3355
|
var _a, _b, _c;
|
|
2854
3356
|
const join = options.join;
|
|
2855
3357
|
if (!join) return;
|
|
2856
|
-
const sourceColumnNames = sourceTable.
|
|
3358
|
+
const sourceColumnNames = sourceTable.schema.fields.map((field) => field.name);
|
|
2857
3359
|
const childTable = (_a = options.tables) == null ? void 0 : _a[join.child.source];
|
|
2858
3360
|
if (!childTable) throw new Error(`Arrow join source not found: ${join.child.source}`);
|
|
2859
|
-
const childColumnNames = childTable.
|
|
3361
|
+
const childColumnNames = childTable.schema.fields.map((field) => field.name);
|
|
2860
3362
|
if (!sourceColumnNames.includes(join.left)) {
|
|
2861
3363
|
throw new Error(`Arrow join column not found: ${join.left}`);
|
|
2862
3364
|
}
|
|
@@ -2934,267 +3436,115 @@ function compareRows(left, right, orderBy) {
|
|
|
2934
3436
|
const rightValue = right[key.column];
|
|
2935
3437
|
const nulls = key.nulls || "last";
|
|
2936
3438
|
if (leftValue === null || leftValue === void 0 || rightValue === null || rightValue === void 0) {
|
|
2937
|
-
if (leftValue === rightValue) continue;
|
|
2938
|
-
const nullResult = leftValue === null || leftValue === void 0 ? -1 : 1;
|
|
2939
|
-
return nulls === "first" ? nullResult : -nullResult;
|
|
2940
|
-
}
|
|
2941
|
-
const result = compareSortValues(leftValue, rightValue);
|
|
2942
|
-
if (result) return (key.direction === "desc" ? -1 : 1) * result;
|
|
2943
|
-
}
|
|
2944
|
-
return 0;
|
|
2945
|
-
}
|
|
2946
|
-
function compareSortValues(left, right) {
|
|
2947
|
-
if (left instanceof Date && right instanceof Date)
|
|
2948
|
-
return compareNumbers(left.getTime(), right.getTime());
|
|
2949
|
-
if (typeof left === "bigint" && typeof right === "bigint")
|
|
2950
|
-
return left === right ? 0 : left < right ? -1 : 1;
|
|
2951
|
-
if (typeof left === "number" && typeof right === "number") return compareNumbers(left, right);
|
|
2952
|
-
if (typeof left === "string" && typeof right === "string")
|
|
2953
|
-
return left === right ? 0 : left < right ? -1 : 1;
|
|
2954
|
-
if (typeof left === "boolean" && typeof right === "boolean") return Number(left) - Number(right);
|
|
2955
|
-
throw new Error(
|
|
2956
|
-
`Arrow relational order cannot compare ${getValueType(left)} with ${getValueType(right)}.`
|
|
2957
|
-
);
|
|
2958
|
-
}
|
|
2959
|
-
function aggregateRows(rows, groupBy, aggregates) {
|
|
2960
|
-
const groups = /* @__PURE__ */ new Map();
|
|
2961
|
-
for (const row of rows) {
|
|
2962
|
-
const key = serializeGroupKey(groupBy.map((column) => row[column]));
|
|
2963
|
-
const group = groups.get(key) || [];
|
|
2964
|
-
group.push(row);
|
|
2965
|
-
groups.set(key, group);
|
|
2966
|
-
}
|
|
2967
|
-
if (!groupBy.length && !rows.length) groups.set("[]", []);
|
|
2968
|
-
return [...groups.values()].map((group) => {
|
|
2969
|
-
var _a;
|
|
2970
|
-
const output = {};
|
|
2971
|
-
for (const column of groupBy) output[column] = ((_a = group[0]) == null ? void 0 : _a[column]) ?? null;
|
|
2972
|
-
for (const aggregate of aggregates)
|
|
2973
|
-
output[aggregate.name] = evaluateAggregate(aggregate, group);
|
|
2974
|
-
return output;
|
|
2975
|
-
});
|
|
2976
|
-
}
|
|
2977
|
-
function serializeGroupKey(values) {
|
|
2978
|
-
return values.map((value) => {
|
|
2979
|
-
if (value === null || value === void 0) return "null:";
|
|
2980
|
-
if (value instanceof Date) return `date:${value.getTime()}`;
|
|
2981
|
-
if (value instanceof Uint8Array) return `bytes:${Array.from(value).join(".")}`;
|
|
2982
|
-
return `${typeof value}:${String(value)}`;
|
|
2983
|
-
}).join("");
|
|
2984
|
-
}
|
|
2985
|
-
function createEmptyRelationalResult(sourceData, outputColumns, options, additionalFields) {
|
|
2986
|
-
const sourceFields = new Map(sourceData.schema.fields.map((field) => [field.name, field]));
|
|
2987
|
-
const expressions = new Map(
|
|
2988
|
-
(options.expressions || []).map((expression) => [expression.name, expression])
|
|
2989
|
-
);
|
|
2990
|
-
const aggregates = new Map(
|
|
2991
|
-
(options.aggregates || []).map((aggregate) => [aggregate.name, aggregate])
|
|
2992
|
-
);
|
|
2993
|
-
const fields = outputColumns.map((columnName) => {
|
|
2994
|
-
const sourceField = sourceFields.get(columnName);
|
|
2995
|
-
if (sourceField) return sourceField;
|
|
2996
|
-
const additionalField = additionalFields == null ? void 0 : additionalFields.get(columnName);
|
|
2997
|
-
if (additionalField) return additionalField;
|
|
2998
|
-
const expression = expressions.get(columnName);
|
|
2999
|
-
if (expression)
|
|
3000
|
-
return new arrow6.Field(columnName, getExpressionType(expression, sourceFields), true);
|
|
3001
|
-
const aggregate = aggregates.get(columnName);
|
|
3002
|
-
if (aggregate)
|
|
3003
|
-
return new arrow6.Field(columnName, getAggregateType(aggregate, sourceFields), true);
|
|
3004
|
-
return new arrow6.Field(columnName, new arrow6.Utf8(), true);
|
|
3005
|
-
});
|
|
3006
|
-
const columns = {};
|
|
3007
|
-
for (const field of fields) columns[field.name] = arrow6.vectorFromArray([], field.type);
|
|
3008
|
-
return wrapArrowTable(new arrow6.Table(new arrow6.Schema(fields), columns));
|
|
3009
|
-
}
|
|
3010
|
-
function getExpressionType(expression, sourceFields) {
|
|
3011
|
-
var _a;
|
|
3012
|
-
if (expression.expression.op === "literal") {
|
|
3013
|
-
const value = expression.expression.value;
|
|
3014
|
-
if (typeof value === "boolean") return new arrow6.Bool();
|
|
3015
|
-
if (typeof value === "number") return new arrow6.Float64();
|
|
3016
|
-
return new arrow6.Utf8();
|
|
3017
|
-
}
|
|
3018
|
-
if (expression.expression.op === "column") {
|
|
3019
|
-
return ((_a = sourceFields.get(expression.expression.column)) == null ? void 0 : _a.type) || new arrow6.Utf8();
|
|
3020
|
-
}
|
|
3021
|
-
return new arrow6.Float64();
|
|
3022
|
-
}
|
|
3023
|
-
function getAggregateType(aggregate, sourceFields) {
|
|
3024
|
-
var _a;
|
|
3025
|
-
if (aggregate.function === "count") return new arrow6.Int32();
|
|
3026
|
-
return aggregate.column ? ((_a = sourceFields.get(aggregate.column)) == null ? void 0 : _a.type) || new arrow6.Float64() : new arrow6.Float64();
|
|
3027
|
-
}
|
|
3028
|
-
function evaluateAggregate(aggregate, rows) {
|
|
3029
|
-
if (aggregate.function === "count")
|
|
3030
|
-
return aggregate.column ? rows.filter((row) => row[aggregate.column] != null).length : rows.length;
|
|
3031
|
-
const values = rows.map((row) => row[aggregate.column]).filter((value) => value != null);
|
|
3032
|
-
if (!values.length) return null;
|
|
3033
|
-
if (!values.every((value) => typeof value === "number"))
|
|
3034
|
-
throw new Error(`Arrow aggregate ${aggregate.function} requires numeric values.`);
|
|
3035
|
-
const numericValues = values;
|
|
3036
|
-
switch (aggregate.function) {
|
|
3037
|
-
case "sum":
|
|
3038
|
-
return numericValues.reduce((sum, value) => sum + value, 0);
|
|
3039
|
-
case "avg":
|
|
3040
|
-
return numericValues.reduce((sum, value) => sum + value, 0) / numericValues.length;
|
|
3041
|
-
case "min":
|
|
3042
|
-
return Math.min(...numericValues);
|
|
3043
|
-
case "max":
|
|
3044
|
-
return Math.max(...numericValues);
|
|
3045
|
-
}
|
|
3046
|
-
}
|
|
3047
|
-
function getPredicateColumns(sourceTable, columnNames) {
|
|
3048
|
-
const columns = {};
|
|
3049
|
-
for (const columnName of columnNames) {
|
|
3050
|
-
const column = sourceTable.getChild(columnName);
|
|
3051
|
-
if (!column) {
|
|
3052
|
-
throw new Error(`Arrow query column not found: ${columnName}`);
|
|
3053
|
-
}
|
|
3054
|
-
columns[columnName] = column;
|
|
3055
|
-
}
|
|
3056
|
-
return columns;
|
|
3057
|
-
}
|
|
3058
|
-
function evaluatePredicate(predicate, columns, rowIndex) {
|
|
3059
|
-
if (isLogicalPredicate(predicate)) {
|
|
3060
|
-
const values = predicate.args.map((child) => evaluatePredicate(child, columns, rowIndex));
|
|
3061
|
-
return predicate.op === "and" ? evaluateAnd(values) : evaluateOr(values);
|
|
3062
|
-
}
|
|
3063
|
-
if (isNotPredicate(predicate)) {
|
|
3064
|
-
const value2 = evaluatePredicate(predicate.args[0], columns, rowIndex);
|
|
3065
|
-
return value2 === null ? null : !value2;
|
|
3066
|
-
}
|
|
3067
|
-
const value = getColumnValue(columns, predicate.args[0].property, rowIndex);
|
|
3068
|
-
if (isNullPredicate(predicate)) {
|
|
3069
|
-
return value === null;
|
|
3070
|
-
}
|
|
3071
|
-
if (isInPredicate(predicate)) {
|
|
3072
|
-
return evaluateIn(value, predicate.args[1]);
|
|
3073
|
-
}
|
|
3074
|
-
return evaluateComparison(value, predicate);
|
|
3075
|
-
}
|
|
3076
|
-
function getColumnValue(columns, columnName, rowIndex) {
|
|
3077
|
-
const column = columns[columnName];
|
|
3078
|
-
if (!column) {
|
|
3079
|
-
throw new Error(`Arrow query could not read predicate column "${columnName}".`);
|
|
3080
|
-
}
|
|
3081
|
-
return column.get(rowIndex);
|
|
3082
|
-
}
|
|
3083
|
-
function evaluateAnd(values) {
|
|
3084
|
-
if (values.includes(false)) {
|
|
3085
|
-
return false;
|
|
3086
|
-
}
|
|
3087
|
-
return values.includes(null) ? null : true;
|
|
3088
|
-
}
|
|
3089
|
-
function evaluateOr(values) {
|
|
3090
|
-
if (values.includes(true)) {
|
|
3091
|
-
return true;
|
|
3092
|
-
}
|
|
3093
|
-
return values.includes(null) ? null : false;
|
|
3094
|
-
}
|
|
3095
|
-
function evaluateIn(value, candidates) {
|
|
3096
|
-
if (value === null || value === void 0) {
|
|
3097
|
-
return null;
|
|
3098
|
-
}
|
|
3099
|
-
for (const candidate of candidates) {
|
|
3100
|
-
if (compareValues(value, candidate) === 0) {
|
|
3101
|
-
return true;
|
|
3102
|
-
}
|
|
3103
|
-
}
|
|
3104
|
-
return false;
|
|
3105
|
-
}
|
|
3106
|
-
function evaluateComparison(value, predicate) {
|
|
3107
|
-
if (value === null || value === void 0) {
|
|
3108
|
-
return null;
|
|
3109
|
-
}
|
|
3110
|
-
const comparison = compareValues(value, predicate.args[1]);
|
|
3111
|
-
switch (predicate.op) {
|
|
3112
|
-
case "=":
|
|
3113
|
-
return comparison === 0;
|
|
3114
|
-
case "<>":
|
|
3115
|
-
return comparison !== 0;
|
|
3116
|
-
case "<":
|
|
3117
|
-
return comparison < 0;
|
|
3118
|
-
case "<=":
|
|
3119
|
-
return comparison <= 0;
|
|
3120
|
-
case ">":
|
|
3121
|
-
return comparison > 0;
|
|
3122
|
-
case ">=":
|
|
3123
|
-
return comparison >= 0;
|
|
3124
|
-
}
|
|
3125
|
-
}
|
|
3126
|
-
function compareValues(left, right) {
|
|
3127
|
-
if (isSQLPredicateParameter(right)) {
|
|
3128
|
-
throw new Error(
|
|
3129
|
-
`Arrow query parameter ":${right.parameter}" must be bound before query execution.`
|
|
3130
|
-
);
|
|
3131
|
-
}
|
|
3132
|
-
if (left instanceof Date && right instanceof Date) {
|
|
3133
|
-
return compareNumbers(left.getTime(), right.getTime());
|
|
3134
|
-
}
|
|
3135
|
-
if (typeof left === "number" && right instanceof Date) {
|
|
3136
|
-
return compareNumbers(left, right.getTime());
|
|
3137
|
-
}
|
|
3138
|
-
if (left instanceof Uint8Array && right instanceof Uint8Array) {
|
|
3139
|
-
return compareBinary(left, right);
|
|
3439
|
+
if (leftValue === rightValue) continue;
|
|
3440
|
+
const nullResult = leftValue === null || leftValue === void 0 ? -1 : 1;
|
|
3441
|
+
return nulls === "first" ? nullResult : -nullResult;
|
|
3442
|
+
}
|
|
3443
|
+
const result = compareSortValues(leftValue, rightValue);
|
|
3444
|
+
if (result) return (key.direction === "desc" ? -1 : 1) * result;
|
|
3140
3445
|
}
|
|
3141
|
-
|
|
3446
|
+
return 0;
|
|
3447
|
+
}
|
|
3448
|
+
function compareSortValues(left, right) {
|
|
3449
|
+
if (left instanceof Date && right instanceof Date)
|
|
3450
|
+
return compareArrowQueryNumbers(left.getTime(), right.getTime());
|
|
3451
|
+
if (typeof left === "bigint" && typeof right === "bigint")
|
|
3142
3452
|
return left === right ? 0 : left < right ? -1 : 1;
|
|
3143
|
-
|
|
3144
|
-
if (typeof left === "
|
|
3145
|
-
return compareValues(left, BigInt(right));
|
|
3146
|
-
}
|
|
3147
|
-
if (typeof left === "number" && typeof right === "bigint" && Number.isSafeInteger(left)) {
|
|
3148
|
-
return compareValues(BigInt(left), right);
|
|
3149
|
-
}
|
|
3150
|
-
if (typeof left === "number" && typeof right === "number") {
|
|
3151
|
-
return compareNumbers(left, right);
|
|
3152
|
-
}
|
|
3153
|
-
if (typeof left === "string" && typeof right === "string") {
|
|
3453
|
+
if (typeof left === "number" && typeof right === "number") return compareArrowQueryNumbers(left, right);
|
|
3454
|
+
if (typeof left === "string" && typeof right === "string")
|
|
3154
3455
|
return left === right ? 0 : left < right ? -1 : 1;
|
|
3155
|
-
|
|
3156
|
-
if (typeof left === "boolean" && typeof right === "boolean") {
|
|
3157
|
-
return Number(left) - Number(right);
|
|
3158
|
-
}
|
|
3456
|
+
if (typeof left === "boolean" && typeof right === "boolean") return Number(left) - Number(right);
|
|
3159
3457
|
throw new Error(
|
|
3160
|
-
`Arrow
|
|
3458
|
+
`Arrow relational order cannot compare ${getArrowQueryValueType(left)} with ${getArrowQueryValueType(right)}.`
|
|
3161
3459
|
);
|
|
3162
3460
|
}
|
|
3163
|
-
function
|
|
3164
|
-
|
|
3165
|
-
|
|
3461
|
+
function aggregateRows(rows, groupBy, aggregates) {
|
|
3462
|
+
const groups = /* @__PURE__ */ new Map();
|
|
3463
|
+
for (const row of rows) {
|
|
3464
|
+
const key = serializeGroupKey(groupBy.map((column) => row[column]));
|
|
3465
|
+
const group = groups.get(key) || [];
|
|
3466
|
+
group.push(row);
|
|
3467
|
+
groups.set(key, group);
|
|
3166
3468
|
}
|
|
3167
|
-
|
|
3469
|
+
if (!groupBy.length && !rows.length) groups.set("[]", []);
|
|
3470
|
+
return [...groups.values()].map((group) => {
|
|
3471
|
+
var _a;
|
|
3472
|
+
const output = {};
|
|
3473
|
+
for (const column of groupBy) output[column] = ((_a = group[0]) == null ? void 0 : _a[column]) ?? null;
|
|
3474
|
+
for (const aggregate of aggregates)
|
|
3475
|
+
output[aggregate.name] = evaluateAggregate(aggregate, group);
|
|
3476
|
+
return output;
|
|
3477
|
+
});
|
|
3168
3478
|
}
|
|
3169
|
-
function
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
if (
|
|
3173
|
-
|
|
3174
|
-
}
|
|
3175
|
-
}
|
|
3176
|
-
return left.length === right.length ? 0 : left.length < right.length ? -1 : 1;
|
|
3479
|
+
function serializeGroupKey(values) {
|
|
3480
|
+
return values.map((value) => {
|
|
3481
|
+
if (value === null || value === void 0) return "null:";
|
|
3482
|
+
if (value instanceof Date) return `date:${value.getTime()}`;
|
|
3483
|
+
if (value instanceof Uint8Array) return `bytes:${Array.from(value).join(".")}`;
|
|
3484
|
+
return `${typeof value}:${String(value)}`;
|
|
3485
|
+
}).join("");
|
|
3177
3486
|
}
|
|
3178
|
-
function
|
|
3179
|
-
|
|
3180
|
-
|
|
3487
|
+
function createEmptyRelationalResult(sourceData, outputColumns, options, additionalFields) {
|
|
3488
|
+
const sourceFields = new Map(sourceData.schema.fields.map((field) => [field.name, field]));
|
|
3489
|
+
const expressions = new Map(
|
|
3490
|
+
(options.expressions || []).map((expression) => [expression.name, expression])
|
|
3491
|
+
);
|
|
3492
|
+
const aggregates = new Map(
|
|
3493
|
+
(options.aggregates || []).map((aggregate) => [aggregate.name, aggregate])
|
|
3494
|
+
);
|
|
3495
|
+
const fields = outputColumns.map((columnName) => {
|
|
3496
|
+
const sourceField = sourceFields.get(columnName);
|
|
3497
|
+
if (sourceField) return sourceField;
|
|
3498
|
+
const additionalField = additionalFields == null ? void 0 : additionalFields.get(columnName);
|
|
3499
|
+
if (additionalField) return additionalField;
|
|
3500
|
+
const expression = expressions.get(columnName);
|
|
3501
|
+
if (expression)
|
|
3502
|
+
return new arrow8.Field(columnName, getExpressionType(expression, sourceFields), true);
|
|
3503
|
+
const aggregate = aggregates.get(columnName);
|
|
3504
|
+
if (aggregate)
|
|
3505
|
+
return new arrow8.Field(columnName, getAggregateType(aggregate, sourceFields), true);
|
|
3506
|
+
return new arrow8.Field(columnName, new arrow8.Utf8(), true);
|
|
3507
|
+
});
|
|
3508
|
+
const columns = {};
|
|
3509
|
+
for (const field of fields) columns[field.name] = arrow8.vectorFromArray([], field.type);
|
|
3510
|
+
return new arrow8.Table(new arrow8.Schema(fields), columns);
|
|
3511
|
+
}
|
|
3512
|
+
function getExpressionType(expression, sourceFields) {
|
|
3513
|
+
var _a;
|
|
3514
|
+
if (expression.expression.op === "literal") {
|
|
3515
|
+
const value = expression.expression.value;
|
|
3516
|
+
if (typeof value === "boolean") return new arrow8.Bool();
|
|
3517
|
+
if (typeof value === "number") return new arrow8.Float64();
|
|
3518
|
+
return new arrow8.Utf8();
|
|
3181
3519
|
}
|
|
3182
|
-
if (
|
|
3183
|
-
return
|
|
3520
|
+
if (expression.expression.op === "column") {
|
|
3521
|
+
return ((_a = sourceFields.get(expression.expression.column)) == null ? void 0 : _a.type) || new arrow8.Utf8();
|
|
3184
3522
|
}
|
|
3185
|
-
return
|
|
3186
|
-
}
|
|
3187
|
-
function isLogicalPredicate(predicate) {
|
|
3188
|
-
return predicate.op === "and" || predicate.op === "or";
|
|
3523
|
+
return new arrow8.Float64();
|
|
3189
3524
|
}
|
|
3190
|
-
function
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
return predicate.op === "isNull";
|
|
3525
|
+
function getAggregateType(aggregate, sourceFields) {
|
|
3526
|
+
var _a;
|
|
3527
|
+
if (aggregate.function === "count") return new arrow8.Int32();
|
|
3528
|
+
return aggregate.column ? ((_a = sourceFields.get(aggregate.column)) == null ? void 0 : _a.type) || new arrow8.Float64() : new arrow8.Float64();
|
|
3195
3529
|
}
|
|
3196
|
-
function
|
|
3197
|
-
|
|
3530
|
+
function evaluateAggregate(aggregate, rows) {
|
|
3531
|
+
if (aggregate.function === "count")
|
|
3532
|
+
return aggregate.column ? rows.filter((row) => row[aggregate.column] != null).length : rows.length;
|
|
3533
|
+
const values = rows.map((row) => row[aggregate.column]).filter((value) => value != null);
|
|
3534
|
+
if (!values.length) return null;
|
|
3535
|
+
if (!values.every((value) => typeof value === "number"))
|
|
3536
|
+
throw new Error(`Arrow aggregate ${aggregate.function} requires numeric values.`);
|
|
3537
|
+
const numericValues = values;
|
|
3538
|
+
switch (aggregate.function) {
|
|
3539
|
+
case "sum":
|
|
3540
|
+
return numericValues.reduce((sum, value) => sum + value, 0);
|
|
3541
|
+
case "avg":
|
|
3542
|
+
return numericValues.reduce((sum, value) => sum + value, 0) / numericValues.length;
|
|
3543
|
+
case "min":
|
|
3544
|
+
return Math.min(...numericValues);
|
|
3545
|
+
case "max":
|
|
3546
|
+
return Math.max(...numericValues);
|
|
3547
|
+
}
|
|
3198
3548
|
}
|
|
3199
3549
|
function getScanStep(plan) {
|
|
3200
3550
|
const step = plan.find((candidate) => candidate.kind === "scan");
|
|
@@ -3225,13 +3575,6 @@ function throwIfAborted(signal) {
|
|
|
3225
3575
|
throw error;
|
|
3226
3576
|
}
|
|
3227
3577
|
}
|
|
3228
|
-
function wrapArrowTable(data) {
|
|
3229
|
-
return {
|
|
3230
|
-
shape: "arrow-table",
|
|
3231
|
-
schema: convertArrowToSchema(data.schema),
|
|
3232
|
-
data
|
|
3233
|
-
};
|
|
3234
|
-
}
|
|
3235
3578
|
|
|
3236
3579
|
// dist/arrow-table-source.js
|
|
3237
3580
|
var ArrowTableSource = class extends DataSource {
|
|
@@ -3242,15 +3585,15 @@ var ArrowTableSource = class extends DataSource {
|
|
|
3242
3585
|
/** Discovers query-visible fields and capabilities without evaluating any rows. */
|
|
3243
3586
|
async getQueryMetadata(options = {}) {
|
|
3244
3587
|
throwIfAborted2(options.signal);
|
|
3245
|
-
const schema = convertArrowToSchema(this.data.
|
|
3588
|
+
const schema = convertArrowToSchema(this.data.schema);
|
|
3246
3589
|
return createScanQueryMetadata({
|
|
3247
|
-
sourceType: "arrow
|
|
3590
|
+
sourceType: "arrow",
|
|
3248
3591
|
queryType: "table",
|
|
3249
3592
|
execution: { status: "supported", method: "read" },
|
|
3250
3593
|
schema,
|
|
3251
3594
|
capabilities: { table: ARROW_TABLE_QUERY_CAPABILITIES },
|
|
3252
3595
|
columnRoles: getArrowColumnRoles(schema.fields.map((field) => field.name)),
|
|
3253
|
-
statistics: { rowCount: this.data.
|
|
3596
|
+
statistics: { rowCount: this.data.numRows }
|
|
3254
3597
|
});
|
|
3255
3598
|
}
|
|
3256
3599
|
/** Executes a portable query and returns an Arrow table result. */
|
|
@@ -3263,8 +3606,11 @@ var ArrowTableSource = class extends DataSource {
|
|
|
3263
3606
|
}
|
|
3264
3607
|
/** Executes a query as one bounded Arrow batch. */
|
|
3265
3608
|
async *read(options = {}) {
|
|
3266
|
-
const
|
|
3267
|
-
|
|
3609
|
+
const data = this.query(options);
|
|
3610
|
+
if (data.batches.length)
|
|
3611
|
+
yield* data.batches;
|
|
3612
|
+
else
|
|
3613
|
+
yield makeArrowMetadataBatch(data.schema);
|
|
3268
3614
|
}
|
|
3269
3615
|
};
|
|
3270
3616
|
function getArrowColumnRoles(columnNames) {
|
|
@@ -3290,8 +3636,210 @@ function throwIfAborted2(signal) {
|
|
|
3290
3636
|
}
|
|
3291
3637
|
|
|
3292
3638
|
// dist/duckdb-sql-source.js
|
|
3639
|
+
var import_apache_arrow2 = require("apache-arrow");
|
|
3640
|
+
|
|
3641
|
+
// dist/duckdb-table-query.js
|
|
3293
3642
|
var import_apache_arrow = require("apache-arrow");
|
|
3294
|
-
var
|
|
3643
|
+
var SCALAR_TYPES = {
|
|
3644
|
+
TINYINT: "int8",
|
|
3645
|
+
SMALLINT: "int16",
|
|
3646
|
+
INTEGER: "int32",
|
|
3647
|
+
BIGINT: "int64",
|
|
3648
|
+
FLOAT: "float32",
|
|
3649
|
+
REAL: "float32",
|
|
3650
|
+
DOUBLE: "float64",
|
|
3651
|
+
BOOLEAN: "bool",
|
|
3652
|
+
VARCHAR: "utf8"
|
|
3653
|
+
};
|
|
3654
|
+
var INTEGER_TYPES = { int8: "TINYINT", int16: "SMALLINT", int32: "INTEGER", int64: "BIGINT" };
|
|
3655
|
+
function getIntegerType(type) {
|
|
3656
|
+
return typeof type === "string" && Object.hasOwn(INTEGER_TYPES, type) ? INTEGER_TYPES[type] : void 0;
|
|
3657
|
+
}
|
|
3658
|
+
function isDuckDBQuerySchema(schema) {
|
|
3659
|
+
return schema.fields.length > 0 && schema.fields.every((field) => {
|
|
3660
|
+
var _a;
|
|
3661
|
+
return Object.hasOwn(SCALAR_TYPES, ((_a = field.metadata) == null ? void 0 : _a.sqlType) || "");
|
|
3662
|
+
});
|
|
3663
|
+
}
|
|
3664
|
+
function makeDuckDBTableQuerySource(source, table, sourceSchema) {
|
|
3665
|
+
if (!isDuckDBQuerySchema(sourceSchema))
|
|
3666
|
+
throw new Error("Unsupported DuckDB table schema for the common query API");
|
|
3667
|
+
const schema = {
|
|
3668
|
+
...sourceSchema,
|
|
3669
|
+
fields: sourceSchema.fields.map((field) => {
|
|
3670
|
+
var _a, _b;
|
|
3671
|
+
const type = SCALAR_TYPES[((_a = field.metadata) == null ? void 0 : _a.sqlType) || ""];
|
|
3672
|
+
if (!type)
|
|
3673
|
+
throw new Error(`Unsupported DuckDB query field type: ${(_b = field.metadata) == null ? void 0 : _b.sqlType}`);
|
|
3674
|
+
return { ...field, type };
|
|
3675
|
+
})
|
|
3676
|
+
};
|
|
3677
|
+
const tableName = `${getQualifiedTableName(table)} AS query_input`;
|
|
3678
|
+
const ordered = table.tableType === "BASE TABLE" && !schema.fields.some((field) => field.name.toLowerCase() === "rowid");
|
|
3679
|
+
const order = ordered ? " ORDER BY query_input.rowid" : "";
|
|
3680
|
+
const metadata = createQueryMetadata({
|
|
3681
|
+
sourceType: "duckdb-table",
|
|
3682
|
+
queryType: "table",
|
|
3683
|
+
execution: { status: "supported", method: "read" },
|
|
3684
|
+
schema,
|
|
3685
|
+
queryCapabilities: {
|
|
3686
|
+
table: {
|
|
3687
|
+
projection: "pushdown",
|
|
3688
|
+
predicate: "pushdown+residual",
|
|
3689
|
+
cast: "pushdown",
|
|
3690
|
+
offset: ordered ? "pushdown" : "residual",
|
|
3691
|
+
limit: ordered ? "pushdown" : "residual",
|
|
3692
|
+
streaming: false,
|
|
3693
|
+
cancellation: false
|
|
3694
|
+
}
|
|
3695
|
+
}
|
|
3696
|
+
});
|
|
3697
|
+
async function* readStatement(sql, outputSchema, signal) {
|
|
3698
|
+
signal == null ? void 0 : signal.throwIfAborted();
|
|
3699
|
+
const rows = await source.queryRows(sql, { signal });
|
|
3700
|
+
signal == null ? void 0 : signal.throwIfAborted();
|
|
3701
|
+
const arrowSchema = convertSchemaToArrow(outputSchema);
|
|
3702
|
+
const vectors = arrowSchema.fields.map((field, index) => (0, import_apache_arrow.vectorFromArray)(rows.map((row) => {
|
|
3703
|
+
const value = row[field.name];
|
|
3704
|
+
if (value == null)
|
|
3705
|
+
return null;
|
|
3706
|
+
const type = outputSchema.fields[index].type;
|
|
3707
|
+
if (type === "int64")
|
|
3708
|
+
return BigInt(value);
|
|
3709
|
+
if (type === "float32" || type === "float64")
|
|
3710
|
+
return Number(value);
|
|
3711
|
+
return value;
|
|
3712
|
+
}), field.type));
|
|
3713
|
+
yield new import_apache_arrow.RecordBatch(arrowSchema, (0, import_apache_arrow.makeData)({
|
|
3714
|
+
type: new import_apache_arrow.Struct(arrowSchema.fields),
|
|
3715
|
+
length: rows.length,
|
|
3716
|
+
children: vectors.map((vector) => vector.data[0])
|
|
3717
|
+
}));
|
|
3718
|
+
}
|
|
3719
|
+
function plan(query) {
|
|
3720
|
+
var _a;
|
|
3721
|
+
validateTableQueryLimit(query.limit);
|
|
3722
|
+
if (query.offset !== void 0 && (!Number.isSafeInteger(query.offset) || query.offset < 0)) {
|
|
3723
|
+
throw new Error("DuckDB query offset must be a non-negative safe integer.");
|
|
3724
|
+
}
|
|
3725
|
+
const selections = query.select ?? schema.fields.map((field) => field.name);
|
|
3726
|
+
const columns = [];
|
|
3727
|
+
const fields = [];
|
|
3728
|
+
for (const [index, selection] of selections.entries()) {
|
|
3729
|
+
const expression = typeof selection !== "string" && "expression" in selection ? selection.expression : void 0;
|
|
3730
|
+
if (typeof selection !== "string" && "array" in selection)
|
|
3731
|
+
return null;
|
|
3732
|
+
const cast = (expression == null ? void 0 : expression.type) === "CallExpression" && expression.callee.type === "Identifier" && expression.callee.name === "cast" ? expression.arguments : void 0;
|
|
3733
|
+
const column = typeof selection === "string" ? selection : "column" in selection ? selection.column : ((_a = cast == null ? void 0 : cast[0]) == null ? void 0 : _a.type) === "Identifier" ? cast[0].name : void 0;
|
|
3734
|
+
const field = schema.fields.find((candidate) => candidate.name === column);
|
|
3735
|
+
if (!field)
|
|
3736
|
+
return null;
|
|
3737
|
+
let sql2 = quoteSqlIdentifier(field.name);
|
|
3738
|
+
const name = typeof selection === "string" ? selection : selection.as ?? (expression ? `column${index + 1}` : field.name);
|
|
3739
|
+
if (expression) {
|
|
3740
|
+
if ((cast == null ? void 0 : cast.length) !== 2 || cast[1].type !== "Literal")
|
|
3741
|
+
return null;
|
|
3742
|
+
const target = getExactCast(field, cast[1].value);
|
|
3743
|
+
if (!target)
|
|
3744
|
+
return null;
|
|
3745
|
+
sql2 = `CAST(${sql2} AS ${target.sqlType})`;
|
|
3746
|
+
fields.push({ name, type: target.type, nullable: field.nullable });
|
|
3747
|
+
} else {
|
|
3748
|
+
fields.push({ ...field, name });
|
|
3749
|
+
}
|
|
3750
|
+
columns.push(`${sql2} AS ${quoteSqlIdentifier(name)}`);
|
|
3751
|
+
}
|
|
3752
|
+
if (new Set(fields.map((field) => field.name.toLowerCase())).size !== fields.length)
|
|
3753
|
+
return null;
|
|
3754
|
+
const predicate = query.where && compilePredicate(query.where, schema);
|
|
3755
|
+
if (query.where && !predicate)
|
|
3756
|
+
return null;
|
|
3757
|
+
if (!ordered && (query.offset !== void 0 || query.limit !== void 0))
|
|
3758
|
+
return null;
|
|
3759
|
+
const sql = `SELECT ${columns.join(", ")} FROM ${tableName}` + (predicate ? ` WHERE ${predicate}` : "") + order + (query.limit === void 0 ? "" : ` LIMIT ${query.limit}`) + (query.offset === void 0 ? "" : ` OFFSET ${query.offset}`);
|
|
3760
|
+
const outputSchema = { ...schema, fields };
|
|
3761
|
+
return {
|
|
3762
|
+
schema: outputSchema,
|
|
3763
|
+
/** Executes the planned SQL when the consumer starts reading. */
|
|
3764
|
+
async *read(options) {
|
|
3765
|
+
yield* readStatement(sql, outputSchema, options == null ? void 0 : options.signal);
|
|
3766
|
+
}
|
|
3767
|
+
};
|
|
3768
|
+
}
|
|
3769
|
+
return {
|
|
3770
|
+
/** Returns metadata without issuing another catalog query. */
|
|
3771
|
+
async getQueryMetadata() {
|
|
3772
|
+
return metadata;
|
|
3773
|
+
},
|
|
3774
|
+
/** Accepts only exact whole queries. */
|
|
3775
|
+
async planQueryPushdown(query, options) {
|
|
3776
|
+
var _a;
|
|
3777
|
+
(_a = options == null ? void 0 : options.signal) == null ? void 0 : _a.throwIfAborted();
|
|
3778
|
+
return plan(query);
|
|
3779
|
+
},
|
|
3780
|
+
/** Checks the exact CAST subset using the same compiler as whole-query execution. */
|
|
3781
|
+
supportsQueryProjectionPushdown(selection) {
|
|
3782
|
+
return Boolean(plan({ select: [selection] }));
|
|
3783
|
+
},
|
|
3784
|
+
/** Executes simple reads natively when possible and applies other operators in Arrow. */
|
|
3785
|
+
async *read(options = {}) {
|
|
3786
|
+
const { columns, predicate, limit, offset, signal } = options;
|
|
3787
|
+
const accepted = plan({ select: columns, where: predicate, limit, offset });
|
|
3788
|
+
if (accepted && limit !== 0) {
|
|
3789
|
+
yield* accepted.read({ signal });
|
|
3790
|
+
return;
|
|
3791
|
+
}
|
|
3792
|
+
for await (const batch of executeTableScanBatches(async function* (readSignal) {
|
|
3793
|
+
for await (const batch2 of readStatement(`SELECT * FROM ${tableName}${order}`, schema, readSignal)) {
|
|
3794
|
+
yield new import_apache_arrow.Table(batch2.schema, [batch2]);
|
|
3795
|
+
}
|
|
3796
|
+
}, options, {
|
|
3797
|
+
filter: (batch2, predicate2, filterSignal) => {
|
|
3798
|
+
if (!predicate2)
|
|
3799
|
+
return batch2;
|
|
3800
|
+
return filterArrowTable(batch2, predicate2, { signal: filterSignal });
|
|
3801
|
+
}
|
|
3802
|
+
})) {
|
|
3803
|
+
yield* batch.batches.length ? batch.batches : [makeArrowMetadataBatch(batch.schema)];
|
|
3804
|
+
}
|
|
3805
|
+
}
|
|
3806
|
+
};
|
|
3807
|
+
}
|
|
3808
|
+
function getExactCast(field, target) {
|
|
3809
|
+
if (target === "float64" && ["int8", "int16", "int32"].includes(String(field.type))) {
|
|
3810
|
+
return { type: target, sqlType: "DOUBLE" };
|
|
3811
|
+
}
|
|
3812
|
+
if (target !== "int16" && target !== "int32" && target !== "int64")
|
|
3813
|
+
return null;
|
|
3814
|
+
const sourceType = getIntegerType(field.type);
|
|
3815
|
+
if (!sourceType || Number(String(field.type).slice(3)) > Number(target.slice(3)))
|
|
3816
|
+
return null;
|
|
3817
|
+
return { type: target, sqlType: INTEGER_TYPES[target] };
|
|
3818
|
+
}
|
|
3819
|
+
function compilePredicate(predicate, schema) {
|
|
3820
|
+
if (!("op" in predicate) || !["=", "<>", "<", "<=", ">", ">="].includes(predicate.op))
|
|
3821
|
+
return null;
|
|
3822
|
+
const [property, value] = predicate.args;
|
|
3823
|
+
if (!property || typeof property !== "object" || !("property" in property))
|
|
3824
|
+
return null;
|
|
3825
|
+
const field = schema.fields.find((candidate) => candidate.name === property.property);
|
|
3826
|
+
if (!field || !getIntegerType(field.type))
|
|
3827
|
+
return null;
|
|
3828
|
+
if (typeof value !== "bigint" && (typeof value !== "number" || !Number.isSafeInteger(value)))
|
|
3829
|
+
return null;
|
|
3830
|
+
if (value < -(2n ** 63n) || value >= 2n ** 63n)
|
|
3831
|
+
return null;
|
|
3832
|
+
const column = quoteSqlIdentifier(field.name);
|
|
3833
|
+
if (!("casts" in property))
|
|
3834
|
+
return `${column} ${predicate.op} ${value}`;
|
|
3835
|
+
if (!Array.isArray(property.casts) || property.casts.length !== 1)
|
|
3836
|
+
return null;
|
|
3837
|
+
const cast = getExactCast(field, property.casts[0]);
|
|
3838
|
+
return cast ? `CAST(${column} AS ${cast.sqlType}) ${predicate.op} ${value}` : null;
|
|
3839
|
+
}
|
|
3840
|
+
|
|
3841
|
+
// dist/duckdb-sql-source.js
|
|
3842
|
+
var VERSION2 = true ? "5.0.0-beta.11" : "latest";
|
|
3295
3843
|
var DuckDBSQLSource = {
|
|
3296
3844
|
dataType: null,
|
|
3297
3845
|
batchType: null,
|
|
@@ -3330,12 +3878,32 @@ var DuckDBSQLSource = {
|
|
|
3330
3878
|
},
|
|
3331
3879
|
defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
|
|
3332
3880
|
testURL: (url) => /^duckdb:\/\//i.test(url),
|
|
3333
|
-
|
|
3881
|
+
createDataSourceSync: (data, options, coreApi) => new DuckDBSQLDataSource(data, options, coreApi)
|
|
3334
3882
|
};
|
|
3335
3883
|
var DuckDBSQLDataSource = class extends SQLDataSource {
|
|
3884
|
+
/** Resolves a reusable common query handle for a named DuckDB table. */
|
|
3885
|
+
async getTable(name, options = {}) {
|
|
3886
|
+
var _a;
|
|
3887
|
+
const table = await this.resolveTable(name, options.signal);
|
|
3888
|
+
const schema = await this.getTableSchema(table);
|
|
3889
|
+
(_a = options.signal) == null ? void 0 : _a.throwIfAborted();
|
|
3890
|
+
return makeDuckDBTableQuerySource(this, table, schema);
|
|
3891
|
+
}
|
|
3892
|
+
/** Reports whether the table's schema can be represented exactly by the common reader. */
|
|
3893
|
+
async getTableMetadata(name, options = {}) {
|
|
3894
|
+
const metadata = await super.getTableMetadata(name, options);
|
|
3895
|
+
return {
|
|
3896
|
+
...metadata,
|
|
3897
|
+
queryable: Boolean(metadata.schema && isDuckDBQuerySchema(metadata.schema))
|
|
3898
|
+
};
|
|
3899
|
+
}
|
|
3900
|
+
/** Lists table names without claiming compatibility before schema discovery. */
|
|
3901
|
+
async getTablesMetadata(options = {}) {
|
|
3902
|
+
return (await super.getTablesMetadata(options)).map(({ name }) => ({ name }));
|
|
3903
|
+
}
|
|
3336
3904
|
/** Uploads an Arrow table through IPC. Supported by the browser DuckDB-Wasm adapter. */
|
|
3337
3905
|
async insertArrowTable(table, options) {
|
|
3338
|
-
await this.insertArrowIPC((0,
|
|
3906
|
+
await this.insertArrowIPC((0, import_apache_arrow2.tableToIPC)(table, "stream"), options);
|
|
3339
3907
|
}
|
|
3340
3908
|
/** Uploads a complete Arrow IPC stream to a browser DuckDB-Wasm table. */
|
|
3341
3909
|
async insertArrowIPC(bytes, options) {
|
|
@@ -3484,7 +4052,7 @@ async function createDuckDBBrowserAdapter(context) {
|
|
|
3484
4052
|
const connection = await database.connect();
|
|
3485
4053
|
const executeRows = async (sqlText, options = {}) => {
|
|
3486
4054
|
const result = await executeDuckDBBrowserQuery(connection, sqlText, options.parameters);
|
|
3487
|
-
return convertArrowToTable2((0,
|
|
4055
|
+
return convertArrowToTable2((0, import_apache_arrow2.tableFromIPC)(result), "object-row-table").data;
|
|
3488
4056
|
};
|
|
3489
4057
|
return {
|
|
3490
4058
|
capabilities: {
|
|
@@ -3576,7 +4144,7 @@ async function createDuckDBBrowserAdapter(context) {
|
|
|
3576
4144
|
},
|
|
3577
4145
|
async executeArrow(sqlText, options = {}) {
|
|
3578
4146
|
const result = await executeDuckDBBrowserQuery(connection, sqlText, options.parameters);
|
|
3579
|
-
return
|
|
4147
|
+
return (0, import_apache_arrow2.tableFromIPC)(result);
|
|
3580
4148
|
}
|
|
3581
4149
|
};
|
|
3582
4150
|
}
|
|
@@ -3608,7 +4176,7 @@ function getDuckDBDatabasePath(url, options) {
|
|
|
3608
4176
|
}
|
|
3609
4177
|
|
|
3610
4178
|
// dist/snowflake-sql-source.js
|
|
3611
|
-
var VERSION3 = true ? "5.0.0-beta.
|
|
4179
|
+
var VERSION3 = true ? "5.0.0-beta.11" : "latest";
|
|
3612
4180
|
var SnowflakeSQLSource = {
|
|
3613
4181
|
dataType: null,
|
|
3614
4182
|
batchType: null,
|
|
@@ -3647,7 +4215,7 @@ var SnowflakeSQLSource = {
|
|
|
3647
4215
|
},
|
|
3648
4216
|
defaultOptions: SQL_SOURCE_DEFAULT_OPTIONS,
|
|
3649
4217
|
testURL: (url) => /^(sql\+snowflake|snowflake):\/\//i.test(url),
|
|
3650
|
-
|
|
4218
|
+
createDataSourceSync: (data, options, coreApi) => new SnowflakeSQLDataSource(data, options, coreApi)
|
|
3651
4219
|
};
|
|
3652
4220
|
var SnowflakeSQLDataSource = class extends SQLDataSource {
|
|
3653
4221
|
constructor(data, options, coreApi) {
|
|
@@ -3814,304 +4382,12 @@ function getSnowflakeBaseUrl(url, options) {
|
|
|
3814
4382
|
}
|
|
3815
4383
|
|
|
3816
4384
|
// ../core/src/scan/parse-sql-predicate.ts
|
|
3817
|
-
var
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
function parseSQLPredicate(source, options = {}) {
|
|
3821
|
-
const parser = new SQLPredicateParser(
|
|
3822
|
-
source,
|
|
3823
|
-
options.parameters ?? {},
|
|
3824
|
-
options.preserveParameters === true
|
|
3825
|
-
);
|
|
3826
|
-
const predicate = parser.parse();
|
|
4385
|
+
var import_parser = require("@math.gl/sql/parser");
|
|
4386
|
+
function parseSQLPredicate(source, options) {
|
|
4387
|
+
const predicate = options ? (0, import_parser.parseSQLPredicate)(source, options) : (0, import_parser.parseSQLPredicate)(source);
|
|
3827
4388
|
validateSQLPredicate(predicate);
|
|
3828
4389
|
return predicate;
|
|
3829
4390
|
}
|
|
3830
|
-
var SQLPredicateParser = class {
|
|
3831
|
-
/** Token stream parsed by this parser instance. */
|
|
3832
|
-
tokens;
|
|
3833
|
-
/** Named scalar parameters available to the expression. */
|
|
3834
|
-
parameters;
|
|
3835
|
-
/** Whether named parameters remain references in the parsed AST. */
|
|
3836
|
-
preserveParameters;
|
|
3837
|
-
/** Current index in the token stream. */
|
|
3838
|
-
position = 0;
|
|
3839
|
-
constructor(source, parameters, preserveParameters) {
|
|
3840
|
-
this.tokens = tokenizeSQLPredicate(source);
|
|
3841
|
-
this.parameters = parameters;
|
|
3842
|
-
this.preserveParameters = preserveParameters;
|
|
3843
|
-
}
|
|
3844
|
-
/** Parses the complete token stream. */
|
|
3845
|
-
parse() {
|
|
3846
|
-
const predicate = this.parseOr(0);
|
|
3847
|
-
this.takeSymbol(";");
|
|
3848
|
-
const token = this.peek();
|
|
3849
|
-
if (token) {
|
|
3850
|
-
throw new Error(`SQL predicate contains unexpected token ${JSON.stringify(token.value)}`);
|
|
3851
|
-
}
|
|
3852
|
-
return predicate;
|
|
3853
|
-
}
|
|
3854
|
-
/** Parses left-associative disjunction. */
|
|
3855
|
-
parseOr(depth) {
|
|
3856
|
-
let predicate = this.parseAnd(depth + 1);
|
|
3857
|
-
while (this.takeKeyword("OR")) {
|
|
3858
|
-
predicate = combineLogicalPredicate("or", predicate, this.parseAnd(depth + 1));
|
|
3859
|
-
}
|
|
3860
|
-
return predicate;
|
|
3861
|
-
}
|
|
3862
|
-
/** Parses left-associative conjunction. */
|
|
3863
|
-
parseAnd(depth) {
|
|
3864
|
-
let predicate = this.parseNot(depth + 1);
|
|
3865
|
-
while (this.takeKeyword("AND")) {
|
|
3866
|
-
predicate = combineLogicalPredicate("and", predicate, this.parseNot(depth + 1));
|
|
3867
|
-
}
|
|
3868
|
-
return predicate;
|
|
3869
|
-
}
|
|
3870
|
-
/** Parses boolean negation and parenthesized predicates. */
|
|
3871
|
-
parseNot(depth) {
|
|
3872
|
-
this.assertDepth(depth);
|
|
3873
|
-
if (this.takeKeyword("NOT")) {
|
|
3874
|
-
return { op: "not", args: [this.parseNot(depth + 1)] };
|
|
3875
|
-
}
|
|
3876
|
-
if (this.takeSymbol("(")) {
|
|
3877
|
-
const predicate = this.parseOr(depth + 1);
|
|
3878
|
-
this.expectSymbol(")");
|
|
3879
|
-
return predicate;
|
|
3880
|
-
}
|
|
3881
|
-
return this.parseLeaf();
|
|
3882
|
-
}
|
|
3883
|
-
/** Parses one comparison, membership, or null predicate. */
|
|
3884
|
-
parseLeaf() {
|
|
3885
|
-
var _a, _b;
|
|
3886
|
-
const property = this.parseProperty();
|
|
3887
|
-
if (this.takeKeyword("IS")) {
|
|
3888
|
-
const negated = this.takeKeyword("NOT");
|
|
3889
|
-
this.expectKeyword("NULL");
|
|
3890
|
-
const predicate = { op: "isNull", args: [property] };
|
|
3891
|
-
return negated ? { op: "not", args: [predicate] } : predicate;
|
|
3892
|
-
}
|
|
3893
|
-
const negatedMembership = this.takeKeyword("NOT");
|
|
3894
|
-
if (negatedMembership || this.peekKeyword("IN")) {
|
|
3895
|
-
this.expectKeyword("IN");
|
|
3896
|
-
this.expectSymbol("(");
|
|
3897
|
-
if (((_a = this.peek()) == null ? void 0 : _a.kind) === "symbol" && ((_b = this.peek()) == null ? void 0 : _b.value) === ")") {
|
|
3898
|
-
throw new Error("SQL predicate IN requires at least one scalar value");
|
|
3899
|
-
}
|
|
3900
|
-
const values = [this.parseValue()];
|
|
3901
|
-
while (this.takeSymbol(",")) {
|
|
3902
|
-
values.push(this.parseValue());
|
|
3903
|
-
}
|
|
3904
|
-
this.expectSymbol(")");
|
|
3905
|
-
const predicate = { op: "in", args: [property, values] };
|
|
3906
|
-
return negatedMembership ? { op: "not", args: [predicate] } : predicate;
|
|
3907
|
-
}
|
|
3908
|
-
const token = this.take();
|
|
3909
|
-
if ((token == null ? void 0 : token.kind) !== "symbol" || !["=", "!=", "<>", "<", "<=", ">", ">="].includes(token.value)) {
|
|
3910
|
-
throw new Error("SQL predicate expected a comparison, IN, or IS NULL operator");
|
|
3911
|
-
}
|
|
3912
|
-
const operator = token.value === "!=" ? "<>" : token.value;
|
|
3913
|
-
return {
|
|
3914
|
-
op: operator,
|
|
3915
|
-
args: [property, this.parseValue()]
|
|
3916
|
-
};
|
|
3917
|
-
}
|
|
3918
|
-
/** Parses an identifier into a property reference. */
|
|
3919
|
-
parseProperty() {
|
|
3920
|
-
const token = this.take();
|
|
3921
|
-
if ((token == null ? void 0 : token.kind) !== "identifier") {
|
|
3922
|
-
throw new Error("SQL predicate expected a column identifier");
|
|
3923
|
-
}
|
|
3924
|
-
return token.quoted ? { property: token.value, quoted: true } : { property: token.value };
|
|
3925
|
-
}
|
|
3926
|
-
/** Parses one scalar literal or named parameter. */
|
|
3927
|
-
parseValue() {
|
|
3928
|
-
const negative = this.takeSymbol("-");
|
|
3929
|
-
const token = this.take();
|
|
3930
|
-
if (!token) {
|
|
3931
|
-
throw new Error("SQL predicate expected a scalar value");
|
|
3932
|
-
}
|
|
3933
|
-
if (negative && token.kind !== "number") {
|
|
3934
|
-
throw new Error("SQL predicate only permits unary minus on numeric literals");
|
|
3935
|
-
}
|
|
3936
|
-
if (token.kind === "number") {
|
|
3937
|
-
return parseNumericLiteral(negative ? `-${token.value}` : token.value);
|
|
3938
|
-
}
|
|
3939
|
-
if (negative) {
|
|
3940
|
-
throw new Error("SQL predicate expected a numeric literal after unary minus");
|
|
3941
|
-
}
|
|
3942
|
-
if (token.kind === "string") {
|
|
3943
|
-
return token.value;
|
|
3944
|
-
}
|
|
3945
|
-
if (token.kind === "parameter") {
|
|
3946
|
-
if (this.preserveParameters) {
|
|
3947
|
-
return { parameter: token.value };
|
|
3948
|
-
}
|
|
3949
|
-
if (!Object.hasOwn(this.parameters, token.value)) {
|
|
3950
|
-
throw new Error(`SQL predicate parameter ":${token.value}" requires a value`);
|
|
3951
|
-
}
|
|
3952
|
-
const value = this.parameters[token.value];
|
|
3953
|
-
validateParameterValue(value, token.value);
|
|
3954
|
-
return value;
|
|
3955
|
-
}
|
|
3956
|
-
if (token.kind === "identifier" && /^(TRUE|FALSE)$/i.test(token.value)) {
|
|
3957
|
-
return /^TRUE$/i.test(token.value);
|
|
3958
|
-
}
|
|
3959
|
-
if (token.kind === "identifier" && /^NULL$/i.test(token.value)) {
|
|
3960
|
-
throw new Error("SQL predicate comparisons with NULL must use IS NULL or IS NOT NULL");
|
|
3961
|
-
}
|
|
3962
|
-
throw new Error(`SQL predicate does not support scalar token ${JSON.stringify(token.value)}`);
|
|
3963
|
-
}
|
|
3964
|
-
/** Rejects excessively nested inputs before recursion can exhaust the stack. */
|
|
3965
|
-
assertDepth(depth) {
|
|
3966
|
-
if (depth > MAXIMUM_EXPRESSION_DEPTH) {
|
|
3967
|
-
throw new Error("SQL predicate nesting exceeds its safe limit");
|
|
3968
|
-
}
|
|
3969
|
-
}
|
|
3970
|
-
/** Consumes one required keyword. */
|
|
3971
|
-
expectKeyword(keyword) {
|
|
3972
|
-
if (!this.takeKeyword(keyword)) {
|
|
3973
|
-
throw new Error(`SQL predicate expected ${keyword}`);
|
|
3974
|
-
}
|
|
3975
|
-
}
|
|
3976
|
-
/** Returns whether the next token is a keyword without consuming it. */
|
|
3977
|
-
peekKeyword(keyword) {
|
|
3978
|
-
const token = this.peek();
|
|
3979
|
-
return (token == null ? void 0 : token.kind) === "identifier" && token.value.toUpperCase() === keyword;
|
|
3980
|
-
}
|
|
3981
|
-
/** Consumes one optional keyword. */
|
|
3982
|
-
takeKeyword(keyword) {
|
|
3983
|
-
if (!this.peekKeyword(keyword)) {
|
|
3984
|
-
return false;
|
|
3985
|
-
}
|
|
3986
|
-
this.position++;
|
|
3987
|
-
return true;
|
|
3988
|
-
}
|
|
3989
|
-
/** Consumes one required symbol. */
|
|
3990
|
-
expectSymbol(symbol) {
|
|
3991
|
-
if (!this.takeSymbol(symbol)) {
|
|
3992
|
-
throw new Error(`SQL predicate expected ${JSON.stringify(symbol)}`);
|
|
3993
|
-
}
|
|
3994
|
-
}
|
|
3995
|
-
/** Consumes one optional symbol. */
|
|
3996
|
-
takeSymbol(symbol) {
|
|
3997
|
-
const token = this.peek();
|
|
3998
|
-
if ((token == null ? void 0 : token.kind) !== "symbol" || token.value !== symbol) {
|
|
3999
|
-
return false;
|
|
4000
|
-
}
|
|
4001
|
-
this.position++;
|
|
4002
|
-
return true;
|
|
4003
|
-
}
|
|
4004
|
-
/** Consumes and returns the next token. */
|
|
4005
|
-
take() {
|
|
4006
|
-
const token = this.tokens[this.position];
|
|
4007
|
-
if (token) {
|
|
4008
|
-
this.position++;
|
|
4009
|
-
}
|
|
4010
|
-
return token;
|
|
4011
|
-
}
|
|
4012
|
-
/** Returns the next token without consuming it. */
|
|
4013
|
-
peek() {
|
|
4014
|
-
return this.tokens[this.position];
|
|
4015
|
-
}
|
|
4016
|
-
};
|
|
4017
|
-
function combineLogicalPredicate(operator, left, right) {
|
|
4018
|
-
return left.op === operator ? { op: operator, args: [...left.args, right] } : { op: operator, args: [left, right] };
|
|
4019
|
-
}
|
|
4020
|
-
function parseNumericLiteral(source) {
|
|
4021
|
-
if (/^-?\d+$/.test(source)) {
|
|
4022
|
-
const bigint = BigInt(source);
|
|
4023
|
-
if (bigint > BigInt(Number.MAX_SAFE_INTEGER) || bigint < BigInt(Number.MIN_SAFE_INTEGER)) {
|
|
4024
|
-
return bigint;
|
|
4025
|
-
}
|
|
4026
|
-
}
|
|
4027
|
-
const number = Number(source);
|
|
4028
|
-
if (!Number.isFinite(number)) {
|
|
4029
|
-
throw new Error("SQL predicate numeric literals must be finite");
|
|
4030
|
-
}
|
|
4031
|
-
return number;
|
|
4032
|
-
}
|
|
4033
|
-
function validateParameterValue(value, name) {
|
|
4034
|
-
const probe = { op: "=", args: [{ property: "_" }, value] };
|
|
4035
|
-
try {
|
|
4036
|
-
validateSQLPredicate(probe);
|
|
4037
|
-
} catch {
|
|
4038
|
-
throw new Error(`SQL predicate parameter ":${name}" has an unsupported value`);
|
|
4039
|
-
}
|
|
4040
|
-
}
|
|
4041
|
-
function tokenizeSQLPredicate(source) {
|
|
4042
|
-
if (typeof source !== "string" || source.trim().length === 0 || source.length > MAXIMUM_SQL_LENGTH) {
|
|
4043
|
-
throw new Error("SQL predicate requires a non-empty, safely bounded expression");
|
|
4044
|
-
}
|
|
4045
|
-
const tokens = [];
|
|
4046
|
-
let position = 0;
|
|
4047
|
-
while (position < source.length) {
|
|
4048
|
-
const character = source[position];
|
|
4049
|
-
if (/\s/.test(character)) {
|
|
4050
|
-
position++;
|
|
4051
|
-
continue;
|
|
4052
|
-
}
|
|
4053
|
-
if (character === "'") {
|
|
4054
|
-
const result = readQuotedToken(source, position, "'", "'");
|
|
4055
|
-
tokens.push({ kind: "string", value: result.value });
|
|
4056
|
-
position = result.position;
|
|
4057
|
-
} else if (character === '"') {
|
|
4058
|
-
const result = readQuotedToken(source, position, '"', '"');
|
|
4059
|
-
tokens.push({ kind: "identifier", value: result.value, quoted: true });
|
|
4060
|
-
position = result.position;
|
|
4061
|
-
} else if (character === ":") {
|
|
4062
|
-
const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(source.slice(position + 1));
|
|
4063
|
-
if (!match) {
|
|
4064
|
-
throw new Error(`SQL predicate contains an invalid parameter at character ${position}`);
|
|
4065
|
-
}
|
|
4066
|
-
tokens.push({ kind: "parameter", value: match[0] });
|
|
4067
|
-
position += match[0].length + 1;
|
|
4068
|
-
} else {
|
|
4069
|
-
const numberMatch = /^(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?/.exec(source.slice(position));
|
|
4070
|
-
const identifierMatch = /^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*/.exec(
|
|
4071
|
-
source.slice(position)
|
|
4072
|
-
);
|
|
4073
|
-
if (numberMatch) {
|
|
4074
|
-
tokens.push({ kind: "number", value: numberMatch[0] });
|
|
4075
|
-
position += numberMatch[0].length;
|
|
4076
|
-
} else if (identifierMatch) {
|
|
4077
|
-
tokens.push({ kind: "identifier", value: identifierMatch[0] });
|
|
4078
|
-
position += identifierMatch[0].length;
|
|
4079
|
-
} else {
|
|
4080
|
-
const twoCharacters = source.slice(position, position + 2);
|
|
4081
|
-
if (["<=", ">=", "<>", "!="].includes(twoCharacters)) {
|
|
4082
|
-
tokens.push({ kind: "symbol", value: twoCharacters });
|
|
4083
|
-
position += 2;
|
|
4084
|
-
} else if ("(),;-=<>".includes(character)) {
|
|
4085
|
-
tokens.push({ kind: "symbol", value: character });
|
|
4086
|
-
position++;
|
|
4087
|
-
} else {
|
|
4088
|
-
throw new Error(`SQL predicate contains an unsupported token at character ${position}`);
|
|
4089
|
-
}
|
|
4090
|
-
}
|
|
4091
|
-
}
|
|
4092
|
-
if (tokens.length > MAXIMUM_SQL_TOKENS) {
|
|
4093
|
-
throw new Error("SQL predicate exceeds its safe token limit");
|
|
4094
|
-
}
|
|
4095
|
-
}
|
|
4096
|
-
return tokens;
|
|
4097
|
-
}
|
|
4098
|
-
function readQuotedToken(source, start, quote, escapedQuote) {
|
|
4099
|
-
let value = "";
|
|
4100
|
-
let position = start + 1;
|
|
4101
|
-
while (position < source.length) {
|
|
4102
|
-
if (source[position] !== quote) {
|
|
4103
|
-
value += source[position++];
|
|
4104
|
-
continue;
|
|
4105
|
-
}
|
|
4106
|
-
if (source[position + 1] === escapedQuote) {
|
|
4107
|
-
value += quote;
|
|
4108
|
-
position += 2;
|
|
4109
|
-
continue;
|
|
4110
|
-
}
|
|
4111
|
-
return { value, position: position + 1 };
|
|
4112
|
-
}
|
|
4113
|
-
throw new Error(`SQL predicate contains an unterminated quoted value at character ${start}`);
|
|
4114
|
-
}
|
|
4115
4391
|
|
|
4116
4392
|
// ../core/src/scan/bind-sql-predicate.ts
|
|
4117
4393
|
function bindSQLPredicate(predicate, parameters) {
|