@loaders.gl/parquet 5.0.0-alpha.7 → 5.0.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/delta-source-loader-types.js +1 -1
- package/dist/delta-source.js +1 -1
- package/dist/geoparquet-loader.js +1 -1
- package/dist/iceberg-rest-catalog.js +1 -1
- package/dist/iceberg-rest-catalog.js.map +1 -1
- package/dist/index.cjs +261 -81
- package/dist/index.cjs.map +3 -3
- package/dist/lib/constants.js +1 -1
- package/dist/parquet-dataset-source.cjs +257 -77
- package/dist/parquet-dataset-source.cjs.map +3 -3
- package/dist/parquet-js-loader-types.js +1 -1
- package/dist/parquet-js-writer.js +1 -1
- package/dist/parquet-loader-base.js +1 -1
- package/dist/parquet-source-loader-types.js +1 -1
- package/dist/parquet-source-loader.cjs +257 -77
- package/dist/parquet-source-loader.cjs.map +3 -3
- package/dist/parquet-source-worker.js +20 -95
- package/dist/parquet-source-worker.js.map +4 -4
- package/dist/parquet-worker-node.cjs +24373 -17868
- package/dist/parquet-worker-node.cjs.map +4 -4
- package/dist/parquet-worker.js +5 -1
- package/dist/parquet-worker.js.map +2 -2
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +15 -6
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/package.json +13 -18
- package/src/iceberg-rest-catalog.ts +1 -1
- package/src/parquetjs/codecs/rle.ts +17 -6
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { ParquetFormat } from "./parquet-format.js";
|
|
5
5
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
6
6
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
7
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
7
|
+
const VERSION = typeof "5.0.0-alpha.8" !== 'undefined' ? "5.0.0-alpha.8" : 'latest';
|
|
8
8
|
/** Default option bag for the experimental TypeScript-backed Parquet loader. */
|
|
9
9
|
const DEFAULT_PARQUET_JS_OPTIONS = {
|
|
10
10
|
columns: undefined,
|
|
@@ -7,7 +7,7 @@ import { encodeTableToParquetJs } from "./lib/encoders/encode-table-to-parquet-j
|
|
|
7
7
|
import { ParquetFormat } from "./parquet-format.js";
|
|
8
8
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
9
9
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
10
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
10
|
+
const VERSION = typeof "5.0.0-alpha.8" !== 'undefined' ? "5.0.0-alpha.8" : 'latest';
|
|
11
11
|
/** Default option bag for the experimental parquetjs writer. */
|
|
12
12
|
const DEFAULT_PARQUET_JS_OPTIONS = {};
|
|
13
13
|
/** Plain-row Parquet writer backed by the experimental parquetjs implementation. */
|
|
@@ -5,7 +5,7 @@ import { ParquetFormat } from "./parquet-format.js";
|
|
|
5
5
|
import { PARQUET_LOADER_DEFAULT_OPTIONS } from "./parquet-loader-options.js";
|
|
6
6
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
7
7
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
8
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
8
|
+
const VERSION = typeof "5.0.0-alpha.8" !== 'undefined' ? "5.0.0-alpha.8" : 'latest';
|
|
9
9
|
/** Shared Parquet loader metadata without parser preloading or worker delivery policy. */
|
|
10
10
|
export const PARQUET_LOADER_BASE = {
|
|
11
11
|
...ParquetFormat,
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { ParquetFormat } from "./parquet-format.js";
|
|
5
5
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
6
6
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
7
|
-
const VERSION = typeof "5.0.0-alpha.
|
|
7
|
+
const VERSION = typeof "5.0.0-alpha.8" !== 'undefined' ? "5.0.0-alpha.8" : 'latest';
|
|
8
8
|
/** Loads the runtime Parquet source implementation from its explicit package subpath. */
|
|
9
9
|
async function preloadParquetSourceLoader() {
|
|
10
10
|
const { ParquetSourceLoaderWithParser } = await import('@loaders.gl/parquet/parquet-source-loader');
|
|
@@ -1047,13 +1047,17 @@ var init_src = __esm({
|
|
|
1047
1047
|
|
|
1048
1048
|
// ../loader-utils/src/lib/worker-loader-utils/parse-with-worker.ts
|
|
1049
1049
|
function canParseWithWorker(loader, options) {
|
|
1050
|
-
var _a;
|
|
1050
|
+
var _a, _b, _c;
|
|
1051
|
+
if ((_b = (_a = options == null ? void 0 : options.core) == null ? void 0 : _a.credentials) == null ? void 0 : _b.some(
|
|
1052
|
+
(credential) => credential.type === "request" || "token" in credential && typeof credential.token === "function"
|
|
1053
|
+
))
|
|
1054
|
+
return false;
|
|
1051
1055
|
const workerOptions = getWorkerOptions(options);
|
|
1052
1056
|
const nodeWorkers = workerOptions._nodeWorkers;
|
|
1053
1057
|
if (!isBrowser2 && !nodeWorkers) {
|
|
1054
1058
|
return false;
|
|
1055
1059
|
}
|
|
1056
|
-
if (loader.id === "las" && (((
|
|
1060
|
+
if (loader.id === "las" && (((_c = options == null ? void 0 : options.las) == null ? void 0 : _c.shape) ?? "arrow-table") === "arrow-table") {
|
|
1057
1061
|
return false;
|
|
1058
1062
|
}
|
|
1059
1063
|
if (loader.id === "csv" && !shouldParseCSVWithWorker(options)) {
|
|
@@ -1100,7 +1104,12 @@ function callParseOnMainThread(parseOnMainThread, input, options, context) {
|
|
|
1100
1104
|
return parseOnMainThread(input, void 0, options, context);
|
|
1101
1105
|
}
|
|
1102
1106
|
function getWorkerOptions(options = {}, loader) {
|
|
1103
|
-
const serializedOptions = JSON.parse(
|
|
1107
|
+
const serializedOptions = JSON.parse(
|
|
1108
|
+
JSON.stringify({
|
|
1109
|
+
...options,
|
|
1110
|
+
...options.core ? { core: { ...options.core, authentications: void 0 } } : {}
|
|
1111
|
+
})
|
|
1112
|
+
);
|
|
1104
1113
|
const workerOptions = {
|
|
1105
1114
|
...serializedOptions.core,
|
|
1106
1115
|
...serializedOptions
|
|
@@ -1189,6 +1198,48 @@ var init_async_iteration = __esm({
|
|
|
1189
1198
|
}
|
|
1190
1199
|
});
|
|
1191
1200
|
|
|
1201
|
+
// ../loader-utils/src/lib/request-utils/authentication.ts
|
|
1202
|
+
function resolveCredentials(credentials, authentications = []) {
|
|
1203
|
+
if (credentials.every(isAuthentication)) return credentials;
|
|
1204
|
+
return credentials.map((credential, index) => {
|
|
1205
|
+
if (isAuthentication(credential)) return credential;
|
|
1206
|
+
const AuthenticationClass = authentications.find(
|
|
1207
|
+
(authentication2) => authentication2.type === (credential == null ? void 0 : credential.type)
|
|
1208
|
+
);
|
|
1209
|
+
if (!AuthenticationClass) {
|
|
1210
|
+
throw new Error(`No authentication class registered for credential at index ${index}.`);
|
|
1211
|
+
}
|
|
1212
|
+
let authentication;
|
|
1213
|
+
try {
|
|
1214
|
+
authentication = new AuthenticationClass(credential);
|
|
1215
|
+
} catch {
|
|
1216
|
+
throw new Error(`Invalid authentication configuration at index ${index}.`);
|
|
1217
|
+
}
|
|
1218
|
+
if (!isAuthentication(authentication)) {
|
|
1219
|
+
throw new Error(`Authentication class returned an invalid credential at index ${index}.`);
|
|
1220
|
+
}
|
|
1221
|
+
return authentication;
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
function resolveAuthenticationOptions(options) {
|
|
1225
|
+
var _a, _b;
|
|
1226
|
+
if (!((_b = (_a = options.core) == null ? void 0 : _a.credentials) == null ? void 0 : _b.length)) return options;
|
|
1227
|
+
const credentials = resolveCredentials(options.core.credentials, options.core.authentications);
|
|
1228
|
+
if (credentials === options.core.credentials) return options;
|
|
1229
|
+
return { ...options, core: { ...options.core, credentials } };
|
|
1230
|
+
}
|
|
1231
|
+
function isAuthentication(credential) {
|
|
1232
|
+
if (!credential || typeof credential !== "object") return false;
|
|
1233
|
+
if (typeof credential.id !== "string" || !Array.isArray(credential.origins)) return false;
|
|
1234
|
+
if (credential.type === "request") return typeof credential.authenticate === "function";
|
|
1235
|
+
return (credential.type === "header" || credential.type === "query-parameter") && typeof credential.name === "string" && (typeof credential.token === "string" || typeof credential.token === "function") && Array.isArray(credential.refreshStatusCodes);
|
|
1236
|
+
}
|
|
1237
|
+
var init_authentication = __esm({
|
|
1238
|
+
"../loader-utils/src/lib/request-utils/authentication.ts"() {
|
|
1239
|
+
"use strict";
|
|
1240
|
+
}
|
|
1241
|
+
});
|
|
1242
|
+
|
|
1192
1243
|
// ../loader-utils/src/lib/request-utils/request-cache.ts
|
|
1193
1244
|
function createAbortError3() {
|
|
1194
1245
|
const error = new Error("Request aborted");
|
|
@@ -1423,21 +1474,54 @@ var init_request_cache = __esm({
|
|
|
1423
1474
|
|
|
1424
1475
|
// ../loader-utils/src/lib/request-utils/request-credentials.ts
|
|
1425
1476
|
function createAuthenticatedFetch(options) {
|
|
1426
|
-
const
|
|
1427
|
-
|
|
1428
|
-
const
|
|
1429
|
-
|
|
1477
|
+
const suppliedFetch = options.fetch || ((url, requestOptions) => fetch(url, requestOptions));
|
|
1478
|
+
const existingMetadata = suppliedFetch[AUTHENTICATED_FETCH];
|
|
1479
|
+
const credentialResolutions = new Map(existingMetadata == null ? void 0 : existingMetadata.credentialResolutions);
|
|
1480
|
+
const resolvedCredentials = resolveCredentials(
|
|
1481
|
+
options.credentials.map((credential) => credentialResolutions.get(credential) || credential),
|
|
1482
|
+
options.authentications
|
|
1483
|
+
);
|
|
1484
|
+
options.credentials.forEach((credential, index) => {
|
|
1485
|
+
credentialResolutions.set(credential, resolvedCredentials[index]);
|
|
1486
|
+
});
|
|
1487
|
+
if (!options.fetchOptions && (!resolvedCredentials.length || (existingMetadata == null ? void 0 : existingMetadata.credentials.length) === resolvedCredentials.length && resolvedCredentials.every(
|
|
1488
|
+
(credential, index) => credential === existingMetadata.credentials[index]
|
|
1489
|
+
)))
|
|
1490
|
+
return suppliedFetch;
|
|
1491
|
+
const baseFetch = (existingMetadata == null ? void 0 : existingMetadata.fetch) || suppliedFetch;
|
|
1492
|
+
const credentials = existingMetadata ? [.../* @__PURE__ */ new Set([...resolvedCredentials, ...existingMetadata.credentials])] : resolvedCredentials;
|
|
1493
|
+
const fetchOptions = mergeFetchOptions(
|
|
1494
|
+
(existingMetadata == null ? void 0 : existingMetadata.fetchOptions) || {},
|
|
1495
|
+
options.fetchOptions
|
|
1496
|
+
);
|
|
1497
|
+
const tokenCredentials = credentials.filter(
|
|
1498
|
+
(credential) => credential.type !== "request"
|
|
1499
|
+
);
|
|
1500
|
+
const requestCredentials = credentials.filter(
|
|
1501
|
+
(credential) => credential.type === "request"
|
|
1502
|
+
);
|
|
1503
|
+
for (const credential of requestCredentials) normalizeRequestOrigins(credential.origins);
|
|
1430
1504
|
const authenticatedFetch = async (url, requestOptions = {}) => {
|
|
1431
|
-
var _a, _b, _c;
|
|
1505
|
+
var _a, _b, _c, _d;
|
|
1506
|
+
requestOptions = mergeFetchOptions(fetchOptions, requestOptions);
|
|
1507
|
+
if ((_a = requestOptions.signal) == null ? void 0 : _a.aborted) throw createAbortError4();
|
|
1508
|
+
if (!credentials.length) return baseFetch(url, requestOptions);
|
|
1432
1509
|
const resolvedURL = parseHTTPURL(url);
|
|
1433
|
-
|
|
1434
|
-
const
|
|
1435
|
-
const response = await baseFetch(
|
|
1510
|
+
const authorization = resolvedURL ? await authorizeRequest(resolvedURL, requestOptions, tokenCredentials) : { url, options: requestOptions, credentials: [] };
|
|
1511
|
+
const signedRequest = await authenticateRequest(authorization, requestCredentials, "request");
|
|
1512
|
+
const response = await baseFetch(signedRequest.url, signedRequest.options);
|
|
1436
1513
|
const refreshableCredentials = authorization.credentials.filter(
|
|
1437
1514
|
({ credential }) => typeof credential.token === "function" && credential.refreshStatusCodes.includes(response.status)
|
|
1438
1515
|
);
|
|
1439
|
-
|
|
1440
|
-
|
|
1516
|
+
const shouldRetrySignature = signedRequest.credentials.some(
|
|
1517
|
+
(credential) => {
|
|
1518
|
+
var _a2;
|
|
1519
|
+
return (_a2 = credential.refreshStatusCodes) == null ? void 0 : _a2.includes(response.status);
|
|
1520
|
+
}
|
|
1521
|
+
);
|
|
1522
|
+
if (!refreshableCredentials.length && !shouldRetrySignature || !isReplayableRequest(requestOptions) || !isReplayableRequest(signedRequest.options))
|
|
1523
|
+
return response;
|
|
1524
|
+
if ((_b = requestOptions.signal) == null ? void 0 : _b.aborted) throw createAbortError4();
|
|
1441
1525
|
const refreshedTokens = /* @__PURE__ */ new Map();
|
|
1442
1526
|
await Promise.all(
|
|
1443
1527
|
refreshableCredentials.map(async ({ credential }) => {
|
|
@@ -1445,24 +1529,111 @@ function createAuthenticatedFetch(options) {
|
|
|
1445
1529
|
if (token) refreshedTokens.set(credential, token);
|
|
1446
1530
|
})
|
|
1447
1531
|
);
|
|
1448
|
-
if (!refreshedTokens.size) return response;
|
|
1449
|
-
await ((
|
|
1532
|
+
if (!refreshedTokens.size && !shouldRetrySignature) return response;
|
|
1533
|
+
await ((_c = response.body) == null ? void 0 : _c.cancel().catch(() => {
|
|
1450
1534
|
}));
|
|
1451
|
-
if ((
|
|
1452
|
-
const replayAuthorization = await authorizeRequest(
|
|
1535
|
+
if ((_d = requestOptions.signal) == null ? void 0 : _d.aborted) throw createAbortError4();
|
|
1536
|
+
const replayAuthorization = resolvedURL ? await authorizeRequest(
|
|
1453
1537
|
resolvedURL,
|
|
1454
1538
|
requestOptions,
|
|
1455
|
-
|
|
1539
|
+
tokenCredentials,
|
|
1456
1540
|
refreshedTokens,
|
|
1457
1541
|
authorization.credentials
|
|
1542
|
+
) : { url, options: requestOptions };
|
|
1543
|
+
const replayRequest = await authenticateRequest(
|
|
1544
|
+
replayAuthorization,
|
|
1545
|
+
requestCredentials,
|
|
1546
|
+
"retry",
|
|
1547
|
+
response
|
|
1458
1548
|
);
|
|
1459
|
-
return baseFetch(
|
|
1549
|
+
return baseFetch(replayRequest.url, replayRequest.options);
|
|
1460
1550
|
};
|
|
1461
1551
|
Object.defineProperty(authenticatedFetch, AUTHENTICATED_FETCH, {
|
|
1462
|
-
value:
|
|
1552
|
+
value: {
|
|
1553
|
+
fetch: baseFetch,
|
|
1554
|
+
credentials,
|
|
1555
|
+
credentialResolutions,
|
|
1556
|
+
fetchOptions
|
|
1557
|
+
}
|
|
1463
1558
|
});
|
|
1464
1559
|
return authenticatedFetch;
|
|
1465
1560
|
}
|
|
1561
|
+
function getAuthenticatedFetch(options = {}) {
|
|
1562
|
+
var _a, _b, _c;
|
|
1563
|
+
const fetchOption = options.fetch ?? ((_a = options.core) == null ? void 0 : _a.fetch);
|
|
1564
|
+
let fetchFunction;
|
|
1565
|
+
if (typeof fetchOption === "function") {
|
|
1566
|
+
fetchFunction = fetchOption;
|
|
1567
|
+
} else {
|
|
1568
|
+
fetchFunction = (url, requestOptions) => fetch(url, requestOptions);
|
|
1569
|
+
}
|
|
1570
|
+
return createAuthenticatedFetch({
|
|
1571
|
+
fetch: fetchFunction,
|
|
1572
|
+
fetchOptions: typeof fetchOption === "object" && fetchOption ? fetchOption : void 0,
|
|
1573
|
+
credentials: ((_b = options.core) == null ? void 0 : _b.credentials) || [],
|
|
1574
|
+
authentications: (_c = options.core) == null ? void 0 : _c.authentications
|
|
1575
|
+
});
|
|
1576
|
+
}
|
|
1577
|
+
function mergeFetchOptions(defaultOptions, requestOptions) {
|
|
1578
|
+
const mergedOptions = { ...defaultOptions, ...requestOptions };
|
|
1579
|
+
if (defaultOptions.headers || (requestOptions == null ? void 0 : requestOptions.headers)) {
|
|
1580
|
+
const headers = new Headers(defaultOptions.headers);
|
|
1581
|
+
new Headers(requestOptions == null ? void 0 : requestOptions.headers).forEach((value, key) => headers.set(key, value));
|
|
1582
|
+
mergedOptions.headers = headers;
|
|
1583
|
+
}
|
|
1584
|
+
return mergedOptions;
|
|
1585
|
+
}
|
|
1586
|
+
async function authenticateRequest(request, credentials, reason, response) {
|
|
1587
|
+
const appliedCredentials = [];
|
|
1588
|
+
const signal = request.options.signal;
|
|
1589
|
+
for (const credential of credentials) {
|
|
1590
|
+
const origin = getRequestOrigin(request.url);
|
|
1591
|
+
if (!origin || !normalizeRequestOrigins(credential.origins).includes(origin)) continue;
|
|
1592
|
+
if (signal == null ? void 0 : signal.aborted) throw createAbortError4();
|
|
1593
|
+
const authenticatedRequest = await credential.authenticate({
|
|
1594
|
+
url: request.url,
|
|
1595
|
+
options: { ...request.options, headers: new Headers(request.options.headers) },
|
|
1596
|
+
reason,
|
|
1597
|
+
response: response ? { status: response.status, headers: new Headers(response.headers) } : void 0
|
|
1598
|
+
});
|
|
1599
|
+
if (!authenticatedRequest || typeof authenticatedRequest.url !== "string" || !authenticatedRequest.options || typeof authenticatedRequest.options !== "object") {
|
|
1600
|
+
throw new Error("Authentication callback returned an invalid request.");
|
|
1601
|
+
}
|
|
1602
|
+
request = {
|
|
1603
|
+
url: authenticatedRequest.url,
|
|
1604
|
+
options: { ...authenticatedRequest.options, ...signal ? { signal } : {} }
|
|
1605
|
+
};
|
|
1606
|
+
appliedCredentials.push(credential);
|
|
1607
|
+
}
|
|
1608
|
+
if (signal == null ? void 0 : signal.aborted) throw createAbortError4();
|
|
1609
|
+
return { ...request, credentials: appliedCredentials };
|
|
1610
|
+
}
|
|
1611
|
+
function getRequestOrigin(url) {
|
|
1612
|
+
try {
|
|
1613
|
+
const parsedURL = new URL(url);
|
|
1614
|
+
if (!parsedURL.host || parsedURL.username || parsedURL.password) return null;
|
|
1615
|
+
return parsedURL.origin !== "null" ? parsedURL.origin : `${parsedURL.protocol}//${parsedURL.host}`;
|
|
1616
|
+
} catch {
|
|
1617
|
+
return null;
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
function normalizeRequestOrigins(origins) {
|
|
1621
|
+
if (!origins.length)
|
|
1622
|
+
throw new Error("Request authentication requires at least one exact origin.");
|
|
1623
|
+
return origins.map((origin) => {
|
|
1624
|
+
const resolvedOrigin = getRequestOrigin(origin);
|
|
1625
|
+
let url;
|
|
1626
|
+
try {
|
|
1627
|
+
url = new URL(origin);
|
|
1628
|
+
} catch {
|
|
1629
|
+
throw new Error("Invalid request authentication origin.");
|
|
1630
|
+
}
|
|
1631
|
+
if (!resolvedOrigin || url.pathname && url.pathname !== "/" || url.search || url.hash) {
|
|
1632
|
+
throw new Error("Invalid request authentication origin.");
|
|
1633
|
+
}
|
|
1634
|
+
return resolvedOrigin;
|
|
1635
|
+
});
|
|
1636
|
+
}
|
|
1466
1637
|
async function authorizeRequest(sourceURL, requestOptions, credentials, tokenOverrides = /* @__PURE__ */ new Map(), previousCredentials = []) {
|
|
1467
1638
|
const url = new URL(sourceURL);
|
|
1468
1639
|
const headers = new Headers(requestOptions.headers);
|
|
@@ -1531,6 +1702,7 @@ var AUTHENTICATED_FETCH, refreshPromises;
|
|
|
1531
1702
|
var init_request_credentials = __esm({
|
|
1532
1703
|
"../loader-utils/src/lib/request-utils/request-credentials.ts"() {
|
|
1533
1704
|
"use strict";
|
|
1705
|
+
init_authentication();
|
|
1534
1706
|
AUTHENTICATED_FETCH = /* @__PURE__ */ Symbol("loaders.gl.authenticated-fetch");
|
|
1535
1707
|
refreshPromises = /* @__PURE__ */ new WeakMap();
|
|
1536
1708
|
}
|
|
@@ -1776,6 +1948,7 @@ var init_range_request_scheduler = __esm({
|
|
|
1776
1948
|
/** Optional event callback for range batching diagnostics. */
|
|
1777
1949
|
onEvent;
|
|
1778
1950
|
pendingRequests = [];
|
|
1951
|
+
/** Pending batch window, independent of in-flight transport requests. */
|
|
1779
1952
|
flushTimer = null;
|
|
1780
1953
|
/** Creates a scheduler for one group of byte-range-addressable resources. */
|
|
1781
1954
|
constructor(props = {}) {
|
|
@@ -1852,7 +2025,7 @@ var init_range_request_scheduler = __esm({
|
|
|
1852
2025
|
}
|
|
1853
2026
|
/** Immediately starts loading the currently queued requests. */
|
|
1854
2027
|
flush() {
|
|
1855
|
-
if (this.flushTimer) {
|
|
2028
|
+
if (this.flushTimer !== null) {
|
|
1856
2029
|
clearTimeout(this.flushTimer);
|
|
1857
2030
|
this.flushTimer = null;
|
|
1858
2031
|
}
|
|
@@ -1874,9 +2047,9 @@ var init_range_request_scheduler = __esm({
|
|
|
1874
2047
|
this.fetchMergedRequest(mergedRequest);
|
|
1875
2048
|
}
|
|
1876
2049
|
}
|
|
1877
|
-
/**
|
|
2050
|
+
/** Starts a fixed batch window only when no flush is already scheduled. */
|
|
1878
2051
|
scheduleFlush() {
|
|
1879
|
-
if (this.flushTimer) {
|
|
2052
|
+
if (this.flushTimer !== null) {
|
|
1880
2053
|
return;
|
|
1881
2054
|
}
|
|
1882
2055
|
this.flushTimer = setTimeout(() => this.flush(), this.props.batchDelayMs);
|
|
@@ -2396,39 +2569,25 @@ var init_blob_file = __esm({
|
|
|
2396
2569
|
|
|
2397
2570
|
// ../loader-utils/src/lib/sources/data-source.ts
|
|
2398
2571
|
function getFetchFunction(options) {
|
|
2399
|
-
|
|
2400
|
-
const fetchOption = (options == null ? void 0 : options.fetch) ?? ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a.fetch);
|
|
2401
|
-
let fetchFunction;
|
|
2402
|
-
if (typeof fetchOption === "function") {
|
|
2403
|
-
fetchFunction = (url, fetchOptions) => fetchOption(url, fetchOptions);
|
|
2404
|
-
} else if (fetchOption) {
|
|
2405
|
-
fetchFunction = (url, requestOptions) => fetch(url, mergeFetchOptions(fetchOption, requestOptions));
|
|
2406
|
-
} else {
|
|
2407
|
-
fetchFunction = (url, requestOptions) => fetch(url, requestOptions);
|
|
2408
|
-
}
|
|
2409
|
-
return createAuthenticatedFetch({
|
|
2410
|
-
fetch: fetchFunction,
|
|
2411
|
-
credentials: ((_b = options == null ? void 0 : options.core) == null ? void 0 : _b.credentials) || []
|
|
2412
|
-
});
|
|
2572
|
+
return getAuthenticatedFetch(options);
|
|
2413
2573
|
}
|
|
2414
|
-
function
|
|
2415
|
-
|
|
2416
|
-
if (
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
}
|
|
2421
|
-
function mergeHeaders(defaultHeaders, requestHeaders) {
|
|
2422
|
-
const headers = new Headers(defaultHeaders);
|
|
2423
|
-
if (requestHeaders) {
|
|
2424
|
-
new Headers(requestHeaders).forEach((value, key) => headers.set(key, value));
|
|
2574
|
+
function getSourceLoaderOptions(options) {
|
|
2575
|
+
var _a;
|
|
2576
|
+
if (options.loadOptions !== void 0 || ((_a = options.core) == null ? void 0 : _a.loadOptions) !== void 0) {
|
|
2577
|
+
throw new Error(
|
|
2578
|
+
"Source loadOptions and core.loadOptions were removed. Use core and root loader namespaces directly."
|
|
2579
|
+
);
|
|
2425
2580
|
}
|
|
2426
|
-
return headers;
|
|
2427
|
-
}
|
|
2428
|
-
function normalizeDirectLoaderOptions(options) {
|
|
2429
2581
|
const loadOptions = { ...options };
|
|
2430
2582
|
if (options == null ? void 0 : options.core) {
|
|
2431
2583
|
loadOptions.core = { ...options.core };
|
|
2584
|
+
const parserCore = loadOptions.core;
|
|
2585
|
+
for (const key of ["type", "attributions", "loaders", "onError", "loadOptions"]) {
|
|
2586
|
+
delete parserCore[key];
|
|
2587
|
+
}
|
|
2588
|
+
if (Object.keys(parserCore).length === 0) {
|
|
2589
|
+
delete loadOptions.core;
|
|
2590
|
+
}
|
|
2432
2591
|
}
|
|
2433
2592
|
const topLevelBaseUri = typeof loadOptions.baseUri === "string" ? loadOptions.baseUri : void 0;
|
|
2434
2593
|
const topLevelBaseUrl = typeof loadOptions.baseUrl === "string" ? loadOptions.baseUrl : void 0;
|
|
@@ -2466,6 +2625,7 @@ var init_data_source = __esm({
|
|
|
2466
2625
|
init_file_aliases();
|
|
2467
2626
|
init_log();
|
|
2468
2627
|
init_request_credentials();
|
|
2628
|
+
init_authentication();
|
|
2469
2629
|
UNAVAILABLE_CORE_API = {
|
|
2470
2630
|
fetchFile: unavailableCoreApiMethod("fetchFile"),
|
|
2471
2631
|
parse: unavailableCoreApiMethod("parse"),
|
|
@@ -2480,7 +2640,6 @@ var init_data_source = __esm({
|
|
|
2480
2640
|
core: {
|
|
2481
2641
|
type: "auto",
|
|
2482
2642
|
attributions: [],
|
|
2483
|
-
loadOptions: {},
|
|
2484
2643
|
loaders: [],
|
|
2485
2644
|
onError: void 0
|
|
2486
2645
|
}
|
|
@@ -2489,9 +2648,9 @@ var init_data_source = __esm({
|
|
|
2489
2648
|
options;
|
|
2490
2649
|
data;
|
|
2491
2650
|
url;
|
|
2492
|
-
/**
|
|
2651
|
+
/** Shared parser options derived from the source options, excluding source-only core controls. */
|
|
2493
2652
|
loadOptions;
|
|
2494
|
-
/** A resolved fetch function extracted from
|
|
2653
|
+
/** A resolved fetch function extracted from the shared core options. */
|
|
2495
2654
|
fetch;
|
|
2496
2655
|
/** Shared source-level runtime hooks, when supplied by the source factory. */
|
|
2497
2656
|
coreApi;
|
|
@@ -2499,15 +2658,13 @@ var init_data_source = __esm({
|
|
|
2499
2658
|
hasCoreApi;
|
|
2500
2659
|
_needsRefresh = true;
|
|
2501
2660
|
constructor(data, options, defaultOptions, coreApi) {
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
this.options = { ...options };
|
|
2507
|
-
}
|
|
2661
|
+
this.options = mergeOptions(
|
|
2662
|
+
mergeOptions(_DataSource.defaultOptions, defaultOptions || {}),
|
|
2663
|
+
options
|
|
2664
|
+
);
|
|
2508
2665
|
this.data = data;
|
|
2509
2666
|
this.url = typeof data === "string" ? resolvePath(data) : "";
|
|
2510
|
-
const loadOptions =
|
|
2667
|
+
const loadOptions = resolveAuthenticationOptions(getSourceLoaderOptions(this.options));
|
|
2511
2668
|
this.loadOptions = loadOptions;
|
|
2512
2669
|
const fetch2 = getFetchFunction(loadOptions);
|
|
2513
2670
|
this.coreApi = coreApi || UNAVAILABLE_CORE_API;
|
|
@@ -2515,7 +2672,11 @@ var init_data_source = __esm({
|
|
|
2515
2672
|
this.fetch = fetch2;
|
|
2516
2673
|
}
|
|
2517
2674
|
setProps(options) {
|
|
2518
|
-
|
|
2675
|
+
const mergedOptions = mergeOptions(this.options, options);
|
|
2676
|
+
const loadOptions = resolveAuthenticationOptions(getSourceLoaderOptions(mergedOptions));
|
|
2677
|
+
this.options = mergedOptions;
|
|
2678
|
+
this.loadOptions = loadOptions;
|
|
2679
|
+
this.fetch = getFetchFunction(loadOptions);
|
|
2519
2680
|
this.setNeedsRefresh();
|
|
2520
2681
|
}
|
|
2521
2682
|
/** Mark this data source as needing a refresh (redraw) */
|
|
@@ -2851,6 +3012,10 @@ function getDataTypeFromTypedArray(array) {
|
|
|
2851
3012
|
return "int32";
|
|
2852
3013
|
case Uint32Array:
|
|
2853
3014
|
return "uint32";
|
|
3015
|
+
case BigInt64Array:
|
|
3016
|
+
return "int64";
|
|
3017
|
+
case BigUint64Array:
|
|
3018
|
+
return "uint64";
|
|
2854
3019
|
case Float32Array:
|
|
2855
3020
|
return "float32";
|
|
2856
3021
|
case Float64Array:
|
|
@@ -17718,6 +17883,18 @@ function readUnsignedVarIntNumber(cursor) {
|
|
|
17718
17883
|
}
|
|
17719
17884
|
throw new Error("invalid RLE run header");
|
|
17720
17885
|
}
|
|
17886
|
+
function encodeUnsignedVarIntNumber(value) {
|
|
17887
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
17888
|
+
throw new Error("RLE run header must be a non-negative safe integer");
|
|
17889
|
+
}
|
|
17890
|
+
const bytes = [];
|
|
17891
|
+
while (value >= 128) {
|
|
17892
|
+
bytes.push(value % 128 | 128);
|
|
17893
|
+
value = Math.floor(value / 128);
|
|
17894
|
+
}
|
|
17895
|
+
bytes.push(value);
|
|
17896
|
+
return Uint8Array.from(bytes);
|
|
17897
|
+
}
|
|
17721
17898
|
function assertReadable(cursor, byteLength) {
|
|
17722
17899
|
const size = cursor.size ?? cursor.buffer.length;
|
|
17723
17900
|
if (byteLength < 0 || cursor.offset + byteLength > size) {
|
|
@@ -17739,10 +17916,7 @@ function encodeRunBitpacked(values, opts) {
|
|
|
17739
17916
|
buf[Math.floor(b / 8)] |= 1 << b % 8;
|
|
17740
17917
|
}
|
|
17741
17918
|
}
|
|
17742
|
-
return concatUint8Arrays2([
|
|
17743
|
-
Uint8Array.from(import_varint.default.encode(paddedValues.length / 8 << 1 | 1)),
|
|
17744
|
-
buf
|
|
17745
|
-
]);
|
|
17919
|
+
return concatUint8Arrays2([encodeUnsignedVarIntNumber(paddedValues.length / 8 << 1 | 1), buf]);
|
|
17746
17920
|
}
|
|
17747
17921
|
function encodeRunRepeated(value, count, opts) {
|
|
17748
17922
|
const bitWidth = opts.bitWidth;
|
|
@@ -17751,15 +17925,13 @@ function encodeRunRepeated(value, count, opts) {
|
|
|
17751
17925
|
buf[i] = value & 255;
|
|
17752
17926
|
value = Math.floor(value / 256);
|
|
17753
17927
|
}
|
|
17754
|
-
return concatUint8Arrays2([
|
|
17928
|
+
return concatUint8Arrays2([encodeUnsignedVarIntNumber(count << 1), buf]);
|
|
17755
17929
|
}
|
|
17756
|
-
var import_varint;
|
|
17757
17930
|
var init_rle = __esm({
|
|
17758
17931
|
"src/parquetjs/codecs/rle.ts"() {
|
|
17759
17932
|
"use strict";
|
|
17760
17933
|
init_declare();
|
|
17761
17934
|
init_binary_utils();
|
|
17762
|
-
import_varint = __toESM(require("varint"), 1);
|
|
17763
17935
|
}
|
|
17764
17936
|
});
|
|
17765
17937
|
|
|
@@ -30997,7 +31169,7 @@ var PARQUET_LOADER_DEFAULT_OPTIONS = {
|
|
|
30997
31169
|
};
|
|
30998
31170
|
|
|
30999
31171
|
// dist/parquet-loader-base.js
|
|
31000
|
-
var VERSION4 = true ? "5.0.0-alpha.
|
|
31172
|
+
var VERSION4 = true ? "5.0.0-alpha.8" : "latest";
|
|
31001
31173
|
var PARQUET_LOADER_BASE = {
|
|
31002
31174
|
...ParquetFormat,
|
|
31003
31175
|
dataType: null,
|
|
@@ -31347,7 +31519,7 @@ var PARQUET_SOURCE_CAPABILITIES = Object.freeze({
|
|
|
31347
31519
|
});
|
|
31348
31520
|
|
|
31349
31521
|
// dist/parquet-source-loader-types.js
|
|
31350
|
-
var VERSION7 = true ? "5.0.0-alpha.
|
|
31522
|
+
var VERSION7 = true ? "5.0.0-alpha.8" : "latest";
|
|
31351
31523
|
async function preloadParquetSourceLoader2() {
|
|
31352
31524
|
const { ParquetSourceLoaderWithParser: ParquetSourceLoaderWithParser3 } = await Promise.resolve().then(() => (init_parquet_source_loader(), parquet_source_loader_exports));
|
|
31353
31525
|
return ParquetSourceLoaderWithParser3;
|
|
@@ -31863,7 +32035,6 @@ function toPrimitiveBytes2(value) {
|
|
|
31863
32035
|
}
|
|
31864
32036
|
|
|
31865
32037
|
// dist/parquetjs/codecs/rle.js
|
|
31866
|
-
var import_varint2 = __toESM(require("varint"), 1);
|
|
31867
32038
|
function encodeValues6(type, values, opts) {
|
|
31868
32039
|
if (!("bitWidth" in opts)) {
|
|
31869
32040
|
throw new Error("bitWidth is required");
|
|
@@ -32442,6 +32613,18 @@ function readUnsignedVarIntNumber3(cursor) {
|
|
|
32442
32613
|
}
|
|
32443
32614
|
throw new Error("invalid RLE run header");
|
|
32444
32615
|
}
|
|
32616
|
+
function encodeUnsignedVarIntNumber2(value) {
|
|
32617
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
32618
|
+
throw new Error("RLE run header must be a non-negative safe integer");
|
|
32619
|
+
}
|
|
32620
|
+
const bytes = [];
|
|
32621
|
+
while (value >= 128) {
|
|
32622
|
+
bytes.push(value % 128 | 128);
|
|
32623
|
+
value = Math.floor(value / 128);
|
|
32624
|
+
}
|
|
32625
|
+
bytes.push(value);
|
|
32626
|
+
return Uint8Array.from(bytes);
|
|
32627
|
+
}
|
|
32445
32628
|
function assertReadable3(cursor, byteLength) {
|
|
32446
32629
|
const size = cursor.size ?? cursor.buffer.length;
|
|
32447
32630
|
if (byteLength < 0 || cursor.offset + byteLength > size) {
|
|
@@ -32461,10 +32644,7 @@ function encodeRunBitpacked2(values, opts) {
|
|
|
32461
32644
|
buf[Math.floor(b / 8)] |= 1 << b % 8;
|
|
32462
32645
|
}
|
|
32463
32646
|
}
|
|
32464
|
-
return concatUint8Arrays([
|
|
32465
|
-
Uint8Array.from(import_varint2.default.encode(paddedValues.length / 8 << 1 | 1)),
|
|
32466
|
-
buf
|
|
32467
|
-
]);
|
|
32647
|
+
return concatUint8Arrays([encodeUnsignedVarIntNumber2(paddedValues.length / 8 << 1 | 1), buf]);
|
|
32468
32648
|
}
|
|
32469
32649
|
function encodeRunRepeated2(value, count, opts) {
|
|
32470
32650
|
const bitWidth = opts.bitWidth;
|
|
@@ -32473,7 +32653,7 @@ function encodeRunRepeated2(value, count, opts) {
|
|
|
32473
32653
|
buf[i] = value & 255;
|
|
32474
32654
|
value = Math.floor(value / 256);
|
|
32475
32655
|
}
|
|
32476
|
-
return concatUint8Arrays([
|
|
32656
|
+
return concatUint8Arrays([encodeUnsignedVarIntNumber2(count << 1), buf]);
|
|
32477
32657
|
}
|
|
32478
32658
|
|
|
32479
32659
|
// dist/parquetjs/codecs/dictionary.js
|