@vitessce/config 4.0.3 → 4.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1271 -824
- package/dist-tsc/generate-config-anndata.d.ts.map +1 -1
- package/dist-tsc/generate-config-anndata.js +3 -1
- package/dist-tsc/generate-config-anndata.js.map +1 -1
- package/dist-tsc/generate-config.d.ts +1 -9
- package/dist-tsc/generate-config.d.ts.map +1 -1
- package/dist-tsc/generate-config.js +3 -3
- package/dist-tsc/generate-config.js.map +1 -1
- package/dist-tsc/generate-config.test.js +2 -2
- package/dist-tsc/generate-config.test.js.map +1 -1
- package/package.json +7 -7
- package/src/generate-config-anndata.js +4 -1
- package/src/generate-config.js +9 -3
- package/src/generate-config.test.js +2 -2
package/dist/index.js
CHANGED
|
@@ -42,10 +42,10 @@ var hasRequiredPluralize;
|
|
|
42
42
|
function requirePluralize() {
|
|
43
43
|
if (hasRequiredPluralize) return pluralize$1.exports;
|
|
44
44
|
hasRequiredPluralize = 1;
|
|
45
|
-
(function(
|
|
45
|
+
(function(module, exports) {
|
|
46
46
|
(function(root, pluralize2) {
|
|
47
47
|
if (typeof commonjsRequire === "function" && true && true) {
|
|
48
|
-
|
|
48
|
+
module.exports = pluralize2();
|
|
49
49
|
} else {
|
|
50
50
|
root.pluralize = pluralize2();
|
|
51
51
|
}
|
|
@@ -444,10 +444,10 @@ var hasRequiredLoglevel;
|
|
|
444
444
|
function requireLoglevel() {
|
|
445
445
|
if (hasRequiredLoglevel) return loglevel$1.exports;
|
|
446
446
|
hasRequiredLoglevel = 1;
|
|
447
|
-
(function(
|
|
447
|
+
(function(module) {
|
|
448
448
|
(function(root, definition) {
|
|
449
|
-
if (
|
|
450
|
-
|
|
449
|
+
if (module.exports) {
|
|
450
|
+
module.exports = definition();
|
|
451
451
|
} else {
|
|
452
452
|
root.log = definition();
|
|
453
453
|
}
|
|
@@ -1433,10 +1433,10 @@ class VitessceConfig {
|
|
|
1433
1433
|
* @returns {VitessceConfig} A new config instance, with values set to match
|
|
1434
1434
|
* the config parameter.
|
|
1435
1435
|
*/
|
|
1436
|
-
static fromJSON(
|
|
1437
|
-
const { name, description, version: schemaVersion } =
|
|
1436
|
+
static fromJSON(config) {
|
|
1437
|
+
const { name, description, version: schemaVersion } = config;
|
|
1438
1438
|
const vc = new VitessceConfig({ schemaVersion, name, description });
|
|
1439
|
-
|
|
1439
|
+
config.datasets.forEach((d) => {
|
|
1440
1440
|
const newDataset = vc.addDataset(d.name, d.description, { uid: d.uid });
|
|
1441
1441
|
d.files.forEach((f) => {
|
|
1442
1442
|
newDataset.addFile({
|
|
@@ -1447,9 +1447,9 @@ class VitessceConfig {
|
|
|
1447
1447
|
});
|
|
1448
1448
|
});
|
|
1449
1449
|
});
|
|
1450
|
-
Object.keys(
|
|
1450
|
+
Object.keys(config.coordinationSpace).forEach((cType) => {
|
|
1451
1451
|
if (cType !== CoordinationType.DATASET) {
|
|
1452
|
-
const cObj =
|
|
1452
|
+
const cObj = config.coordinationSpace[cType];
|
|
1453
1453
|
vc.config.coordinationSpace[cType] = {};
|
|
1454
1454
|
Object.entries(cObj).forEach(([cScopeName, cScopeValue]) => {
|
|
1455
1455
|
const scope = new VitessceConfigCoordinationScope(cType, cScopeName);
|
|
@@ -1458,7 +1458,7 @@ class VitessceConfig {
|
|
|
1458
1458
|
});
|
|
1459
1459
|
}
|
|
1460
1460
|
});
|
|
1461
|
-
|
|
1461
|
+
config.layout.forEach((c) => {
|
|
1462
1462
|
const newView = new VitessceConfigView(
|
|
1463
1463
|
c.component,
|
|
1464
1464
|
c.coordinationScopes,
|
|
@@ -2095,10 +2095,10 @@ async function generateConfig$1(fileUrls, hintTitle = null) {
|
|
|
2095
2095
|
return vc.toJSON();
|
|
2096
2096
|
});
|
|
2097
2097
|
}
|
|
2098
|
-
function
|
|
2098
|
+
function stripPrefix(path) {
|
|
2099
2099
|
return path.slice(1);
|
|
2100
2100
|
}
|
|
2101
|
-
function
|
|
2101
|
+
function fetchRange(url, offset, length, opts = {}) {
|
|
2102
2102
|
if (offset !== void 0 && length !== void 0) {
|
|
2103
2103
|
opts = {
|
|
2104
2104
|
...opts,
|
|
@@ -2110,7 +2110,7 @@ function fetch_range(url, offset, length, opts = {}) {
|
|
|
2110
2110
|
}
|
|
2111
2111
|
return fetch(url, opts);
|
|
2112
2112
|
}
|
|
2113
|
-
function
|
|
2113
|
+
function mergeInit(storeOverrides, requestOverrides) {
|
|
2114
2114
|
return {
|
|
2115
2115
|
...storeOverrides,
|
|
2116
2116
|
...requestOverrides,
|
|
@@ -2133,7 +2133,7 @@ function resolve(root, path) {
|
|
|
2133
2133
|
resolved.search = base.search;
|
|
2134
2134
|
return resolved;
|
|
2135
2135
|
}
|
|
2136
|
-
async function
|
|
2136
|
+
async function handleResponse(response) {
|
|
2137
2137
|
if (response.status === 404) {
|
|
2138
2138
|
return void 0;
|
|
2139
2139
|
}
|
|
@@ -2142,55 +2142,156 @@ async function handle_response(response) {
|
|
|
2142
2142
|
}
|
|
2143
2143
|
throw new Error(`Unexpected response status ${response.status} ${response.statusText}`);
|
|
2144
2144
|
}
|
|
2145
|
-
async function fetch_suffix(url, suffix_length, init, use_suffix_request) {
|
|
2146
|
-
if (use_suffix_request) {
|
|
2147
|
-
return fetch(url, {
|
|
2148
|
-
...init,
|
|
2149
|
-
headers: { ...init.headers, Range: `bytes=-${suffix_length}` }
|
|
2150
|
-
});
|
|
2151
|
-
}
|
|
2152
|
-
let response = await fetch(url, { ...init, method: "HEAD" });
|
|
2153
|
-
if (!response.ok) {
|
|
2154
|
-
return response;
|
|
2155
|
-
}
|
|
2156
|
-
let content_length = response.headers.get("Content-Length");
|
|
2157
|
-
let length = Number(content_length);
|
|
2158
|
-
return fetch_range(url, length - suffix_length, length, init);
|
|
2159
|
-
}
|
|
2160
2145
|
class FetchStore {
|
|
2146
|
+
#fetch;
|
|
2161
2147
|
#overrides;
|
|
2162
|
-
#
|
|
2148
|
+
#useSuffixRequest;
|
|
2163
2149
|
constructor(url, options = {}) {
|
|
2164
2150
|
this.url = url;
|
|
2151
|
+
this.#fetch = options.fetch ?? ((request) => fetch(request));
|
|
2165
2152
|
this.#overrides = options.overrides ?? {};
|
|
2166
|
-
this.#
|
|
2153
|
+
this.#useSuffixRequest = options.useSuffixRequest ?? false;
|
|
2167
2154
|
}
|
|
2168
|
-
#
|
|
2169
|
-
return
|
|
2155
|
+
#buildRequest(url, init) {
|
|
2156
|
+
return new Request(url, mergeInit(this.#overrides, init));
|
|
2170
2157
|
}
|
|
2171
2158
|
async get(key, options = {}) {
|
|
2172
2159
|
let href = resolve(this.url, key).href;
|
|
2173
|
-
let
|
|
2174
|
-
|
|
2160
|
+
let request = this.#buildRequest(href, options);
|
|
2161
|
+
let response = await this.#fetch(request);
|
|
2162
|
+
return handleResponse(response);
|
|
2175
2163
|
}
|
|
2176
2164
|
async getRange(key, range, options = {}) {
|
|
2177
2165
|
let url = resolve(this.url, key);
|
|
2178
|
-
let init = this.#merge_init(options);
|
|
2179
2166
|
let response;
|
|
2180
2167
|
if ("suffixLength" in range) {
|
|
2181
|
-
response = await
|
|
2168
|
+
response = await this.#fetchSuffix(url, range.suffixLength, options);
|
|
2182
2169
|
} else {
|
|
2183
|
-
|
|
2170
|
+
let rangeInit = {
|
|
2171
|
+
...options,
|
|
2172
|
+
headers: {
|
|
2173
|
+
...options.headers,
|
|
2174
|
+
Range: `bytes=${range.offset}-${range.offset + range.length - 1}`
|
|
2175
|
+
}
|
|
2176
|
+
};
|
|
2177
|
+
let request = this.#buildRequest(url, rangeInit);
|
|
2178
|
+
response = await this.#fetch(request);
|
|
2184
2179
|
}
|
|
2185
|
-
return
|
|
2180
|
+
return handleResponse(response);
|
|
2186
2181
|
}
|
|
2182
|
+
async #fetchSuffix(url, suffixLength, options) {
|
|
2183
|
+
if (this.#useSuffixRequest) {
|
|
2184
|
+
let init = {
|
|
2185
|
+
...options,
|
|
2186
|
+
headers: { ...options.headers, Range: `bytes=-${suffixLength}` }
|
|
2187
|
+
};
|
|
2188
|
+
return this.#fetch(this.#buildRequest(url, init));
|
|
2189
|
+
}
|
|
2190
|
+
let headRequest = this.#buildRequest(url, {
|
|
2191
|
+
...options,
|
|
2192
|
+
method: "HEAD"
|
|
2193
|
+
});
|
|
2194
|
+
let response = await this.#fetch(headRequest);
|
|
2195
|
+
if (!response.ok) {
|
|
2196
|
+
return response;
|
|
2197
|
+
}
|
|
2198
|
+
let contentLength = response.headers.get("Content-Length");
|
|
2199
|
+
let length = Number(contentLength);
|
|
2200
|
+
let offset = length - suffixLength;
|
|
2201
|
+
let rangeInit = {
|
|
2202
|
+
...options,
|
|
2203
|
+
headers: {
|
|
2204
|
+
...options.headers,
|
|
2205
|
+
Range: `bytes=${offset}-${length - 1}`
|
|
2206
|
+
}
|
|
2207
|
+
};
|
|
2208
|
+
return this.#fetch(this.#buildRequest(url, rangeInit));
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
class ZarritaError extends Error {
|
|
2212
|
+
}
|
|
2213
|
+
class NotFoundError extends ZarritaError {
|
|
2214
|
+
constructor(context, options = {}) {
|
|
2215
|
+
super(`Not found: ${context}`, { cause: options.cause });
|
|
2216
|
+
this._tag = "NotFoundError";
|
|
2217
|
+
this.name = "NotFoundError";
|
|
2218
|
+
this.path = options.path;
|
|
2219
|
+
this.found = options.found;
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
class InvalidMetadataError extends ZarritaError {
|
|
2223
|
+
constructor(message, options = {}) {
|
|
2224
|
+
super(message, { cause: options.cause });
|
|
2225
|
+
this._tag = "InvalidMetadataError";
|
|
2226
|
+
this.name = "InvalidMetadataError";
|
|
2227
|
+
this.path = options.path;
|
|
2228
|
+
}
|
|
2229
|
+
}
|
|
2230
|
+
class UnknownCodecError extends ZarritaError {
|
|
2231
|
+
constructor(codec) {
|
|
2232
|
+
super(`Unknown codec: ${codec}`);
|
|
2233
|
+
this._tag = "UnknownCodecError";
|
|
2234
|
+
this.name = "UnknownCodecError";
|
|
2235
|
+
this.codec = codec;
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
class CodecPipelineError extends ZarritaError {
|
|
2239
|
+
constructor(options) {
|
|
2240
|
+
const parts = [
|
|
2241
|
+
`Failed to ${options.direction} chunk`,
|
|
2242
|
+
options.codec && `via codec "${options.codec}"`,
|
|
2243
|
+
options.chunkPath && `at ${options.chunkPath}`
|
|
2244
|
+
].filter(Boolean);
|
|
2245
|
+
super(parts.join(" "), { cause: options.cause });
|
|
2246
|
+
this._tag = "CodecPipelineError";
|
|
2247
|
+
this.name = "CodecPipelineError";
|
|
2248
|
+
this.direction = options.direction;
|
|
2249
|
+
this.codec = options.codec;
|
|
2250
|
+
this.chunkPath = options.chunkPath;
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
class UnsupportedError extends ZarritaError {
|
|
2254
|
+
constructor(feature) {
|
|
2255
|
+
super(`Unsupported: ${feature}`);
|
|
2256
|
+
this._tag = "UnsupportedError";
|
|
2257
|
+
this.name = "UnsupportedError";
|
|
2258
|
+
this.feature = feature;
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
function unimplementedEncode(codecName) {
|
|
2262
|
+
return () => {
|
|
2263
|
+
throw new UnsupportedError(`${codecName} encode`);
|
|
2264
|
+
};
|
|
2265
|
+
}
|
|
2266
|
+
class BitroundCodec {
|
|
2267
|
+
constructor(configuration, _meta) {
|
|
2268
|
+
this.kind = "array_to_array";
|
|
2269
|
+
this.encode = unimplementedEncode("bitround");
|
|
2270
|
+
if (configuration.keepbits < 0) {
|
|
2271
|
+
throw new InvalidMetadataError("keepbits must be zero or positive");
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
static fromConfig(configuration, meta) {
|
|
2275
|
+
return new BitroundCodec(configuration, meta);
|
|
2276
|
+
}
|
|
2277
|
+
/**
|
|
2278
|
+
* Decode a chunk of data (no-op).
|
|
2279
|
+
* @param arr - The chunk to decode
|
|
2280
|
+
* @returns The decoded chunk
|
|
2281
|
+
*/
|
|
2282
|
+
decode(arr) {
|
|
2283
|
+
return arr;
|
|
2284
|
+
}
|
|
2285
|
+
}
|
|
2286
|
+
function isArrayBufferLike(x) {
|
|
2287
|
+
return x instanceof ArrayBuffer || x instanceof SharedArrayBuffer;
|
|
2187
2288
|
}
|
|
2188
2289
|
class BoolArray {
|
|
2189
2290
|
#bytes;
|
|
2190
2291
|
constructor(x, byteOffset, length) {
|
|
2191
2292
|
if (typeof x === "number") {
|
|
2192
2293
|
this.#bytes = new Uint8Array(x);
|
|
2193
|
-
} else if (x
|
|
2294
|
+
} else if (isArrayBufferLike(x)) {
|
|
2194
2295
|
this.#bytes = new Uint8Array(x, byteOffset, length);
|
|
2195
2296
|
} else {
|
|
2196
2297
|
this.#bytes = new Uint8Array(Array.from(x, (v) => v ? 1 : 0));
|
|
@@ -2234,7 +2335,7 @@ class ByteStringArray {
|
|
|
2234
2335
|
this.#encoder = new TextEncoder();
|
|
2235
2336
|
if (typeof x === "number") {
|
|
2236
2337
|
this._data = new Uint8Array(x * chars);
|
|
2237
|
-
} else if (x
|
|
2338
|
+
} else if (isArrayBufferLike(x)) {
|
|
2238
2339
|
if (length)
|
|
2239
2340
|
length = length * chars;
|
|
2240
2341
|
this._data = new Uint8Array(x, byteOffset, length);
|
|
@@ -2288,7 +2389,7 @@ class UnicodeStringArray {
|
|
|
2288
2389
|
this.chars = chars;
|
|
2289
2390
|
if (typeof x === "number") {
|
|
2290
2391
|
this.#data = new Int32Array(x * chars);
|
|
2291
|
-
} else if (x
|
|
2392
|
+
} else if (isArrayBufferLike(x)) {
|
|
2292
2393
|
if (length)
|
|
2293
2394
|
length *= chars;
|
|
2294
2395
|
this.#data = new Int32Array(x, byteOffset, length);
|
|
@@ -2347,19 +2448,43 @@ class UnicodeStringArray {
|
|
|
2347
2448
|
}
|
|
2348
2449
|
}
|
|
2349
2450
|
}
|
|
2350
|
-
function
|
|
2351
|
-
const str = JSON.stringify(o,
|
|
2451
|
+
function jsonEncodeObject(o) {
|
|
2452
|
+
const str = JSON.stringify(o, (_key, value) => {
|
|
2453
|
+
if (typeof value === "number") {
|
|
2454
|
+
if (Number.isNaN(value))
|
|
2455
|
+
return "NaN";
|
|
2456
|
+
if (value === Infinity)
|
|
2457
|
+
return "Infinity";
|
|
2458
|
+
if (value === -Infinity)
|
|
2459
|
+
return "-Infinity";
|
|
2460
|
+
}
|
|
2461
|
+
return value;
|
|
2462
|
+
}, 2);
|
|
2352
2463
|
return new TextEncoder().encode(str);
|
|
2353
2464
|
}
|
|
2354
|
-
function
|
|
2465
|
+
function assertSharedArrayBufferAvailable() {
|
|
2466
|
+
if (typeof SharedArrayBuffer === "undefined") {
|
|
2467
|
+
throw new Error("SharedArrayBuffer is not available. In browsers, this requires Cross-Origin-Opener-Policy and Cross-Origin-Embedder-Policy headers to be set.");
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
2470
|
+
function createBuffer(byteLength, useShared) {
|
|
2471
|
+
{
|
|
2472
|
+
return new SharedArrayBuffer(byteLength);
|
|
2473
|
+
}
|
|
2474
|
+
}
|
|
2475
|
+
function jsonDecodeObject(bytes) {
|
|
2355
2476
|
const str = new TextDecoder().decode(bytes);
|
|
2356
|
-
|
|
2477
|
+
try {
|
|
2478
|
+
return JSON.parse(str);
|
|
2479
|
+
} catch (cause) {
|
|
2480
|
+
throw new InvalidMetadataError("Failed to decode JSON", { cause });
|
|
2481
|
+
}
|
|
2357
2482
|
}
|
|
2358
|
-
function
|
|
2359
|
-
const numFlips =
|
|
2360
|
-
const endByteIndex =
|
|
2483
|
+
function byteswapInplace(view, bytesPerElement2) {
|
|
2484
|
+
const numFlips = bytesPerElement2 / 2;
|
|
2485
|
+
const endByteIndex = bytesPerElement2 - 1;
|
|
2361
2486
|
let t = 0;
|
|
2362
|
-
for (let i = 0; i < view.length; i +=
|
|
2487
|
+
for (let i = 0; i < view.length; i += bytesPerElement2) {
|
|
2363
2488
|
for (let j = 0; j < numFlips; j += 1) {
|
|
2364
2489
|
t = view[i + j];
|
|
2365
2490
|
view[i + j] = view[i + endByteIndex - j];
|
|
@@ -2367,16 +2492,16 @@ function byteswap_inplace(view, bytes_per_element2) {
|
|
|
2367
2492
|
}
|
|
2368
2493
|
}
|
|
2369
2494
|
}
|
|
2370
|
-
function
|
|
2371
|
-
if (
|
|
2495
|
+
function getCtr(dataType) {
|
|
2496
|
+
if (dataType === "v2:object") {
|
|
2372
2497
|
return globalThis.Array;
|
|
2373
2498
|
}
|
|
2374
|
-
let match =
|
|
2499
|
+
let match = dataType.match(/v2:([US])(\d+)/);
|
|
2375
2500
|
if (match) {
|
|
2376
2501
|
let [, kind, chars] = match;
|
|
2377
2502
|
return (kind === "U" ? UnicodeStringArray : ByteStringArray).bind(null, Number(chars));
|
|
2378
2503
|
}
|
|
2379
|
-
if (
|
|
2504
|
+
if (dataType === "string") {
|
|
2380
2505
|
return globalThis.Array;
|
|
2381
2506
|
}
|
|
2382
2507
|
let ctr = {
|
|
@@ -2392,11 +2517,13 @@ function get_ctr(data_type) {
|
|
|
2392
2517
|
float32: Float32Array,
|
|
2393
2518
|
float64: Float64Array,
|
|
2394
2519
|
bool: BoolArray
|
|
2395
|
-
}[
|
|
2396
|
-
|
|
2520
|
+
}[dataType];
|
|
2521
|
+
if (!ctr) {
|
|
2522
|
+
throw new InvalidMetadataError(`Unknown or unsupported dataType: ${dataType}`);
|
|
2523
|
+
}
|
|
2397
2524
|
return ctr;
|
|
2398
2525
|
}
|
|
2399
|
-
function
|
|
2526
|
+
function getStrides(shape, order) {
|
|
2400
2527
|
const rank = shape.length;
|
|
2401
2528
|
if (typeof order === "string") {
|
|
2402
2529
|
order = order === "C" ? Array.from({ length: rank }, (_, i) => i) : Array.from({ length: rank }, (_, i) => rank - 1 - i);
|
|
@@ -2410,25 +2537,27 @@ function get_strides(shape, order) {
|
|
|
2410
2537
|
}
|
|
2411
2538
|
return stride;
|
|
2412
2539
|
}
|
|
2413
|
-
function
|
|
2540
|
+
function createChunkKeyEncoder({ name, configuration }) {
|
|
2414
2541
|
if (name === "default") {
|
|
2415
2542
|
const separator = configuration?.separator ?? "/";
|
|
2416
|
-
return (
|
|
2543
|
+
return (chunkCoords) => ["c", ...chunkCoords].join(separator);
|
|
2417
2544
|
}
|
|
2418
2545
|
if (name === "v2") {
|
|
2419
2546
|
const separator = configuration?.separator ?? ".";
|
|
2420
|
-
return (
|
|
2547
|
+
return (chunkCoords) => chunkCoords.join(separator) || "0";
|
|
2421
2548
|
}
|
|
2422
|
-
throw new
|
|
2549
|
+
throw new InvalidMetadataError(`Unknown chunk key encoding: ${name}`);
|
|
2423
2550
|
}
|
|
2424
|
-
function
|
|
2551
|
+
function coerceDtype(dtype) {
|
|
2425
2552
|
if (dtype === "|O") {
|
|
2426
|
-
return {
|
|
2553
|
+
return { dataType: "v2:object" };
|
|
2427
2554
|
}
|
|
2428
2555
|
let match = dtype.match(/^([<|>])(.*)$/);
|
|
2429
|
-
|
|
2556
|
+
if (!match) {
|
|
2557
|
+
throw new InvalidMetadataError(`Invalid dtype: ${dtype}`);
|
|
2558
|
+
}
|
|
2430
2559
|
let [, endian, rest] = match;
|
|
2431
|
-
let
|
|
2560
|
+
let dataType = {
|
|
2432
2561
|
b1: "bool",
|
|
2433
2562
|
i1: "int8",
|
|
2434
2563
|
u1: "uint8",
|
|
@@ -2442,33 +2571,73 @@ function coerce_dtype(dtype) {
|
|
|
2442
2571
|
f4: "float32",
|
|
2443
2572
|
f8: "float64"
|
|
2444
2573
|
}[rest] ?? (rest.startsWith("S") || rest.startsWith("U") ? `v2:${rest}` : void 0);
|
|
2445
|
-
|
|
2574
|
+
if (!dataType) {
|
|
2575
|
+
throw new InvalidMetadataError(`Unsupported or unknown dtype: ${dtype}`);
|
|
2576
|
+
}
|
|
2446
2577
|
if (endian === "|") {
|
|
2447
|
-
return {
|
|
2578
|
+
return { dataType };
|
|
2448
2579
|
}
|
|
2449
|
-
return {
|
|
2580
|
+
return { dataType, endian: endian === "<" ? "little" : "big" };
|
|
2450
2581
|
}
|
|
2451
|
-
function
|
|
2582
|
+
function isFixedScaleOffsetConfig(filter) {
|
|
2583
|
+
return (filter.id === "fixedscaleoffset" || filter.id === "numcodecs.fixedscaleoffset") && typeof filter.scale === "number" && typeof filter.offset === "number" && (filter.astype === void 0 || typeof filter.astype === "string") && (filter.dtype === void 0 || typeof filter.dtype === "string");
|
|
2584
|
+
}
|
|
2585
|
+
function v2ToV3ArrayMetadata(meta, attributes = {}) {
|
|
2452
2586
|
let codecs = [];
|
|
2453
|
-
let dtype =
|
|
2587
|
+
let dtype = coerceDtype(meta.dtype);
|
|
2454
2588
|
if (meta.order === "F") {
|
|
2455
2589
|
codecs.push({ name: "transpose", configuration: { order: "F" } });
|
|
2456
2590
|
}
|
|
2591
|
+
for (let filter of meta.filters ?? []) {
|
|
2592
|
+
if (filter.id === "fixedscaleoffset" || filter.id === "numcodecs.fixedscaleoffset") {
|
|
2593
|
+
if (!isFixedScaleOffsetConfig(filter)) {
|
|
2594
|
+
throw new InvalidMetadataError(`Invalid fixedscaleoffset filter: ${JSON.stringify(filter)}`);
|
|
2595
|
+
}
|
|
2596
|
+
codecs.push({
|
|
2597
|
+
name: "scale_offset",
|
|
2598
|
+
configuration: {
|
|
2599
|
+
scale: filter.scale,
|
|
2600
|
+
offset: filter.offset
|
|
2601
|
+
}
|
|
2602
|
+
});
|
|
2603
|
+
let astype = filter.astype ?? filter.dtype;
|
|
2604
|
+
if (astype !== void 0 && astype !== meta.dtype) {
|
|
2605
|
+
let castTarget = coerceDtype(astype).dataType;
|
|
2606
|
+
if (!isDataType(castTarget, "number") && !isDataType(castTarget, "bigint")) {
|
|
2607
|
+
throw new InvalidMetadataError(`fixedscaleoffset astype must be a numeric data type, got ${astype}`);
|
|
2608
|
+
}
|
|
2609
|
+
codecs.push({
|
|
2610
|
+
name: "cast_value",
|
|
2611
|
+
configuration: {
|
|
2612
|
+
data_type: castTarget,
|
|
2613
|
+
// `np.around` uses banker's rounding (round-half-to-even).
|
|
2614
|
+
rounding: "nearest-even",
|
|
2615
|
+
// Matches de-facto numpy integer-overflow behavior.
|
|
2616
|
+
out_of_range: "wrap"
|
|
2617
|
+
}
|
|
2618
|
+
});
|
|
2619
|
+
}
|
|
2620
|
+
continue;
|
|
2621
|
+
}
|
|
2622
|
+
let { id, ...configuration } = filter;
|
|
2623
|
+
codecs.push({ name: `numcodecs.${id}`, configuration });
|
|
2624
|
+
}
|
|
2457
2625
|
if ("endian" in dtype && dtype.endian === "big") {
|
|
2458
2626
|
codecs.push({ name: "bytes", configuration: { endian: "big" } });
|
|
2459
2627
|
}
|
|
2460
|
-
for (let { id, ...configuration } of meta.filters ?? []) {
|
|
2461
|
-
codecs.push({ name: id, configuration });
|
|
2462
|
-
}
|
|
2463
2628
|
if (meta.compressor) {
|
|
2464
2629
|
let { id, ...configuration } = meta.compressor;
|
|
2465
|
-
codecs.push({ name: id
|
|
2630
|
+
codecs.push({ name: `numcodecs.${id}`, configuration });
|
|
2631
|
+
}
|
|
2632
|
+
let dimensionNames;
|
|
2633
|
+
if (globalThis.Array.isArray(attributes._ARRAY_DIMENSIONS)) {
|
|
2634
|
+
dimensionNames = attributes._ARRAY_DIMENSIONS;
|
|
2466
2635
|
}
|
|
2467
2636
|
return {
|
|
2468
2637
|
zarr_format: 3,
|
|
2469
2638
|
node_type: "array",
|
|
2470
2639
|
shape: meta.shape,
|
|
2471
|
-
data_type: dtype.
|
|
2640
|
+
data_type: dtype.dataType,
|
|
2472
2641
|
chunk_grid: {
|
|
2473
2642
|
name: "regular",
|
|
2474
2643
|
configuration: {
|
|
@@ -2483,44 +2652,56 @@ function v2_to_v3_array_metadata(meta, attributes = {}) {
|
|
|
2483
2652
|
},
|
|
2484
2653
|
codecs,
|
|
2485
2654
|
fill_value: meta.fill_value,
|
|
2655
|
+
dimension_names: dimensionNames,
|
|
2486
2656
|
attributes
|
|
2487
2657
|
};
|
|
2488
2658
|
}
|
|
2489
|
-
function
|
|
2659
|
+
function v2ToV3GroupMetadata(_meta, attributes = {}) {
|
|
2490
2660
|
return {
|
|
2491
2661
|
zarr_format: 3,
|
|
2492
2662
|
node_type: "group",
|
|
2493
2663
|
attributes
|
|
2494
2664
|
};
|
|
2495
2665
|
}
|
|
2496
|
-
function
|
|
2666
|
+
function isDataType(dtype, query) {
|
|
2497
2667
|
if (query !== "number" && query !== "bigint" && query !== "boolean" && query !== "object" && query !== "string") {
|
|
2498
2668
|
return dtype === query;
|
|
2499
2669
|
}
|
|
2500
|
-
let
|
|
2670
|
+
let isBoolean = dtype === "bool";
|
|
2501
2671
|
if (query === "boolean")
|
|
2502
|
-
return
|
|
2503
|
-
let
|
|
2672
|
+
return isBoolean;
|
|
2673
|
+
let isString = dtype.startsWith("v2:U") || dtype.startsWith("v2:S") || dtype === "string";
|
|
2504
2674
|
if (query === "string")
|
|
2505
|
-
return
|
|
2506
|
-
let
|
|
2675
|
+
return isString;
|
|
2676
|
+
let isBigint = dtype === "int64" || dtype === "uint64";
|
|
2507
2677
|
if (query === "bigint")
|
|
2508
|
-
return
|
|
2509
|
-
let
|
|
2678
|
+
return isBigint;
|
|
2679
|
+
let isObject = dtype === "v2:object";
|
|
2510
2680
|
if (query === "object")
|
|
2511
|
-
return
|
|
2512
|
-
return !
|
|
2681
|
+
return isObject;
|
|
2682
|
+
return !isString && !isBigint && !isBoolean && !isObject;
|
|
2513
2683
|
}
|
|
2514
|
-
function
|
|
2684
|
+
function isShardingCodec(codec) {
|
|
2515
2685
|
return codec?.name === "sharding_indexed";
|
|
2516
2686
|
}
|
|
2517
|
-
function
|
|
2687
|
+
function ensureCorrectScalar(metadata) {
|
|
2518
2688
|
if ((metadata.data_type === "uint64" || metadata.data_type === "int64") && metadata.fill_value != null) {
|
|
2519
2689
|
return BigInt(metadata.fill_value);
|
|
2520
2690
|
}
|
|
2691
|
+
let isFloat = metadata.data_type === "float16" || metadata.data_type === "float32" || metadata.data_type === "float64";
|
|
2692
|
+
if (typeof metadata.fill_value === "string" && isFloat) {
|
|
2693
|
+
let mapping = {
|
|
2694
|
+
NaN: NaN,
|
|
2695
|
+
Infinity: Infinity,
|
|
2696
|
+
"-Infinity": -Infinity
|
|
2697
|
+
};
|
|
2698
|
+
if (metadata.fill_value in mapping) {
|
|
2699
|
+
return mapping[metadata.fill_value];
|
|
2700
|
+
}
|
|
2701
|
+
}
|
|
2521
2702
|
return metadata.fill_value;
|
|
2522
2703
|
}
|
|
2523
|
-
function
|
|
2704
|
+
function rethrowUnless(error, ...errors) {
|
|
2524
2705
|
if (!errors.some((ErrorClass) => error instanceof ErrorClass)) {
|
|
2525
2706
|
throw error;
|
|
2526
2707
|
}
|
|
@@ -2531,7 +2712,13 @@ function assert(expression, msg = "") {
|
|
|
2531
2712
|
}
|
|
2532
2713
|
}
|
|
2533
2714
|
async function decompress(data, { format, signal }) {
|
|
2534
|
-
|
|
2715
|
+
let response;
|
|
2716
|
+
if (data instanceof ArrayBuffer) {
|
|
2717
|
+
response = new Response(data);
|
|
2718
|
+
} else {
|
|
2719
|
+
let bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
2720
|
+
response = new Response(bytes.slice().buffer);
|
|
2721
|
+
}
|
|
2535
2722
|
assert(response.body, "Response does not contain body.");
|
|
2536
2723
|
try {
|
|
2537
2724
|
const decompressedResponse = new Response(response.body.pipeThrough(new DecompressionStream(format), { signal }));
|
|
@@ -2542,37 +2729,13 @@ async function decompress(data, { format, signal }) {
|
|
|
2542
2729
|
throw new Error(`Failed to decode ${format}`);
|
|
2543
2730
|
}
|
|
2544
2731
|
}
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
this.kind = "array_to_array";
|
|
2548
|
-
assert(configuration.keepbits >= 0, "keepbits must be zero or positive");
|
|
2549
|
-
}
|
|
2550
|
-
static fromConfig(configuration, meta) {
|
|
2551
|
-
return new BitroundCodec(configuration, meta);
|
|
2552
|
-
}
|
|
2553
|
-
/**
|
|
2554
|
-
* Encode a chunk of data with bit-rounding.
|
|
2555
|
-
* @param _arr - The chunk to encode
|
|
2556
|
-
*/
|
|
2557
|
-
encode(_arr) {
|
|
2558
|
-
throw new Error("`BitroundCodec.encode` is not implemented. Please open an issue at https://github.com/manzt/zarrita.js/issues.");
|
|
2559
|
-
}
|
|
2560
|
-
/**
|
|
2561
|
-
* Decode a chunk of data (no-op).
|
|
2562
|
-
* @param arr - The chunk to decode
|
|
2563
|
-
* @returns The decoded chunk
|
|
2564
|
-
*/
|
|
2565
|
-
decode(arr) {
|
|
2566
|
-
return arr;
|
|
2567
|
-
}
|
|
2568
|
-
}
|
|
2569
|
-
const LITTLE_ENDIAN_OS = system_is_little_endian();
|
|
2570
|
-
function system_is_little_endian() {
|
|
2732
|
+
const LITTLE_ENDIAN_OS = systemIsLittleEndian();
|
|
2733
|
+
function systemIsLittleEndian() {
|
|
2571
2734
|
const a = new Uint32Array([305419896]);
|
|
2572
2735
|
const b = new Uint8Array(a.buffer, a.byteOffset, a.byteLength);
|
|
2573
2736
|
return !(b[0] === 18);
|
|
2574
2737
|
}
|
|
2575
|
-
function
|
|
2738
|
+
function bytesPerElement(TypedArray) {
|
|
2576
2739
|
if ("BYTES_PER_ELEMENT" in TypedArray) {
|
|
2577
2740
|
return TypedArray.BYTES_PER_ELEMENT;
|
|
2578
2741
|
}
|
|
@@ -2582,9 +2745,9 @@ class BytesCodec {
|
|
|
2582
2745
|
constructor(configuration, meta) {
|
|
2583
2746
|
this.kind = "array_to_bytes";
|
|
2584
2747
|
this.#endian = configuration?.endian;
|
|
2585
|
-
this.#TypedArray =
|
|
2748
|
+
this.#TypedArray = getCtr(meta.dataType);
|
|
2586
2749
|
this.#shape = meta.shape;
|
|
2587
|
-
this.#stride =
|
|
2750
|
+
this.#stride = getStrides(meta.shape, "C");
|
|
2588
2751
|
const sample = new this.#TypedArray(0);
|
|
2589
2752
|
this.#BYTES_PER_ELEMENT = sample.BYTES_PER_ELEMENT;
|
|
2590
2753
|
}
|
|
@@ -2599,13 +2762,21 @@ class BytesCodec {
|
|
|
2599
2762
|
encode(arr) {
|
|
2600
2763
|
let bytes = new Uint8Array(arr.data.buffer);
|
|
2601
2764
|
if (LITTLE_ENDIAN_OS && this.#endian === "big") {
|
|
2602
|
-
|
|
2765
|
+
bytes = bytes.slice();
|
|
2766
|
+
byteswapInplace(bytes, bytesPerElement(this.#TypedArray));
|
|
2603
2767
|
}
|
|
2604
2768
|
return bytes;
|
|
2605
2769
|
}
|
|
2770
|
+
computeEncodedSize(decodedSize) {
|
|
2771
|
+
return decodedSize;
|
|
2772
|
+
}
|
|
2606
2773
|
decode(bytes) {
|
|
2607
2774
|
if (LITTLE_ENDIAN_OS && this.#endian === "big") {
|
|
2608
|
-
|
|
2775
|
+
bytes = bytes.slice();
|
|
2776
|
+
byteswapInplace(bytes, bytesPerElement(this.#TypedArray));
|
|
2777
|
+
}
|
|
2778
|
+
if (bytes.byteOffset % this.#BYTES_PER_ELEMENT !== 0) {
|
|
2779
|
+
bytes = bytes.slice();
|
|
2609
2780
|
}
|
|
2610
2781
|
return {
|
|
2611
2782
|
data: new this.#TypedArray(bytes.buffer, bytes.byteOffset, bytes.byteLength / this.#BYTES_PER_ELEMENT),
|
|
@@ -2614,42 +2785,359 @@ class BytesCodec {
|
|
|
2614
2785
|
};
|
|
2615
2786
|
}
|
|
2616
2787
|
}
|
|
2788
|
+
const SPECIAL_FLOATS = {
|
|
2789
|
+
NaN: NaN,
|
|
2790
|
+
Infinity: Infinity,
|
|
2791
|
+
"-Infinity": -Infinity
|
|
2792
|
+
};
|
|
2793
|
+
const FLOAT_BYTES = {
|
|
2794
|
+
float16: 2,
|
|
2795
|
+
float32: 4,
|
|
2796
|
+
float64: 8
|
|
2797
|
+
};
|
|
2798
|
+
function isFloatType(dataType) {
|
|
2799
|
+
return dataType in FLOAT_BYTES;
|
|
2800
|
+
}
|
|
2801
|
+
function isBigintType(dataType) {
|
|
2802
|
+
return dataType === "int64" || dataType === "uint64";
|
|
2803
|
+
}
|
|
2804
|
+
function hexToFloat(hex, byteWidth) {
|
|
2805
|
+
const int = BigInt(hex);
|
|
2806
|
+
const buf = new ArrayBuffer(byteWidth);
|
|
2807
|
+
const view = new DataView(buf);
|
|
2808
|
+
if (byteWidth === 2) {
|
|
2809
|
+
if (typeof view.getFloat16 !== "function") {
|
|
2810
|
+
throw new UnsupportedError("float16 hex-encoded scalar decoding (requires DataView.prototype.getFloat16)");
|
|
2811
|
+
}
|
|
2812
|
+
view.setUint16(0, Number(int));
|
|
2813
|
+
return view.getFloat16(0);
|
|
2814
|
+
}
|
|
2815
|
+
if (byteWidth === 4) {
|
|
2816
|
+
view.setUint32(0, Number(int));
|
|
2817
|
+
return view.getFloat32(0);
|
|
2818
|
+
}
|
|
2819
|
+
view.setBigUint64(0, int);
|
|
2820
|
+
return view.getFloat64(0);
|
|
2821
|
+
}
|
|
2822
|
+
function parseJsonScalar(dataType, value) {
|
|
2823
|
+
if (isBigintType(dataType)) {
|
|
2824
|
+
if (typeof value !== "number" || !Number.isInteger(value)) {
|
|
2825
|
+
throw new InvalidMetadataError(`Expected an integer value for data type "${dataType}", got ${JSON.stringify(value)}`);
|
|
2826
|
+
}
|
|
2827
|
+
return BigInt(value);
|
|
2828
|
+
}
|
|
2829
|
+
if (typeof value === "number") {
|
|
2830
|
+
if (!isFloatType(dataType) && !Number.isInteger(value)) {
|
|
2831
|
+
throw new InvalidMetadataError(`Expected an integer value for data type "${dataType}", got ${value}`);
|
|
2832
|
+
}
|
|
2833
|
+
return value;
|
|
2834
|
+
}
|
|
2835
|
+
if (!isFloatType(dataType)) {
|
|
2836
|
+
throw new InvalidMetadataError(`String-encoded scalar "${value}" is not valid for non-float data type "${dataType}"`);
|
|
2837
|
+
}
|
|
2838
|
+
if (value in SPECIAL_FLOATS) {
|
|
2839
|
+
return SPECIAL_FLOATS[value];
|
|
2840
|
+
}
|
|
2841
|
+
return hexToFloat(value, FLOAT_BYTES[dataType]);
|
|
2842
|
+
}
|
|
2843
|
+
const SUPPORTED$2 = /* @__PURE__ */ new Set([
|
|
2844
|
+
"int8",
|
|
2845
|
+
"uint8",
|
|
2846
|
+
"int16",
|
|
2847
|
+
"uint16",
|
|
2848
|
+
"int32",
|
|
2849
|
+
"uint32",
|
|
2850
|
+
"int64",
|
|
2851
|
+
"uint64",
|
|
2852
|
+
"float16",
|
|
2853
|
+
"float32",
|
|
2854
|
+
"float64"
|
|
2855
|
+
]);
|
|
2856
|
+
const INT_BOUNDS = {
|
|
2857
|
+
int8: [-128, 2 ** 7 - 1],
|
|
2858
|
+
uint8: [0, 2 ** 8 - 1],
|
|
2859
|
+
int16: [-32768, 2 ** 15 - 1],
|
|
2860
|
+
uint16: [0, 2 ** 16 - 1],
|
|
2861
|
+
int32: [-2147483648, 2 ** 31 - 1],
|
|
2862
|
+
uint32: [0, 2 ** 32 - 1]
|
|
2863
|
+
};
|
|
2864
|
+
const BIGINT_BOUNDS = {
|
|
2865
|
+
int64: [-(2n ** 63n), 2n ** 63n - 1n],
|
|
2866
|
+
uint64: [0n, 2n ** 64n - 1n]
|
|
2867
|
+
};
|
|
2868
|
+
function parseScalarMapEntries(entries, srcType, tgtType) {
|
|
2869
|
+
return entries.map(([src, tgt]) => ({
|
|
2870
|
+
src: parseJsonScalar(srcType, src),
|
|
2871
|
+
tgt: parseJsonScalar(tgtType, tgt)
|
|
2872
|
+
}));
|
|
2873
|
+
}
|
|
2874
|
+
function scalarMapLookup(value, entries) {
|
|
2875
|
+
for (const entry of entries) {
|
|
2876
|
+
if (typeof entry.src === "number" && Number.isNaN(entry.src)) {
|
|
2877
|
+
if (typeof value === "number" && Number.isNaN(value)) {
|
|
2878
|
+
return entry.tgt;
|
|
2879
|
+
}
|
|
2880
|
+
} else if (value === entry.src) {
|
|
2881
|
+
return entry.tgt;
|
|
2882
|
+
}
|
|
2883
|
+
}
|
|
2884
|
+
return void 0;
|
|
2885
|
+
}
|
|
2886
|
+
function roundNearestEven(value) {
|
|
2887
|
+
if (!Number.isFinite(value))
|
|
2888
|
+
return value;
|
|
2889
|
+
if (Math.abs(value - Math.trunc(value)) === 0.5) {
|
|
2890
|
+
const floor = Math.floor(value);
|
|
2891
|
+
const ceil = Math.ceil(value);
|
|
2892
|
+
return floor % 2 === 0 ? floor : ceil;
|
|
2893
|
+
}
|
|
2894
|
+
return Math.round(value);
|
|
2895
|
+
}
|
|
2896
|
+
function nearestAway(value) {
|
|
2897
|
+
return Math.sign(value) * Math.floor(Math.abs(value) + 0.5);
|
|
2898
|
+
}
|
|
2899
|
+
function getRoundingFn(mode) {
|
|
2900
|
+
switch (mode) {
|
|
2901
|
+
case "nearest-even":
|
|
2902
|
+
return roundNearestEven;
|
|
2903
|
+
case "towards-zero":
|
|
2904
|
+
return Math.trunc;
|
|
2905
|
+
case "towards-positive":
|
|
2906
|
+
return Math.ceil;
|
|
2907
|
+
case "towards-negative":
|
|
2908
|
+
return Math.floor;
|
|
2909
|
+
case "nearest-away":
|
|
2910
|
+
return nearestAway;
|
|
2911
|
+
}
|
|
2912
|
+
}
|
|
2913
|
+
function makeIntRangeCheck(lo, hi, outOfRange) {
|
|
2914
|
+
const range = hi - lo + 1;
|
|
2915
|
+
switch (outOfRange) {
|
|
2916
|
+
case "clamp":
|
|
2917
|
+
return (v) => v < lo ? lo : v > hi ? hi : v;
|
|
2918
|
+
case "wrap":
|
|
2919
|
+
return (v) => v >= lo && v <= hi ? v : ((v - lo) % range + range) % range + lo;
|
|
2920
|
+
default:
|
|
2921
|
+
return (v) => {
|
|
2922
|
+
if (v >= lo && v <= hi)
|
|
2923
|
+
return v;
|
|
2924
|
+
throw new Error(`Value ${v} out of range [${lo}, ${hi}]. Set out_of_range='clamp' or out_of_range='wrap' to handle this.`);
|
|
2925
|
+
};
|
|
2926
|
+
}
|
|
2927
|
+
}
|
|
2928
|
+
function makeBigintRangeCheck(lo, hi, outOfRange) {
|
|
2929
|
+
const range = hi - lo + 1n;
|
|
2930
|
+
switch (outOfRange) {
|
|
2931
|
+
case "clamp":
|
|
2932
|
+
return (v) => v < lo ? lo : v > hi ? hi : v;
|
|
2933
|
+
case "wrap":
|
|
2934
|
+
return (v) => v >= lo && v <= hi ? v : ((v - lo) % range + range) % range + lo;
|
|
2935
|
+
default:
|
|
2936
|
+
return (v) => {
|
|
2937
|
+
if (v >= lo && v <= hi)
|
|
2938
|
+
return v;
|
|
2939
|
+
throw new Error(`Value ${v} out of range [${lo}, ${hi}]. Set out_of_range='clamp' or out_of_range='wrap' to handle this.`);
|
|
2940
|
+
};
|
|
2941
|
+
}
|
|
2942
|
+
}
|
|
2943
|
+
class CastValueCodec {
|
|
2944
|
+
constructor(arrayType, encodedType, rounding, outOfRange, decodeMapEntries, encodeMapEntries) {
|
|
2945
|
+
this.kind = "array_to_array";
|
|
2946
|
+
this.encode = unimplementedEncode("cast_value");
|
|
2947
|
+
this.#encodedType = encodedType;
|
|
2948
|
+
this.#arrayTypeCtr = getCtr(arrayType);
|
|
2949
|
+
this.#decodeValue = buildConverter(encodedType, arrayType, rounding, outOfRange, decodeMapEntries);
|
|
2950
|
+
this.#encodeFillValue = buildConverter(arrayType, encodedType, rounding, outOfRange, encodeMapEntries);
|
|
2951
|
+
}
|
|
2952
|
+
#encodedType;
|
|
2953
|
+
#arrayTypeCtr;
|
|
2954
|
+
#decodeValue;
|
|
2955
|
+
#encodeFillValue;
|
|
2956
|
+
/** Return updated metadata reflecting the type and fill value after encoding. */
|
|
2957
|
+
getEncodedMeta(meta) {
|
|
2958
|
+
let fillValue = meta.fillValue;
|
|
2959
|
+
if (fillValue != null) {
|
|
2960
|
+
fillValue = this.#encodeFillValue(fillValue);
|
|
2961
|
+
}
|
|
2962
|
+
return { ...meta, dataType: this.#encodedType, fillValue };
|
|
2963
|
+
}
|
|
2964
|
+
static fromConfig(config, meta) {
|
|
2965
|
+
const arrayType = meta.dataType;
|
|
2966
|
+
const encodedType = config.data_type;
|
|
2967
|
+
if (!SUPPORTED$2.has(arrayType)) {
|
|
2968
|
+
throw new InvalidMetadataError(`cast_value codec does not support array data type: ${arrayType}`);
|
|
2969
|
+
}
|
|
2970
|
+
if (!SUPPORTED$2.has(encodedType)) {
|
|
2971
|
+
throw new InvalidMetadataError(`cast_value codec does not support encoded data type: ${encodedType}`);
|
|
2972
|
+
}
|
|
2973
|
+
const rounding = config.rounding ?? "nearest-even";
|
|
2974
|
+
const decodeMapEntries = config.scalar_map?.decode ? parseScalarMapEntries(config.scalar_map.decode, encodedType, arrayType) : [];
|
|
2975
|
+
const encodeMapEntries = config.scalar_map?.encode ? parseScalarMapEntries(config.scalar_map.encode, arrayType, encodedType) : [];
|
|
2976
|
+
return new CastValueCodec(arrayType, encodedType, rounding, config.out_of_range, decodeMapEntries, encodeMapEntries);
|
|
2977
|
+
}
|
|
2978
|
+
decode(chunk) {
|
|
2979
|
+
const input = chunk.data;
|
|
2980
|
+
const out = new this.#arrayTypeCtr(input.length);
|
|
2981
|
+
for (let i = 0; i < input.length; i++) {
|
|
2982
|
+
out[i] = this.#decodeValue(input[i]);
|
|
2983
|
+
}
|
|
2984
|
+
return { data: out, shape: chunk.shape, stride: chunk.stride };
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
function buildConverter(sourceType, targetType, rounding, outOfRange, mapEntries) {
|
|
2988
|
+
const srcIsFloat = isFloatType(sourceType);
|
|
2989
|
+
const srcIsBigint = isBigintType(sourceType);
|
|
2990
|
+
const dstIsFloat = isFloatType(targetType);
|
|
2991
|
+
const dstIsBigint = isBigintType(targetType);
|
|
2992
|
+
let baseFn;
|
|
2993
|
+
if (srcIsFloat && dstIsFloat) {
|
|
2994
|
+
if (rounding !== "nearest-even") {
|
|
2995
|
+
throw new InvalidMetadataError(`cast_value float -> float only supports "nearest-even" rounding, got "${rounding}"`);
|
|
2996
|
+
}
|
|
2997
|
+
baseFn = (v) => v;
|
|
2998
|
+
} else if (srcIsFloat && !dstIsFloat && !dstIsBigint) {
|
|
2999
|
+
const round = getRoundingFn(rounding);
|
|
3000
|
+
const check = makeIntRangeCheck(...INT_BOUNDS[targetType], outOfRange);
|
|
3001
|
+
baseFn = (v) => {
|
|
3002
|
+
if (!Number.isFinite(v)) {
|
|
3003
|
+
throw new Error(`Cannot cast ${v} to integer type without scalar_map`);
|
|
3004
|
+
}
|
|
3005
|
+
return check(round(v));
|
|
3006
|
+
};
|
|
3007
|
+
} else if (srcIsFloat && dstIsBigint) {
|
|
3008
|
+
const round = getRoundingFn(rounding);
|
|
3009
|
+
const check = makeBigintRangeCheck(...BIGINT_BOUNDS[targetType], outOfRange);
|
|
3010
|
+
baseFn = (v) => {
|
|
3011
|
+
if (!Number.isFinite(v)) {
|
|
3012
|
+
throw new Error(`Cannot cast ${v} to integer type without scalar_map`);
|
|
3013
|
+
}
|
|
3014
|
+
return check(BigInt(round(v)));
|
|
3015
|
+
};
|
|
3016
|
+
} else if (!srcIsFloat && !srcIsBigint && dstIsFloat) {
|
|
3017
|
+
baseFn = (v) => v;
|
|
3018
|
+
} else if (srcIsBigint && dstIsFloat) {
|
|
3019
|
+
baseFn = (v) => Number(v);
|
|
3020
|
+
} else if (!srcIsFloat && !srcIsBigint && !dstIsFloat && !dstIsBigint) {
|
|
3021
|
+
baseFn = makeIntRangeCheck(...INT_BOUNDS[targetType], outOfRange);
|
|
3022
|
+
} else if (!srcIsFloat && !srcIsBigint && dstIsBigint) {
|
|
3023
|
+
const check = makeBigintRangeCheck(...BIGINT_BOUNDS[targetType], outOfRange);
|
|
3024
|
+
baseFn = (v) => check(BigInt(v));
|
|
3025
|
+
} else if (srcIsBigint && !dstIsFloat && !dstIsBigint) {
|
|
3026
|
+
const check = makeIntRangeCheck(...INT_BOUNDS[targetType], outOfRange);
|
|
3027
|
+
baseFn = (v) => check(Number(v));
|
|
3028
|
+
} else if (srcIsBigint && dstIsBigint) {
|
|
3029
|
+
baseFn = makeBigintRangeCheck(...BIGINT_BOUNDS[targetType], outOfRange);
|
|
3030
|
+
} else {
|
|
3031
|
+
throw new Error(`Unhandled type combination: ${sourceType} -> ${targetType}`);
|
|
3032
|
+
}
|
|
3033
|
+
if (mapEntries.length === 0) {
|
|
3034
|
+
return baseFn;
|
|
3035
|
+
}
|
|
3036
|
+
const fn = (v) => {
|
|
3037
|
+
const mapped = scalarMapLookup(v, mapEntries);
|
|
3038
|
+
if (mapped !== void 0)
|
|
3039
|
+
return mapped;
|
|
3040
|
+
return baseFn(v);
|
|
3041
|
+
};
|
|
3042
|
+
return fn;
|
|
3043
|
+
}
|
|
2617
3044
|
class Crc32cCodec {
|
|
2618
3045
|
constructor() {
|
|
2619
3046
|
this.kind = "bytes_to_bytes";
|
|
3047
|
+
this.encode = unimplementedEncode("crc32c");
|
|
2620
3048
|
}
|
|
2621
3049
|
static fromConfig() {
|
|
2622
3050
|
return new Crc32cCodec();
|
|
2623
3051
|
}
|
|
2624
|
-
encode(_) {
|
|
2625
|
-
throw new Error("Not implemented");
|
|
2626
|
-
}
|
|
2627
3052
|
decode(arr) {
|
|
2628
3053
|
return new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength - 4);
|
|
2629
3054
|
}
|
|
3055
|
+
computeEncodedSize(decodedSize) {
|
|
3056
|
+
return decodedSize + 4;
|
|
3057
|
+
}
|
|
3058
|
+
}
|
|
3059
|
+
const SUPPORTED$1 = /* @__PURE__ */ new Set([
|
|
3060
|
+
"int8",
|
|
3061
|
+
"uint8",
|
|
3062
|
+
"int16",
|
|
3063
|
+
"uint16",
|
|
3064
|
+
"int32",
|
|
3065
|
+
"uint32",
|
|
3066
|
+
"int64",
|
|
3067
|
+
"uint64",
|
|
3068
|
+
"float16",
|
|
3069
|
+
"float32",
|
|
3070
|
+
"float64"
|
|
3071
|
+
]);
|
|
3072
|
+
function assertCOrFContiguous(shape, stride) {
|
|
3073
|
+
const n = shape.length;
|
|
3074
|
+
let c = n === 0 || stride[n - 1] === 1;
|
|
3075
|
+
for (let i = n - 2; i >= 0 && c; i--) {
|
|
3076
|
+
c = stride[i] === stride[i + 1] * shape[i + 1];
|
|
3077
|
+
}
|
|
3078
|
+
if (c)
|
|
3079
|
+
return;
|
|
3080
|
+
let f = n === 0 || stride[0] === 1;
|
|
3081
|
+
for (let i = 1; i < n && f; i++) {
|
|
3082
|
+
f = stride[i] === stride[i - 1] * shape[i - 1];
|
|
3083
|
+
}
|
|
3084
|
+
if (f)
|
|
3085
|
+
return;
|
|
3086
|
+
throw new Error(`DeltaCodec requires C- or Fortran-contiguous strides, got shape=${JSON.stringify(shape)} stride=${JSON.stringify(stride)}`);
|
|
3087
|
+
}
|
|
3088
|
+
class DeltaCodec {
|
|
3089
|
+
constructor(ctr) {
|
|
3090
|
+
this.kind = "array_to_array";
|
|
3091
|
+
this.#ctr = ctr;
|
|
3092
|
+
}
|
|
3093
|
+
#ctr;
|
|
3094
|
+
static fromConfig(_config, meta) {
|
|
3095
|
+
if (!SUPPORTED$1.has(meta.dataType)) {
|
|
3096
|
+
throw new InvalidMetadataError(`Delta codec does not support data type: ${meta.dataType}`);
|
|
3097
|
+
}
|
|
3098
|
+
return new DeltaCodec(getCtr(meta.dataType));
|
|
3099
|
+
}
|
|
3100
|
+
encode(chunk) {
|
|
3101
|
+
assertCOrFContiguous(chunk.shape, chunk.stride);
|
|
3102
|
+
const src = chunk.data;
|
|
3103
|
+
const out = new this.#ctr(src.length);
|
|
3104
|
+
out[0] = src[0];
|
|
3105
|
+
for (let i = 1; i < src.length; i++) {
|
|
3106
|
+
out[i] = src[i] - src[i - 1];
|
|
3107
|
+
}
|
|
3108
|
+
return { data: out, shape: chunk.shape, stride: chunk.stride };
|
|
3109
|
+
}
|
|
3110
|
+
decode(chunk) {
|
|
3111
|
+
assertCOrFContiguous(chunk.shape, chunk.stride);
|
|
3112
|
+
const src = chunk.data;
|
|
3113
|
+
const out = new this.#ctr(src.length);
|
|
3114
|
+
out[0] = src[0];
|
|
3115
|
+
for (let i = 1; i < src.length; i++) {
|
|
3116
|
+
out[i] = out[i - 1] + src[i];
|
|
3117
|
+
}
|
|
3118
|
+
return { data: out, shape: chunk.shape, stride: chunk.stride };
|
|
3119
|
+
}
|
|
2630
3120
|
}
|
|
2631
3121
|
class GzipCodec {
|
|
2632
3122
|
constructor() {
|
|
2633
3123
|
this.kind = "bytes_to_bytes";
|
|
3124
|
+
this.encode = unimplementedEncode("gzip");
|
|
2634
3125
|
}
|
|
2635
3126
|
static fromConfig(_) {
|
|
2636
3127
|
return new GzipCodec();
|
|
2637
3128
|
}
|
|
2638
|
-
encode(_bytes) {
|
|
2639
|
-
throw new Error("Gzip encoding is not enabled by default. Please register a custom codec with `numcodecs/gzip`.");
|
|
2640
|
-
}
|
|
2641
3129
|
async decode(bytes) {
|
|
2642
3130
|
const buffer = await decompress(bytes, { format: "gzip" });
|
|
2643
3131
|
return new Uint8Array(buffer);
|
|
2644
3132
|
}
|
|
2645
3133
|
}
|
|
2646
|
-
function
|
|
3134
|
+
function throwOnNanReplacer(_key, value) {
|
|
2647
3135
|
assert(!Number.isNaN(value), "JsonCodec allow_nan is false but NaN was encountered during encoding.");
|
|
2648
3136
|
assert(value !== Number.POSITIVE_INFINITY, "JsonCodec allow_nan is false but Infinity was encountered during encoding.");
|
|
2649
3137
|
assert(value !== Number.NEGATIVE_INFINITY, "JsonCodec allow_nan is false but -Infinity was encountered during encoding.");
|
|
2650
3138
|
return value;
|
|
2651
3139
|
}
|
|
2652
|
-
function
|
|
3140
|
+
function sortKeysReplacer(_key, value) {
|
|
2653
3141
|
return value instanceof Object && !Array.isArray(value) ? Object.keys(value).sort().reduce((sorted, key) => {
|
|
2654
3142
|
sorted[key] = value[key];
|
|
2655
3143
|
return sorted;
|
|
@@ -2668,7 +3156,7 @@ class JsonCodec {
|
|
|
2668
3156
|
separators = [", ", ": "];
|
|
2669
3157
|
}
|
|
2670
3158
|
}
|
|
2671
|
-
this.#
|
|
3159
|
+
this.#encoderConfig = {
|
|
2672
3160
|
encoding,
|
|
2673
3161
|
skipkeys,
|
|
2674
3162
|
ensure_ascii,
|
|
@@ -2678,59 +3166,142 @@ class JsonCodec {
|
|
|
2678
3166
|
separators,
|
|
2679
3167
|
sort_keys
|
|
2680
3168
|
};
|
|
2681
|
-
this.#
|
|
3169
|
+
this.#decoderConfig = { strict };
|
|
2682
3170
|
}
|
|
2683
|
-
#
|
|
2684
|
-
#
|
|
3171
|
+
#encoderConfig;
|
|
3172
|
+
#decoderConfig;
|
|
2685
3173
|
static fromConfig(configuration) {
|
|
2686
3174
|
return new JsonCodec(configuration);
|
|
2687
3175
|
}
|
|
2688
3176
|
encode(buf) {
|
|
2689
|
-
const { indent, encoding, ensure_ascii, check_circular, allow_nan, sort_keys } = this.#
|
|
3177
|
+
const { indent, encoding, ensure_ascii, check_circular, allow_nan, sort_keys } = this.#encoderConfig;
|
|
2690
3178
|
assert(encoding === "utf-8", "JsonCodec does not yet support non-utf-8 encoding.");
|
|
2691
|
-
const
|
|
3179
|
+
const replacerFunctions = [];
|
|
2692
3180
|
assert(check_circular, "JsonCodec does not yet support skipping the check for circular references during encoding.");
|
|
2693
3181
|
if (!allow_nan) {
|
|
2694
|
-
|
|
3182
|
+
replacerFunctions.push(throwOnNanReplacer);
|
|
2695
3183
|
}
|
|
2696
3184
|
if (sort_keys) {
|
|
2697
|
-
|
|
3185
|
+
replacerFunctions.push(sortKeysReplacer);
|
|
2698
3186
|
}
|
|
2699
3187
|
const items = Array.from(buf.data);
|
|
2700
3188
|
items.push("|O");
|
|
2701
3189
|
items.push(buf.shape);
|
|
2702
3190
|
let replacer;
|
|
2703
|
-
if (
|
|
3191
|
+
if (replacerFunctions.length) {
|
|
2704
3192
|
replacer = (key, value) => {
|
|
2705
|
-
let
|
|
2706
|
-
for (let
|
|
2707
|
-
|
|
3193
|
+
let newValue = value;
|
|
3194
|
+
for (let subReplacer of replacerFunctions) {
|
|
3195
|
+
newValue = subReplacer(key, newValue);
|
|
2708
3196
|
}
|
|
2709
|
-
return
|
|
3197
|
+
return newValue;
|
|
2710
3198
|
};
|
|
2711
3199
|
}
|
|
2712
|
-
let
|
|
3200
|
+
let jsonStr = JSON.stringify(items, replacer, indent);
|
|
2713
3201
|
if (ensure_ascii) {
|
|
2714
|
-
|
|
2715
|
-
const
|
|
2716
|
-
const
|
|
2717
|
-
return `\\u${
|
|
3202
|
+
jsonStr = jsonStr.replace(/[\u007F-\uFFFF]/g, (chr) => {
|
|
3203
|
+
const fullStr = `0000${chr.charCodeAt(0).toString(16)}`;
|
|
3204
|
+
const subStr = fullStr.substring(fullStr.length - 4);
|
|
3205
|
+
return `\\u${subStr}`;
|
|
2718
3206
|
});
|
|
2719
3207
|
}
|
|
2720
|
-
return new TextEncoder().encode(
|
|
3208
|
+
return new TextEncoder().encode(jsonStr);
|
|
2721
3209
|
}
|
|
2722
3210
|
decode(bytes) {
|
|
2723
|
-
const { strict } = this.#
|
|
3211
|
+
const { strict } = this.#decoderConfig;
|
|
2724
3212
|
assert(strict, "JsonCodec does not yet support non-strict decoding.");
|
|
2725
|
-
const items =
|
|
3213
|
+
const items = jsonDecodeObject(bytes);
|
|
2726
3214
|
const shape = items.pop();
|
|
2727
3215
|
items.pop();
|
|
2728
3216
|
assert(shape, "0D not implemented for JsonCodec.");
|
|
2729
|
-
const stride =
|
|
3217
|
+
const stride = getStrides(shape, "C");
|
|
2730
3218
|
const data = items;
|
|
2731
3219
|
return { data, shape, stride };
|
|
2732
3220
|
}
|
|
2733
3221
|
}
|
|
3222
|
+
const SUPPORTED = /* @__PURE__ */ new Set([
|
|
3223
|
+
"int8",
|
|
3224
|
+
"uint8",
|
|
3225
|
+
"int16",
|
|
3226
|
+
"uint16",
|
|
3227
|
+
"int32",
|
|
3228
|
+
"uint32",
|
|
3229
|
+
"int64",
|
|
3230
|
+
"uint64",
|
|
3231
|
+
"float16",
|
|
3232
|
+
"float32",
|
|
3233
|
+
"float64"
|
|
3234
|
+
]);
|
|
3235
|
+
class ScaleOffsetCodec {
|
|
3236
|
+
constructor(scale, offset, ctr) {
|
|
3237
|
+
this.kind = "array_to_array";
|
|
3238
|
+
this.encode = unimplementedEncode("scale_offset");
|
|
3239
|
+
this.#scale = scale;
|
|
3240
|
+
this.#offset = offset;
|
|
3241
|
+
this.#ctr = ctr;
|
|
3242
|
+
}
|
|
3243
|
+
#ctr;
|
|
3244
|
+
#scale;
|
|
3245
|
+
#offset;
|
|
3246
|
+
static fromConfig(config, meta) {
|
|
3247
|
+
if (!SUPPORTED.has(meta.dataType)) {
|
|
3248
|
+
throw new InvalidMetadataError(`scale_offset codec does not support data type: ${meta.dataType}`);
|
|
3249
|
+
}
|
|
3250
|
+
return new ScaleOffsetCodec(parseJsonScalar(meta.dataType, config.scale ?? 1), parseJsonScalar(meta.dataType, config.offset ?? 0), getCtr(meta.dataType));
|
|
3251
|
+
}
|
|
3252
|
+
decode(chunk) {
|
|
3253
|
+
const src = chunk.data;
|
|
3254
|
+
const out = new this.#ctr(src.length);
|
|
3255
|
+
for (let i = 0; i < src.length; i++) {
|
|
3256
|
+
out[i] = src[i] / this.#scale + this.#offset;
|
|
3257
|
+
}
|
|
3258
|
+
return { data: out, shape: chunk.shape, stride: chunk.stride };
|
|
3259
|
+
}
|
|
3260
|
+
}
|
|
3261
|
+
class ShuffleCodec {
|
|
3262
|
+
constructor(configuration, meta) {
|
|
3263
|
+
this.kind = "bytes_to_bytes";
|
|
3264
|
+
if (meta) {
|
|
3265
|
+
let sample = new (getCtr(meta.dataType))(0);
|
|
3266
|
+
assert("BYTES_PER_ELEMENT" in sample, `Shuffle codec requires a fixed-size dtype, got "${meta.dataType}"`);
|
|
3267
|
+
this.#BYTES_PER_ELEMENT = sample.BYTES_PER_ELEMENT;
|
|
3268
|
+
} else {
|
|
3269
|
+
this.#BYTES_PER_ELEMENT = configuration.elementsize ?? 4;
|
|
3270
|
+
}
|
|
3271
|
+
}
|
|
3272
|
+
#BYTES_PER_ELEMENT;
|
|
3273
|
+
static fromConfig(configuration, meta) {
|
|
3274
|
+
return new ShuffleCodec(configuration, meta);
|
|
3275
|
+
}
|
|
3276
|
+
encode(data) {
|
|
3277
|
+
return shuffle(data, this.#BYTES_PER_ELEMENT);
|
|
3278
|
+
}
|
|
3279
|
+
decode(data) {
|
|
3280
|
+
return unshuffle(data, this.#BYTES_PER_ELEMENT);
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
function shuffle(data, elementSize) {
|
|
3284
|
+
let length = data.length;
|
|
3285
|
+
let nElements = Math.floor(length / elementSize);
|
|
3286
|
+
let result = new Uint8Array(length);
|
|
3287
|
+
for (let byte = 0; byte < elementSize; byte++) {
|
|
3288
|
+
for (let i = 0; i < nElements; i++) {
|
|
3289
|
+
result[byte * nElements + i] = data[i * elementSize + byte];
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
return result;
|
|
3293
|
+
}
|
|
3294
|
+
function unshuffle(data, elementSize) {
|
|
3295
|
+
let length = data.length;
|
|
3296
|
+
let nElements = Math.floor(length / elementSize);
|
|
3297
|
+
let result = new Uint8Array(length);
|
|
3298
|
+
for (let byte = 0; byte < elementSize; byte++) {
|
|
3299
|
+
for (let i = 0; i < nElements; i++) {
|
|
3300
|
+
result[i * elementSize + byte] = data[byte * nElements + i];
|
|
3301
|
+
}
|
|
3302
|
+
}
|
|
3303
|
+
return result;
|
|
3304
|
+
}
|
|
2734
3305
|
function proxy(arr) {
|
|
2735
3306
|
if (arr instanceof BoolArray || arr instanceof ByteStringArray || arr instanceof UnicodeStringArray) {
|
|
2736
3307
|
const arrp = new Proxy(arr, {
|
|
@@ -2746,40 +3317,42 @@ function proxy(arr) {
|
|
|
2746
3317
|
}
|
|
2747
3318
|
return arr;
|
|
2748
3319
|
}
|
|
2749
|
-
function
|
|
3320
|
+
function emptyLike(chunk, order) {
|
|
2750
3321
|
let data;
|
|
2751
3322
|
if (chunk.data instanceof ByteStringArray || chunk.data instanceof UnicodeStringArray) {
|
|
2752
|
-
data = new chunk.constructor(
|
|
2753
|
-
// @ts-expect-error
|
|
2754
|
-
chunk.data.
|
|
2755
|
-
chunk.data.
|
|
3323
|
+
data = new chunk.data.constructor(
|
|
3324
|
+
// @ts-expect-error - the two argument form is not on the shared type
|
|
3325
|
+
chunk.data.chars,
|
|
3326
|
+
chunk.data.length
|
|
2756
3327
|
);
|
|
2757
3328
|
} else {
|
|
2758
|
-
data = new chunk.constructor(chunk.data.length);
|
|
3329
|
+
data = new chunk.data.constructor(chunk.data.length);
|
|
2759
3330
|
}
|
|
2760
3331
|
return {
|
|
2761
3332
|
data,
|
|
2762
3333
|
shape: chunk.shape,
|
|
2763
|
-
stride:
|
|
3334
|
+
stride: getStrides(chunk.shape, order)
|
|
2764
3335
|
};
|
|
2765
3336
|
}
|
|
2766
|
-
function
|
|
2767
|
-
let out =
|
|
2768
|
-
let
|
|
2769
|
-
let size = src.
|
|
2770
|
-
let index = Array(
|
|
2771
|
-
let
|
|
2772
|
-
let
|
|
2773
|
-
for (let
|
|
2774
|
-
let
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
3337
|
+
function convertArrayOrder(src, target) {
|
|
3338
|
+
let out = emptyLike(src, target);
|
|
3339
|
+
let nDims = src.shape.length;
|
|
3340
|
+
let size = src.shape.reduce((a, b) => a * b, 1);
|
|
3341
|
+
let index = Array(nDims).fill(0);
|
|
3342
|
+
let srcData = proxy(src.data);
|
|
3343
|
+
let outData = proxy(out.data);
|
|
3344
|
+
for (let n = 0; n < size; n++) {
|
|
3345
|
+
let srcIdx = 0;
|
|
3346
|
+
let outIdx = 0;
|
|
3347
|
+
for (let dim = 0; dim < nDims; dim++) {
|
|
3348
|
+
srcIdx += index[dim] * src.stride[dim];
|
|
3349
|
+
outIdx += index[dim] * out.stride[dim];
|
|
3350
|
+
}
|
|
3351
|
+
outData[outIdx] = srcData[srcIdx];
|
|
2779
3352
|
index[0] += 1;
|
|
2780
|
-
for (let dim = 0; dim <
|
|
3353
|
+
for (let dim = 0; dim < nDims; dim++) {
|
|
2781
3354
|
if (index[dim] === src.shape[dim]) {
|
|
2782
|
-
if (dim + 1 ===
|
|
3355
|
+
if (dim + 1 === nDims) {
|
|
2783
3356
|
break;
|
|
2784
3357
|
}
|
|
2785
3358
|
index[dim] = 0;
|
|
@@ -2789,13 +3362,13 @@ function convert_array_order(src, target) {
|
|
|
2789
3362
|
}
|
|
2790
3363
|
return out;
|
|
2791
3364
|
}
|
|
2792
|
-
function
|
|
3365
|
+
function getOrder(chunk) {
|
|
2793
3366
|
let rank = chunk.shape.length;
|
|
2794
3367
|
assert(rank === chunk.stride.length, "Shape and stride must have the same length.");
|
|
2795
3368
|
return chunk.stride.map((s, i) => ({ stride: s, index: i })).sort((a, b) => b.stride - a.stride).map((entry) => entry.index);
|
|
2796
3369
|
}
|
|
2797
|
-
function
|
|
2798
|
-
let source =
|
|
3370
|
+
function matchesOrder(chunk, target) {
|
|
3371
|
+
let source = getOrder(chunk);
|
|
2799
3372
|
assert(source.length === target.length, "Orders must match");
|
|
2800
3373
|
return source.every((dim, i) => dim === target[i]);
|
|
2801
3374
|
}
|
|
@@ -2805,70 +3378,64 @@ class TransposeCodec {
|
|
|
2805
3378
|
let value = configuration.order ?? "C";
|
|
2806
3379
|
let rank = meta.shape.length;
|
|
2807
3380
|
let order = new Array(rank);
|
|
2808
|
-
let inverseOrder = new Array(rank);
|
|
2809
3381
|
if (value === "C") {
|
|
2810
3382
|
for (let i = 0; i < rank; ++i) {
|
|
2811
3383
|
order[i] = i;
|
|
2812
|
-
inverseOrder[i] = i;
|
|
2813
3384
|
}
|
|
2814
3385
|
} else if (value === "F") {
|
|
2815
3386
|
for (let i = 0; i < rank; ++i) {
|
|
2816
3387
|
order[i] = rank - i - 1;
|
|
2817
|
-
inverseOrder[i] = rank - i - 1;
|
|
2818
3388
|
}
|
|
2819
3389
|
} else {
|
|
2820
3390
|
order = value;
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
3391
|
+
let seen = new Array(rank);
|
|
3392
|
+
order.forEach((x) => {
|
|
3393
|
+
assert(!seen[x], `Invalid permutation: ${JSON.stringify(value)}`);
|
|
3394
|
+
seen[x] = true;
|
|
2824
3395
|
});
|
|
2825
3396
|
}
|
|
2826
3397
|
this.#order = order;
|
|
2827
|
-
this.#inverseOrder = inverseOrder;
|
|
2828
3398
|
}
|
|
2829
3399
|
#order;
|
|
2830
|
-
#inverseOrder;
|
|
2831
3400
|
static fromConfig(configuration, meta) {
|
|
2832
3401
|
return new TransposeCodec(configuration, meta);
|
|
2833
3402
|
}
|
|
2834
3403
|
encode(arr) {
|
|
2835
|
-
if (
|
|
3404
|
+
if (matchesOrder(arr, this.#order)) {
|
|
2836
3405
|
return arr;
|
|
2837
3406
|
}
|
|
2838
|
-
return
|
|
3407
|
+
return convertArrayOrder(arr, this.#order);
|
|
2839
3408
|
}
|
|
2840
3409
|
decode(arr) {
|
|
2841
3410
|
return {
|
|
2842
3411
|
data: arr.data,
|
|
2843
3412
|
shape: arr.shape,
|
|
2844
|
-
stride:
|
|
3413
|
+
stride: getStrides(arr.shape, this.#order)
|
|
2845
3414
|
};
|
|
2846
3415
|
}
|
|
2847
3416
|
}
|
|
2848
3417
|
class VLenUTF8 {
|
|
2849
3418
|
constructor(shape) {
|
|
2850
3419
|
this.kind = "array_to_bytes";
|
|
3420
|
+
this.encode = unimplementedEncode("vlen-utf8");
|
|
2851
3421
|
this.#shape = shape;
|
|
2852
|
-
this.#strides =
|
|
3422
|
+
this.#strides = getStrides(shape, "C");
|
|
2853
3423
|
}
|
|
2854
3424
|
#shape;
|
|
2855
3425
|
#strides;
|
|
2856
3426
|
static fromConfig(_, meta) {
|
|
2857
3427
|
return new VLenUTF8(meta.shape);
|
|
2858
3428
|
}
|
|
2859
|
-
encode(_chunk) {
|
|
2860
|
-
throw new Error("Method not implemented.");
|
|
2861
|
-
}
|
|
2862
3429
|
decode(bytes) {
|
|
2863
3430
|
let decoder = new TextDecoder();
|
|
2864
3431
|
let view = new DataView(bytes.buffer);
|
|
2865
3432
|
let data = Array(view.getUint32(0, true));
|
|
2866
3433
|
let pos = 4;
|
|
2867
3434
|
for (let i = 0; i < data.length; i++) {
|
|
2868
|
-
let
|
|
3435
|
+
let itemLength = view.getUint32(pos, true);
|
|
2869
3436
|
pos += 4;
|
|
2870
|
-
data[i] = decoder.decode(bytes.buffer.slice(pos, pos +
|
|
2871
|
-
pos +=
|
|
3437
|
+
data[i] = decoder.decode(bytes.buffer.slice(pos, pos + itemLength));
|
|
3438
|
+
pos += itemLength;
|
|
2872
3439
|
}
|
|
2873
3440
|
return { data, shape: this.#shape, stride: this.#strides };
|
|
2874
3441
|
}
|
|
@@ -2876,195 +3443,181 @@ class VLenUTF8 {
|
|
|
2876
3443
|
class ZlibCodec {
|
|
2877
3444
|
constructor() {
|
|
2878
3445
|
this.kind = "bytes_to_bytes";
|
|
3446
|
+
this.encode = unimplementedEncode("zlib");
|
|
2879
3447
|
}
|
|
2880
3448
|
static fromConfig(_) {
|
|
2881
3449
|
return new ZlibCodec();
|
|
2882
3450
|
}
|
|
2883
|
-
encode(_bytes) {
|
|
2884
|
-
throw new Error("Zlib encoding is not enabled by default. Please register a codec with `numcodecs/zlib`.");
|
|
2885
|
-
}
|
|
2886
3451
|
async decode(bytes) {
|
|
2887
3452
|
const buffer = await decompress(bytes, { format: "deflate" });
|
|
2888
3453
|
return new Uint8Array(buffer);
|
|
2889
3454
|
}
|
|
2890
3455
|
}
|
|
2891
|
-
function
|
|
2892
|
-
|
|
3456
|
+
function createDefaultRegistry() {
|
|
3457
|
+
let blosc = () => import("./blosc-DvQQ1ST0.js").then((m) => m.default);
|
|
3458
|
+
let lz4 = () => import("./lz4-BIbM36RN.js").then((m) => m.default);
|
|
3459
|
+
let zstd = () => import("./zstd-CO575QiM.js").then((m) => m.default);
|
|
3460
|
+
let gzip = () => GzipCodec;
|
|
3461
|
+
let zlib = () => ZlibCodec;
|
|
3462
|
+
return (/* @__PURE__ */ new Map()).set("blosc", blosc).set("lz4", lz4).set("zstd", zstd).set("gzip", gzip).set("zlib", zlib).set("transpose", () => TransposeCodec).set("bytes", () => BytesCodec).set("crc32c", () => Crc32cCodec).set("vlen-utf8", () => VLenUTF8).set("json2", () => JsonCodec).set("bitround", () => BitroundCodec).set("cast_value", () => CastValueCodec).set("scale_offset", () => ScaleOffsetCodec).set("numcodecs.blosc", blosc).set("numcodecs.lz4", lz4).set("numcodecs.zstd", zstd).set("numcodecs.gzip", gzip).set("numcodecs.zlib", zlib).set("numcodecs.vlen-utf8", () => VLenUTF8).set("numcodecs.shuffle", () => ShuffleCodec).set("numcodecs.delta", () => DeltaCodec).set("numcodecs.bitround", () => BitroundCodec).set("numcodecs.json2", () => JsonCodec);
|
|
2893
3463
|
}
|
|
2894
|
-
const registry =
|
|
2895
|
-
function
|
|
2896
|
-
let
|
|
3464
|
+
const registry = createDefaultRegistry();
|
|
3465
|
+
function createCodecPipeline(chunkMetadata) {
|
|
3466
|
+
let codecsPromise;
|
|
3467
|
+
function getCodecs() {
|
|
3468
|
+
if (!codecsPromise)
|
|
3469
|
+
codecsPromise = loadCodecs(chunkMetadata);
|
|
3470
|
+
return codecsPromise;
|
|
3471
|
+
}
|
|
3472
|
+
async function runStep(direction, codec, fn) {
|
|
3473
|
+
try {
|
|
3474
|
+
return await fn();
|
|
3475
|
+
} catch (cause) {
|
|
3476
|
+
throw new CodecPipelineError({ direction, codec, cause });
|
|
3477
|
+
}
|
|
3478
|
+
}
|
|
2897
3479
|
return {
|
|
2898
3480
|
async encode(chunk) {
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
chunk = await codec.encode(chunk);
|
|
3481
|
+
let codecs = await getCodecs();
|
|
3482
|
+
for (const { name, codec } of codecs.arrayToArray) {
|
|
3483
|
+
chunk = await runStep("encode", name, () => codec.encode(chunk));
|
|
2903
3484
|
}
|
|
2904
|
-
let bytes = await codecs.
|
|
2905
|
-
for (const codec of codecs.
|
|
2906
|
-
bytes = await codec.encode(bytes);
|
|
3485
|
+
let bytes = await runStep("encode", codecs.arrayToBytes.name, () => codecs.arrayToBytes.codec.encode(chunk));
|
|
3486
|
+
for (const { name, codec } of codecs.bytesToBytes) {
|
|
3487
|
+
bytes = await runStep("encode", name, () => codec.encode(bytes));
|
|
2907
3488
|
}
|
|
2908
3489
|
return bytes;
|
|
2909
3490
|
},
|
|
2910
3491
|
async decode(bytes) {
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
bytes = await
|
|
3492
|
+
let codecs = await getCodecs();
|
|
3493
|
+
for (let i = codecs.bytesToBytes.length - 1; i >= 0; i--) {
|
|
3494
|
+
const { name, codec } = codecs.bytesToBytes[i];
|
|
3495
|
+
bytes = await runStep("decode", name, () => codec.decode(bytes));
|
|
2915
3496
|
}
|
|
2916
|
-
let chunk = await codecs.
|
|
2917
|
-
for (let i = codecs.
|
|
2918
|
-
|
|
3497
|
+
let chunk = await runStep("decode", codecs.arrayToBytes.name, () => codecs.arrayToBytes.codec.decode(bytes));
|
|
3498
|
+
for (let i = codecs.arrayToArray.length - 1; i >= 0; i--) {
|
|
3499
|
+
const { name, codec } = codecs.arrayToArray[i];
|
|
3500
|
+
chunk = await runStep("decode", name, () => codec.decode(chunk));
|
|
2919
3501
|
}
|
|
2920
3502
|
return chunk;
|
|
3503
|
+
},
|
|
3504
|
+
async computeEncodedSize(decodedSize) {
|
|
3505
|
+
let codecs = await getCodecs();
|
|
3506
|
+
let size = applyEncodedSize(codecs.arrayToBytes.name, codecs.arrayToBytes.codec, decodedSize);
|
|
3507
|
+
for (const { name, codec } of codecs.bytesToBytes) {
|
|
3508
|
+
size = applyEncodedSize(name, codec, size);
|
|
3509
|
+
}
|
|
3510
|
+
return size;
|
|
2921
3511
|
}
|
|
2922
3512
|
};
|
|
2923
3513
|
}
|
|
2924
|
-
|
|
2925
|
-
|
|
3514
|
+
function applyEncodedSize(name, codec, size) {
|
|
3515
|
+
if (!codec.computeEncodedSize) {
|
|
3516
|
+
throw new InvalidMetadataError(`Codec "${name}" cannot compute its encoded size; it is not a fixed-size codec and cannot be used in a sharding index pipeline`);
|
|
3517
|
+
}
|
|
3518
|
+
return codec.computeEncodedSize(size);
|
|
3519
|
+
}
|
|
3520
|
+
async function loadCodecs(chunkMeta) {
|
|
3521
|
+
let promises = chunkMeta.codecs.map(async (meta) => {
|
|
2926
3522
|
let Codec = await registry.get(meta.name)?.();
|
|
2927
|
-
|
|
3523
|
+
if (!Codec) {
|
|
3524
|
+
throw new UnknownCodecError(meta.name);
|
|
3525
|
+
}
|
|
2928
3526
|
return { Codec, meta };
|
|
2929
3527
|
});
|
|
2930
|
-
let
|
|
2931
|
-
let
|
|
2932
|
-
let
|
|
3528
|
+
let arrayToArray = [];
|
|
3529
|
+
let arrayToBytes;
|
|
3530
|
+
let bytesToBytes = [];
|
|
3531
|
+
let currentMeta = { ...chunkMeta };
|
|
2933
3532
|
for await (let { Codec, meta } of promises) {
|
|
2934
|
-
let codec = Codec.fromConfig(meta.configuration,
|
|
3533
|
+
let codec = Codec.fromConfig(meta.configuration, currentMeta);
|
|
2935
3534
|
switch (codec.kind) {
|
|
2936
3535
|
case "array_to_array":
|
|
2937
|
-
|
|
3536
|
+
arrayToArray.push({
|
|
3537
|
+
name: meta.name,
|
|
3538
|
+
codec
|
|
3539
|
+
});
|
|
3540
|
+
if (codec.getEncodedMeta) {
|
|
3541
|
+
currentMeta = codec.getEncodedMeta(currentMeta);
|
|
3542
|
+
}
|
|
2938
3543
|
break;
|
|
2939
3544
|
case "array_to_bytes":
|
|
2940
|
-
|
|
3545
|
+
arrayToBytes = {
|
|
3546
|
+
name: meta.name,
|
|
3547
|
+
codec
|
|
3548
|
+
};
|
|
2941
3549
|
break;
|
|
2942
3550
|
default:
|
|
2943
|
-
|
|
3551
|
+
bytesToBytes.push({
|
|
3552
|
+
name: meta.name,
|
|
3553
|
+
codec
|
|
3554
|
+
});
|
|
2944
3555
|
}
|
|
2945
3556
|
}
|
|
2946
|
-
if (!
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
}
|
|
2955
|
-
class NodeNotFoundError extends Error {
|
|
2956
|
-
constructor(context, options = {}) {
|
|
2957
|
-
super(`Node not found: ${context}`, options);
|
|
2958
|
-
this.name = "NodeNotFoundError";
|
|
2959
|
-
}
|
|
2960
|
-
}
|
|
2961
|
-
class KeyError extends Error {
|
|
2962
|
-
constructor(path) {
|
|
2963
|
-
super(`Missing key: ${path}`);
|
|
2964
|
-
this.name = "KeyError";
|
|
2965
|
-
}
|
|
2966
|
-
}
|
|
2967
|
-
async function get_consolidated_metadata(store, metadataKeyOption) {
|
|
2968
|
-
const metadataKey = metadataKeyOption ?? ".zmetadata";
|
|
2969
|
-
let bytes = await store.get(`/${metadataKey}`);
|
|
2970
|
-
if (!bytes) {
|
|
2971
|
-
throw new NodeNotFoundError("v2 consolidated metadata", {
|
|
2972
|
-
cause: new KeyError(`/${metadataKey}`)
|
|
2973
|
-
});
|
|
2974
|
-
}
|
|
2975
|
-
let meta = json_decode_object(bytes);
|
|
2976
|
-
assert(meta.zarr_consolidated_format === 1, "Unsupported consolidated format.");
|
|
2977
|
-
return meta;
|
|
2978
|
-
}
|
|
2979
|
-
function is_meta_key(key) {
|
|
2980
|
-
return key.endsWith(".zarray") || key.endsWith(".zgroup") || key.endsWith(".zattrs") || key.endsWith("zarr.json");
|
|
2981
|
-
}
|
|
2982
|
-
function is_v3(meta) {
|
|
2983
|
-
return "zarr_format" in meta && meta.zarr_format === 3;
|
|
2984
|
-
}
|
|
2985
|
-
async function withConsolidated(store, opts = {}) {
|
|
2986
|
-
let v2_meta = await get_consolidated_metadata(store, opts.metadataKey);
|
|
2987
|
-
let known_meta = {};
|
|
2988
|
-
for (let [key, value] of Object.entries(v2_meta.metadata)) {
|
|
2989
|
-
known_meta[`/${key}`] = value;
|
|
3557
|
+
if (!arrayToBytes) {
|
|
3558
|
+
if (!isTypedArrayLikeMeta(currentMeta)) {
|
|
3559
|
+
throw new InvalidMetadataError(`Cannot encode ${currentMeta.dataType} to bytes without a codec`);
|
|
3560
|
+
}
|
|
3561
|
+
arrayToBytes = {
|
|
3562
|
+
name: "bytes",
|
|
3563
|
+
codec: BytesCodec.fromConfig({ endian: "little" }, currentMeta)
|
|
3564
|
+
};
|
|
2990
3565
|
}
|
|
2991
3566
|
return {
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
return json_encode_object(known_meta[key]);
|
|
2996
|
-
}
|
|
2997
|
-
let maybe_bytes = await store.get(key, opts2);
|
|
2998
|
-
if (is_meta_key(key) && maybe_bytes) {
|
|
2999
|
-
let meta = json_decode_object(maybe_bytes);
|
|
3000
|
-
known_meta[key] = meta;
|
|
3001
|
-
}
|
|
3002
|
-
return maybe_bytes;
|
|
3003
|
-
},
|
|
3004
|
-
// Delegate range requests to the underlying store.
|
|
3005
|
-
// Note: Supporting range requests for consolidated metadata is possible
|
|
3006
|
-
// but unlikely to be useful enough to justify the effort.
|
|
3007
|
-
getRange: store.getRange?.bind(store),
|
|
3008
|
-
contents() {
|
|
3009
|
-
let contents = [];
|
|
3010
|
-
for (let [key, value] of Object.entries(known_meta)) {
|
|
3011
|
-
let parts = key.split("/");
|
|
3012
|
-
let filename = parts.pop();
|
|
3013
|
-
let path = parts.join("/") || "/";
|
|
3014
|
-
if (filename === ".zarray")
|
|
3015
|
-
contents.push({ path, kind: "array" });
|
|
3016
|
-
if (filename === ".zgroup")
|
|
3017
|
-
contents.push({ path, kind: "group" });
|
|
3018
|
-
if (is_v3(value)) {
|
|
3019
|
-
contents.push({ path, kind: value.node_type });
|
|
3020
|
-
}
|
|
3021
|
-
}
|
|
3022
|
-
return contents;
|
|
3023
|
-
}
|
|
3567
|
+
arrayToArray,
|
|
3568
|
+
arrayToBytes,
|
|
3569
|
+
bytesToBytes
|
|
3024
3570
|
};
|
|
3025
3571
|
}
|
|
3572
|
+
function isTypedArrayLikeMeta(meta) {
|
|
3573
|
+
return meta.dataType !== "v2:object" && meta.dataType !== "string";
|
|
3574
|
+
}
|
|
3026
3575
|
const MAX_BIG_UINT = 18446744073709551615n;
|
|
3027
|
-
function
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
let
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3576
|
+
function createShardedChunkGetter(location, shardShape, encodeShardKey, shardingConfig) {
|
|
3577
|
+
if (!location.store.getRange) {
|
|
3578
|
+
throw new UnsupportedError("sharding requires a store with getRange");
|
|
3579
|
+
}
|
|
3580
|
+
let getRange = location.store.getRange.bind(location.store);
|
|
3581
|
+
let indexShape = shardShape.map((d, i) => d / shardingConfig.chunk_shape[i]);
|
|
3582
|
+
let indexCodec = createCodecPipeline({
|
|
3583
|
+
dataType: "uint64",
|
|
3584
|
+
shape: [...indexShape, 2],
|
|
3585
|
+
codecs: shardingConfig.index_codecs,
|
|
3586
|
+
fillValue: null
|
|
3035
3587
|
});
|
|
3588
|
+
let rawIndexSize = 16 * indexShape.reduce((a, b) => a * b, 1);
|
|
3036
3589
|
let cache = {};
|
|
3037
|
-
return async (
|
|
3038
|
-
let
|
|
3039
|
-
let
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
}
|
|
3049
|
-
index = cache[shard_path] = bytes ? await index_codec.decode(bytes) : null;
|
|
3590
|
+
return async (chunkCoord, options) => {
|
|
3591
|
+
let shardCoord = chunkCoord.map((d, i) => Math.floor(d / indexShape[i]));
|
|
3592
|
+
let shardPath = location.resolve(encodeShardKey(shardCoord)).path;
|
|
3593
|
+
if (!(shardPath in cache)) {
|
|
3594
|
+
cache[shardPath] = (async () => {
|
|
3595
|
+
let suffixLength = await indexCodec.computeEncodedSize(rawIndexSize);
|
|
3596
|
+
let bytes = await getRange(shardPath, { suffixLength }, options);
|
|
3597
|
+
return bytes ? await indexCodec.decode(bytes) : null;
|
|
3598
|
+
})().catch((err) => {
|
|
3599
|
+
delete cache[shardPath];
|
|
3600
|
+
throw err;
|
|
3601
|
+
});
|
|
3050
3602
|
}
|
|
3603
|
+
let index = await cache[shardPath];
|
|
3051
3604
|
if (index === null) {
|
|
3052
3605
|
return void 0;
|
|
3053
3606
|
}
|
|
3054
3607
|
let { data, shape, stride } = index;
|
|
3055
|
-
let
|
|
3056
|
-
let offset = data[
|
|
3057
|
-
let length = data[
|
|
3608
|
+
let linearOffset = chunkCoord.map((d, i) => d % shape[i]).reduce((acc, sel, idx) => acc + sel * stride[idx], 0);
|
|
3609
|
+
let offset = data[linearOffset];
|
|
3610
|
+
let length = data[linearOffset + 1];
|
|
3058
3611
|
if (offset === MAX_BIG_UINT && length === MAX_BIG_UINT) {
|
|
3059
3612
|
return void 0;
|
|
3060
3613
|
}
|
|
3061
|
-
return
|
|
3614
|
+
return getRange(shardPath, {
|
|
3062
3615
|
offset: Number(offset),
|
|
3063
3616
|
length: Number(length)
|
|
3064
3617
|
}, options);
|
|
3065
3618
|
};
|
|
3066
3619
|
}
|
|
3067
|
-
var _a;
|
|
3620
|
+
var _a$1;
|
|
3068
3621
|
class Location {
|
|
3069
3622
|
constructor(store, path = "/") {
|
|
3070
3623
|
this.store = store;
|
|
@@ -3086,92 +3639,121 @@ class Group extends Location {
|
|
|
3086
3639
|
return this.#metadata.attributes;
|
|
3087
3640
|
}
|
|
3088
3641
|
}
|
|
3089
|
-
function
|
|
3090
|
-
const
|
|
3091
|
-
return
|
|
3642
|
+
function getArrayOrder(codecs) {
|
|
3643
|
+
const maybeTransposeCodec = codecs.find((c) => c.name === "transpose");
|
|
3644
|
+
return maybeTransposeCodec?.configuration?.order ?? "C";
|
|
3645
|
+
}
|
|
3646
|
+
function projectOrder(order, axes) {
|
|
3647
|
+
let rank = new Map(axes.map((axis, i) => [axis, i]));
|
|
3648
|
+
return order.filter((axis) => rank.has(axis)).map((axis) => rank.get(axis));
|
|
3649
|
+
}
|
|
3650
|
+
function makeStrideGetter(nativeOrder) {
|
|
3651
|
+
return (shape, axes) => {
|
|
3652
|
+
let order = globalThis.Array.isArray(nativeOrder) && axes ? projectOrder(nativeOrder, axes) : nativeOrder;
|
|
3653
|
+
return getStrides(shape, order);
|
|
3654
|
+
};
|
|
3092
3655
|
}
|
|
3093
3656
|
const CONTEXT_MARKER = Symbol("zarrita.context");
|
|
3094
|
-
function
|
|
3095
|
-
let { configuration } = metadata.codecs.find(
|
|
3096
|
-
let
|
|
3097
|
-
|
|
3098
|
-
TypedArray:
|
|
3099
|
-
|
|
3657
|
+
function createContext(location, metadata) {
|
|
3658
|
+
let { configuration } = metadata.codecs.find(isShardingCodec) ?? {};
|
|
3659
|
+
let sharedContext = {
|
|
3660
|
+
encodeChunkKey: createChunkKeyEncoder(metadata.chunk_key_encoding),
|
|
3661
|
+
TypedArray: getCtr(metadata.data_type),
|
|
3662
|
+
fillValue: metadata.fill_value
|
|
3100
3663
|
};
|
|
3101
3664
|
if (configuration) {
|
|
3102
|
-
let
|
|
3665
|
+
let nativeOrder2 = getArrayOrder(configuration.codecs);
|
|
3103
3666
|
return {
|
|
3104
|
-
...
|
|
3667
|
+
...sharedContext,
|
|
3105
3668
|
kind: "sharded",
|
|
3106
|
-
|
|
3107
|
-
codec:
|
|
3108
|
-
|
|
3669
|
+
chunkShape: configuration.chunk_shape,
|
|
3670
|
+
codec: createCodecPipeline({
|
|
3671
|
+
dataType: metadata.data_type,
|
|
3109
3672
|
shape: configuration.chunk_shape,
|
|
3110
|
-
codecs: configuration.codecs
|
|
3673
|
+
codecs: configuration.codecs,
|
|
3674
|
+
fillValue: metadata.fill_value
|
|
3111
3675
|
}),
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
},
|
|
3115
|
-
get_chunk_bytes: create_sharded_chunk_getter(location, metadata.chunk_grid.configuration.chunk_shape, shared_context.encode_chunk_key, configuration)
|
|
3676
|
+
getStrides: makeStrideGetter(nativeOrder2),
|
|
3677
|
+
getChunkBytes: createShardedChunkGetter(location, metadata.chunk_grid.configuration.chunk_shape, sharedContext.encodeChunkKey, configuration)
|
|
3116
3678
|
};
|
|
3117
3679
|
}
|
|
3118
|
-
let
|
|
3680
|
+
let nativeOrder = getArrayOrder(metadata.codecs);
|
|
3119
3681
|
return {
|
|
3120
|
-
...
|
|
3682
|
+
...sharedContext,
|
|
3121
3683
|
kind: "regular",
|
|
3122
|
-
|
|
3123
|
-
codec:
|
|
3124
|
-
|
|
3684
|
+
chunkShape: metadata.chunk_grid.configuration.chunk_shape,
|
|
3685
|
+
codec: createCodecPipeline({
|
|
3686
|
+
dataType: metadata.data_type,
|
|
3125
3687
|
shape: metadata.chunk_grid.configuration.chunk_shape,
|
|
3126
|
-
codecs: metadata.codecs
|
|
3688
|
+
codecs: metadata.codecs,
|
|
3689
|
+
fillValue: metadata.fill_value
|
|
3127
3690
|
}),
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
let chunk_path = location.resolve(chunk_key).path;
|
|
3134
|
-
return location.store.get(chunk_path, options);
|
|
3691
|
+
getStrides: makeStrideGetter(nativeOrder),
|
|
3692
|
+
async getChunkBytes(chunkCoords, options) {
|
|
3693
|
+
let chunkKey = sharedContext.encodeChunkKey(chunkCoords);
|
|
3694
|
+
let chunkPath = location.resolve(chunkKey).path;
|
|
3695
|
+
return location.store.get(chunkPath, options);
|
|
3135
3696
|
}
|
|
3136
3697
|
};
|
|
3137
3698
|
}
|
|
3138
|
-
let Array$1 = class Array2 extends (_a = Location, _a) {
|
|
3699
|
+
let Array$1 = class Array2 extends (_a$1 = Location, _a$1) {
|
|
3139
3700
|
constructor(store, path, metadata) {
|
|
3140
3701
|
super(store, path);
|
|
3141
3702
|
this.kind = "array";
|
|
3142
3703
|
this.#metadata = {
|
|
3143
3704
|
...metadata,
|
|
3144
|
-
fill_value:
|
|
3705
|
+
fill_value: ensureCorrectScalar(metadata)
|
|
3145
3706
|
};
|
|
3146
|
-
this[CONTEXT_MARKER] =
|
|
3707
|
+
this[CONTEXT_MARKER] = createContext(this, this.#metadata);
|
|
3147
3708
|
}
|
|
3148
3709
|
#metadata;
|
|
3149
3710
|
get attrs() {
|
|
3150
3711
|
return this.#metadata.attributes;
|
|
3151
3712
|
}
|
|
3713
|
+
get dimensionNames() {
|
|
3714
|
+
return this.#metadata.dimension_names;
|
|
3715
|
+
}
|
|
3716
|
+
get fillValue() {
|
|
3717
|
+
return this.#metadata.fill_value;
|
|
3718
|
+
}
|
|
3152
3719
|
get shape() {
|
|
3153
3720
|
return this.#metadata.shape;
|
|
3154
3721
|
}
|
|
3155
3722
|
get chunks() {
|
|
3156
|
-
return this[CONTEXT_MARKER].
|
|
3723
|
+
return this[CONTEXT_MARKER].chunkShape;
|
|
3157
3724
|
}
|
|
3158
3725
|
get dtype() {
|
|
3159
3726
|
return this.#metadata.data_type;
|
|
3160
3727
|
}
|
|
3161
|
-
async getChunk(
|
|
3728
|
+
async getChunk(chunkCoords, options, opts) {
|
|
3729
|
+
if (opts?.useSharedArrayBuffer) {
|
|
3730
|
+
assertSharedArrayBufferAvailable();
|
|
3731
|
+
}
|
|
3162
3732
|
let context = this[CONTEXT_MARKER];
|
|
3163
|
-
let
|
|
3164
|
-
if (!
|
|
3165
|
-
let size = context.
|
|
3166
|
-
let data
|
|
3167
|
-
|
|
3733
|
+
let maybeBytes = await context.getChunkBytes(chunkCoords, options);
|
|
3734
|
+
if (!maybeBytes) {
|
|
3735
|
+
let size = context.chunkShape.reduce((a, b) => a * b, 1);
|
|
3736
|
+
let data;
|
|
3737
|
+
if (opts?.useSharedArrayBuffer) {
|
|
3738
|
+
let sample = new context.TypedArray(0);
|
|
3739
|
+
if (!("BYTES_PER_ELEMENT" in sample)) {
|
|
3740
|
+
console.warn("zarrita: useSharedArrayBuffer is not supported for non-buffer-backed data types.");
|
|
3741
|
+
data = new context.TypedArray(size);
|
|
3742
|
+
} else {
|
|
3743
|
+
let buffer = createBuffer(size * sample.BYTES_PER_ELEMENT);
|
|
3744
|
+
data = new context.TypedArray(buffer, 0, size);
|
|
3745
|
+
}
|
|
3746
|
+
} else {
|
|
3747
|
+
data = new context.TypedArray(size);
|
|
3748
|
+
}
|
|
3749
|
+
data.fill(context.fillValue);
|
|
3168
3750
|
return {
|
|
3169
3751
|
data,
|
|
3170
|
-
shape: context.
|
|
3171
|
-
stride: context.
|
|
3752
|
+
shape: context.chunkShape,
|
|
3753
|
+
stride: context.getStrides(context.chunkShape)
|
|
3172
3754
|
};
|
|
3173
3755
|
}
|
|
3174
|
-
return context.codec.decode(
|
|
3756
|
+
return context.codec.decode(maybeBytes);
|
|
3175
3757
|
}
|
|
3176
3758
|
/**
|
|
3177
3759
|
* A helper method to narrow `zarr.Array` Dtype.
|
|
@@ -3191,86 +3773,168 @@ let Array$1 = class Array2 extends (_a = Location, _a) {
|
|
|
3191
3773
|
* ```
|
|
3192
3774
|
*/
|
|
3193
3775
|
is(query) {
|
|
3194
|
-
return
|
|
3776
|
+
return isDataType(this.dtype, query);
|
|
3195
3777
|
}
|
|
3196
3778
|
};
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3779
|
+
function createProxy(target, overrides) {
|
|
3780
|
+
let boundCache = /* @__PURE__ */ new Map();
|
|
3781
|
+
return new Proxy(target, {
|
|
3782
|
+
get(t, prop) {
|
|
3783
|
+
if (prop in overrides)
|
|
3784
|
+
return overrides[prop];
|
|
3785
|
+
let cached = boundCache.get(prop);
|
|
3786
|
+
if (cached !== void 0)
|
|
3787
|
+
return cached;
|
|
3788
|
+
let value = Reflect.get(t, prop, t);
|
|
3789
|
+
if (typeof value === "function") {
|
|
3790
|
+
let bound = value.bind(t);
|
|
3791
|
+
boundCache.set(prop, bound);
|
|
3792
|
+
return bound;
|
|
3793
|
+
}
|
|
3794
|
+
return value;
|
|
3795
|
+
},
|
|
3796
|
+
has(t, prop) {
|
|
3797
|
+
return prop in overrides || Reflect.has(t, prop);
|
|
3798
|
+
},
|
|
3799
|
+
ownKeys(t) {
|
|
3800
|
+
let keys = /* @__PURE__ */ new Set([...Reflect.ownKeys(t), ...Object.keys(overrides)]);
|
|
3801
|
+
return [...keys];
|
|
3802
|
+
},
|
|
3803
|
+
getOwnPropertyDescriptor(t, prop) {
|
|
3804
|
+
if (prop in overrides) {
|
|
3805
|
+
return {
|
|
3806
|
+
configurable: true,
|
|
3807
|
+
enumerable: true,
|
|
3808
|
+
value: overrides[prop]
|
|
3809
|
+
};
|
|
3810
|
+
}
|
|
3811
|
+
return Reflect.getOwnPropertyDescriptor(t, prop);
|
|
3812
|
+
}
|
|
3813
|
+
});
|
|
3814
|
+
}
|
|
3815
|
+
function assertFactoryResult(value) {
|
|
3816
|
+
if (value == null || typeof value !== "object") {
|
|
3817
|
+
throw new Error("Extension factory must return an object of overrides");
|
|
3818
|
+
}
|
|
3819
|
+
}
|
|
3820
|
+
function mergeArrayExtensions(inner, overrides) {
|
|
3821
|
+
let innerExts = inner.arrayExtensions;
|
|
3822
|
+
let freshExts = overrides.arrayExtensions;
|
|
3823
|
+
if (!innerExts?.length)
|
|
3824
|
+
return overrides;
|
|
3825
|
+
if (!freshExts?.length)
|
|
3826
|
+
return { ...overrides, arrayExtensions: innerExts };
|
|
3827
|
+
return { ...overrides, arrayExtensions: [...innerExts, ...freshExts] };
|
|
3828
|
+
}
|
|
3829
|
+
function defineStoreExtension(factory) {
|
|
3830
|
+
return (store, opts) => {
|
|
3831
|
+
let result = factory(store, opts);
|
|
3832
|
+
if (result instanceof Promise) {
|
|
3833
|
+
return result.then((overrides) => {
|
|
3834
|
+
assertFactoryResult(overrides);
|
|
3835
|
+
return createProxy(store, mergeArrayExtensions(store, overrides));
|
|
3836
|
+
});
|
|
3837
|
+
}
|
|
3838
|
+
assertFactoryResult(result);
|
|
3839
|
+
return createProxy(store, mergeArrayExtensions(store, result));
|
|
3840
|
+
};
|
|
3841
|
+
}
|
|
3842
|
+
function applyExtensions(value, extensions) {
|
|
3843
|
+
let result = value;
|
|
3844
|
+
for (let ext of extensions) {
|
|
3845
|
+
if (result instanceof Promise) {
|
|
3846
|
+
result = result.then((v) => ext(v));
|
|
3847
|
+
} else {
|
|
3848
|
+
result = ext(result);
|
|
3849
|
+
}
|
|
3850
|
+
}
|
|
3851
|
+
return result;
|
|
3852
|
+
}
|
|
3853
|
+
function extendArray(array, ...extensions) {
|
|
3854
|
+
return applyExtensions(array, extensions);
|
|
3855
|
+
}
|
|
3856
|
+
async function maybeExtend(array) {
|
|
3857
|
+
let exts = array.store.arrayExtensions;
|
|
3858
|
+
if (!exts?.length)
|
|
3859
|
+
return array;
|
|
3860
|
+
let variadic = extendArray;
|
|
3861
|
+
return await variadic(array, ...exts);
|
|
3862
|
+
}
|
|
3863
|
+
let VERSION_COUNTER = createVersionCounter();
|
|
3864
|
+
function createVersionCounter() {
|
|
3865
|
+
let versionCounts = /* @__PURE__ */ new WeakMap();
|
|
3866
|
+
function getCounts(store) {
|
|
3867
|
+
let counts = versionCounts.get(store) ?? { v2: 0, v3: 0 };
|
|
3868
|
+
versionCounts.set(store, counts);
|
|
3203
3869
|
return counts;
|
|
3204
3870
|
}
|
|
3205
3871
|
return {
|
|
3206
3872
|
increment(store, version) {
|
|
3207
|
-
|
|
3873
|
+
getCounts(store)[version] += 1;
|
|
3208
3874
|
},
|
|
3209
|
-
|
|
3210
|
-
let counts =
|
|
3875
|
+
versionMax(store) {
|
|
3876
|
+
let counts = getCounts(store);
|
|
3211
3877
|
return counts.v3 > counts.v2 ? "v3" : "v2";
|
|
3212
3878
|
}
|
|
3213
3879
|
};
|
|
3214
3880
|
}
|
|
3215
|
-
async function
|
|
3216
|
-
let
|
|
3217
|
-
|
|
3881
|
+
async function loadAttrs(location, signal) {
|
|
3882
|
+
let metaBytes = await location.store.get(location.resolve(".zattrs").path, {
|
|
3883
|
+
signal
|
|
3884
|
+
});
|
|
3885
|
+
if (!metaBytes)
|
|
3218
3886
|
return {};
|
|
3219
|
-
return
|
|
3887
|
+
return jsonDecodeObject(metaBytes);
|
|
3220
3888
|
}
|
|
3221
|
-
async function
|
|
3889
|
+
async function openV2(location, options = {}) {
|
|
3222
3890
|
let loc = "store" in location ? location : new Location(location);
|
|
3891
|
+
let { signal } = options;
|
|
3223
3892
|
let attrs = {};
|
|
3224
3893
|
if (options.attrs ?? true)
|
|
3225
|
-
attrs = await
|
|
3894
|
+
attrs = await loadAttrs(loc, signal);
|
|
3895
|
+
signal?.throwIfAborted();
|
|
3226
3896
|
if (options.kind === "array")
|
|
3227
|
-
return
|
|
3897
|
+
return openArrayV2(loc, attrs, signal);
|
|
3228
3898
|
if (options.kind === "group")
|
|
3229
|
-
return
|
|
3230
|
-
return
|
|
3231
|
-
|
|
3232
|
-
return
|
|
3899
|
+
return openGroupV2(loc, attrs, signal);
|
|
3900
|
+
return openArrayV2(loc, attrs, signal).catch((err) => {
|
|
3901
|
+
rethrowUnless(err, NotFoundError, InvalidMetadataError);
|
|
3902
|
+
return openGroupV2(loc, attrs, signal);
|
|
3233
3903
|
});
|
|
3234
3904
|
}
|
|
3235
|
-
async function
|
|
3905
|
+
async function openArrayV2(location, attrs, signal) {
|
|
3236
3906
|
let { path } = location.resolve(".zarray");
|
|
3237
|
-
let meta = await location.store.get(path);
|
|
3907
|
+
let meta = await location.store.get(path, { signal });
|
|
3238
3908
|
if (!meta) {
|
|
3239
|
-
throw new
|
|
3240
|
-
cause: new KeyError(path)
|
|
3241
|
-
});
|
|
3909
|
+
throw new NotFoundError("v2 array", { path });
|
|
3242
3910
|
}
|
|
3243
3911
|
VERSION_COUNTER.increment(location.store, "v2");
|
|
3244
|
-
return new Array$1(location.store, location.path,
|
|
3912
|
+
return maybeExtend(new Array$1(location.store, location.path, v2ToV3ArrayMetadata(jsonDecodeObject(meta), attrs)));
|
|
3245
3913
|
}
|
|
3246
|
-
async function
|
|
3914
|
+
async function openGroupV2(location, attrs, signal) {
|
|
3247
3915
|
let { path } = location.resolve(".zgroup");
|
|
3248
|
-
let meta = await location.store.get(path);
|
|
3916
|
+
let meta = await location.store.get(path, { signal });
|
|
3249
3917
|
if (!meta) {
|
|
3250
|
-
throw new
|
|
3251
|
-
cause: new KeyError(path)
|
|
3252
|
-
});
|
|
3918
|
+
throw new NotFoundError("v2 group", { path });
|
|
3253
3919
|
}
|
|
3254
3920
|
VERSION_COUNTER.increment(location.store, "v2");
|
|
3255
|
-
return new Group(location.store, location.path,
|
|
3921
|
+
return new Group(location.store, location.path, v2ToV3GroupMetadata(jsonDecodeObject(meta), attrs));
|
|
3256
3922
|
}
|
|
3257
|
-
async function
|
|
3923
|
+
async function _openV3(location, signal) {
|
|
3258
3924
|
let { store, path } = location.resolve("zarr.json");
|
|
3259
|
-
let meta = await location.store.get(path);
|
|
3925
|
+
let meta = await location.store.get(path, { signal });
|
|
3260
3926
|
if (!meta) {
|
|
3261
|
-
throw new
|
|
3262
|
-
cause: new KeyError(path)
|
|
3263
|
-
});
|
|
3927
|
+
throw new NotFoundError("v3 array or group", { path });
|
|
3264
3928
|
}
|
|
3265
|
-
let
|
|
3266
|
-
if (
|
|
3267
|
-
|
|
3929
|
+
let metaDoc = jsonDecodeObject(meta);
|
|
3930
|
+
if (metaDoc.node_type === "array") {
|
|
3931
|
+
metaDoc.fill_value = ensureCorrectScalar(metaDoc);
|
|
3268
3932
|
}
|
|
3269
|
-
return
|
|
3933
|
+
return metaDoc.node_type === "array" ? maybeExtend(new Array$1(store, location.path, metaDoc)) : new Group(store, location.path, metaDoc);
|
|
3270
3934
|
}
|
|
3271
|
-
async function
|
|
3935
|
+
async function openV3(location, options = {}) {
|
|
3272
3936
|
let loc = "store" in location ? location : new Location(location);
|
|
3273
|
-
let node = await
|
|
3937
|
+
let node = await _openV3(loc, options.signal);
|
|
3274
3938
|
VERSION_COUNTER.increment(loc.store, "v3");
|
|
3275
3939
|
if (options.kind === void 0)
|
|
3276
3940
|
return node;
|
|
@@ -3279,20 +3943,129 @@ async function open_v3(location, options = {}) {
|
|
|
3279
3943
|
if (options.kind === "group" && node instanceof Group)
|
|
3280
3944
|
return node;
|
|
3281
3945
|
let kind = node instanceof Array$1 ? "array" : "group";
|
|
3282
|
-
throw new
|
|
3946
|
+
throw new NotFoundError(`${options.kind} at ${loc.path}`, {
|
|
3947
|
+
path: loc.path,
|
|
3948
|
+
found: kind
|
|
3949
|
+
});
|
|
3283
3950
|
}
|
|
3284
3951
|
async function open(location, options = {}) {
|
|
3285
3952
|
let store = "store" in location ? location.store : location;
|
|
3286
|
-
let
|
|
3287
|
-
let
|
|
3288
|
-
let
|
|
3289
|
-
return
|
|
3290
|
-
|
|
3291
|
-
return
|
|
3953
|
+
let versionMax = VERSION_COUNTER.versionMax(store);
|
|
3954
|
+
let openPrimary = versionMax === "v2" ? open.v2 : open.v3;
|
|
3955
|
+
let openSecondary = versionMax === "v2" ? open.v3 : open.v2;
|
|
3956
|
+
return openPrimary(location, options).catch((err) => {
|
|
3957
|
+
rethrowUnless(err, NotFoundError, InvalidMetadataError);
|
|
3958
|
+
return openSecondary(location, options);
|
|
3292
3959
|
});
|
|
3293
3960
|
}
|
|
3294
|
-
open.v2 =
|
|
3295
|
-
open.v3 =
|
|
3961
|
+
open.v2 = openV2;
|
|
3962
|
+
open.v3 = openV3;
|
|
3963
|
+
function isConsolidatedV2(meta) {
|
|
3964
|
+
return typeof meta === "object" && meta !== null && "zarr_consolidated_format" in meta && meta.zarr_consolidated_format === 1 && "metadata" in meta && typeof meta.metadata === "object" && meta.metadata !== null;
|
|
3965
|
+
}
|
|
3966
|
+
function isConsolidatedV3(meta) {
|
|
3967
|
+
return typeof meta === "object" && meta !== null && "zarr_format" in meta && meta.zarr_format === 3 && "node_type" in meta && meta.node_type === "group" && "consolidated_metadata" in meta && typeof meta.consolidated_metadata === "object" && meta.consolidated_metadata !== null && "metadata" in meta.consolidated_metadata && typeof meta.consolidated_metadata.metadata === "object" && meta.consolidated_metadata.metadata !== null;
|
|
3968
|
+
}
|
|
3969
|
+
function isMetaKey(key) {
|
|
3970
|
+
return key.endsWith(".zarray") || key.endsWith(".zgroup") || key.endsWith(".zattrs") || key.endsWith("zarr.json");
|
|
3971
|
+
}
|
|
3972
|
+
function isV3(meta) {
|
|
3973
|
+
return "zarr_format" in meta && meta.zarr_format === 3;
|
|
3974
|
+
}
|
|
3975
|
+
async function loadConsolidatedV2(store, metadataKey) {
|
|
3976
|
+
let key = metadataKey ?? ".zmetadata";
|
|
3977
|
+
let bytes = await store.get(`/${key}`);
|
|
3978
|
+
if (!bytes) {
|
|
3979
|
+
throw new NotFoundError("v2 consolidated metadata", {
|
|
3980
|
+
path: `/${key}`
|
|
3981
|
+
});
|
|
3982
|
+
}
|
|
3983
|
+
let meta = jsonDecodeObject(bytes);
|
|
3984
|
+
if (!isConsolidatedV2(meta)) {
|
|
3985
|
+
throw new InvalidMetadataError("Invalid or unsupported v2 consolidated format", { path: `/${key}` });
|
|
3986
|
+
}
|
|
3987
|
+
let knownMeta = {};
|
|
3988
|
+
for (let [k, value] of Object.entries(meta.metadata)) {
|
|
3989
|
+
knownMeta[`/${k}`] = value;
|
|
3990
|
+
}
|
|
3991
|
+
return knownMeta;
|
|
3992
|
+
}
|
|
3993
|
+
async function loadConsolidatedV3(store) {
|
|
3994
|
+
let bytes = await store.get("/zarr.json");
|
|
3995
|
+
if (!bytes) {
|
|
3996
|
+
throw new NotFoundError("v3 consolidated metadata", {
|
|
3997
|
+
path: "/zarr.json"
|
|
3998
|
+
});
|
|
3999
|
+
}
|
|
4000
|
+
let rootMeta = jsonDecodeObject(bytes);
|
|
4001
|
+
if (!isConsolidatedV3(rootMeta)) {
|
|
4002
|
+
throw new InvalidMetadataError("Root zarr.json does not contain consolidated_metadata", { path: "/zarr.json" });
|
|
4003
|
+
}
|
|
4004
|
+
let knownMeta = {};
|
|
4005
|
+
knownMeta["/zarr.json"] = {
|
|
4006
|
+
zarr_format: 3,
|
|
4007
|
+
node_type: "group",
|
|
4008
|
+
attributes: rootMeta.attributes ?? {}
|
|
4009
|
+
};
|
|
4010
|
+
for (let [path, meta] of Object.entries(rootMeta.consolidated_metadata.metadata)) {
|
|
4011
|
+
let normalized = path.startsWith("/") ? path : `/${path}`;
|
|
4012
|
+
let key = `${normalized}/zarr.json`;
|
|
4013
|
+
knownMeta[key] = meta;
|
|
4014
|
+
}
|
|
4015
|
+
return knownMeta;
|
|
4016
|
+
}
|
|
4017
|
+
function resolveFormats(store, format) {
|
|
4018
|
+
if (format !== void 0) {
|
|
4019
|
+
return globalThis.Array.isArray(format) ? format : [format];
|
|
4020
|
+
}
|
|
4021
|
+
let versionMax = VERSION_COUNTER.versionMax(store);
|
|
4022
|
+
return versionMax === "v3" ? ["v3", "v2"] : ["v2", "v3"];
|
|
4023
|
+
}
|
|
4024
|
+
const withConsolidatedMetadata = defineStoreExtension(async (store, opts = {}) => {
|
|
4025
|
+
let formats = resolveFormats(store, opts.format);
|
|
4026
|
+
let lastError;
|
|
4027
|
+
for (let format of formats) {
|
|
4028
|
+
try {
|
|
4029
|
+
let knownMeta = format === "v2" ? await loadConsolidatedV2(store, opts.metadataKey) : await loadConsolidatedV3(store);
|
|
4030
|
+
return {
|
|
4031
|
+
async get(key, options) {
|
|
4032
|
+
if (knownMeta[key]) {
|
|
4033
|
+
return jsonEncodeObject(knownMeta[key]);
|
|
4034
|
+
}
|
|
4035
|
+
let maybeBytes = await store.get(key, options);
|
|
4036
|
+
if (isMetaKey(key) && maybeBytes) {
|
|
4037
|
+
knownMeta[key] = jsonDecodeObject(maybeBytes);
|
|
4038
|
+
}
|
|
4039
|
+
return maybeBytes;
|
|
4040
|
+
},
|
|
4041
|
+
contents() {
|
|
4042
|
+
let contents = [];
|
|
4043
|
+
for (let [key, value] of Object.entries(knownMeta)) {
|
|
4044
|
+
let parts = key.split("/");
|
|
4045
|
+
let filename = parts.pop();
|
|
4046
|
+
let path = parts.join("/") || "/";
|
|
4047
|
+
if (filename === ".zarray")
|
|
4048
|
+
contents.push({ path, kind: "array" });
|
|
4049
|
+
if (filename === ".zgroup")
|
|
4050
|
+
contents.push({ path, kind: "group" });
|
|
4051
|
+
if (isV3(value)) {
|
|
4052
|
+
contents.push({ path, kind: value.node_type });
|
|
4053
|
+
}
|
|
4054
|
+
}
|
|
4055
|
+
return contents;
|
|
4056
|
+
}
|
|
4057
|
+
};
|
|
4058
|
+
} catch (err) {
|
|
4059
|
+
rethrowUnless(err, NotFoundError, InvalidMetadataError);
|
|
4060
|
+
lastError = err;
|
|
4061
|
+
}
|
|
4062
|
+
}
|
|
4063
|
+
throw lastError;
|
|
4064
|
+
});
|
|
4065
|
+
function extendStore(store, ...extensions) {
|
|
4066
|
+
return applyExtensions(store, extensions);
|
|
4067
|
+
}
|
|
4068
|
+
var _a, _b;
|
|
3296
4069
|
function readBlobAsArrayBuffer(blob) {
|
|
3297
4070
|
if (blob.arrayBuffer) {
|
|
3298
4071
|
return blob.arrayBuffer();
|
|
@@ -3316,7 +4089,7 @@ function isBlob(v) {
|
|
|
3316
4089
|
function isSharedArrayBuffer(b) {
|
|
3317
4090
|
return typeof SharedArrayBuffer !== "undefined" && b instanceof SharedArrayBuffer;
|
|
3318
4091
|
}
|
|
3319
|
-
|
|
4092
|
+
typeof process !== "undefined" && !!(process === null || process === void 0 ? void 0 : process.versions) && typeof ((_a = process === null || process === void 0 ? void 0 : process.versions) === null || _a === void 0 ? void 0 : _a.node) !== "undefined" && typeof ((_b = process === null || process === void 0 ? void 0 : process.versions) === null || _b === void 0 ? void 0 : _b.electron) === "undefined";
|
|
3320
4093
|
function isTypedArraySameAsArrayBuffer(typedArray) {
|
|
3321
4094
|
return typedArray.byteOffset === 0 && typedArray.byteLength === typedArray.buffer.byteLength;
|
|
3322
4095
|
}
|
|
@@ -3347,389 +4120,47 @@ let BlobReader$1 = class BlobReader {
|
|
|
3347
4120
|
return this.blob.slice(offset, offset + length, type);
|
|
3348
4121
|
}
|
|
3349
4122
|
};
|
|
3350
|
-
function inflate(data, buf) {
|
|
3351
|
-
var u8 = Uint8Array;
|
|
3352
|
-
if (data[0] == 3 && data[1] == 0) return buf ? buf : new u8(0);
|
|
3353
|
-
var bitsF = _bitsF, bitsE = _bitsE, decodeTiny = _decodeTiny, get17 = _get17;
|
|
3354
|
-
var noBuf = buf == null;
|
|
3355
|
-
if (noBuf) buf = new u8(data.length >>> 2 << 3);
|
|
3356
|
-
var BFINAL = 0, BTYPE = 0, HLIT = 0, HDIST = 0, HCLEN = 0, ML = 0, MD = 0;
|
|
3357
|
-
var off = 0, pos = 0;
|
|
3358
|
-
var lmap, dmap;
|
|
3359
|
-
while (BFINAL == 0) {
|
|
3360
|
-
BFINAL = bitsF(data, pos, 1);
|
|
3361
|
-
BTYPE = bitsF(data, pos + 1, 2);
|
|
3362
|
-
pos += 3;
|
|
3363
|
-
if (BTYPE == 0) {
|
|
3364
|
-
if ((pos & 7) != 0) pos += 8 - (pos & 7);
|
|
3365
|
-
var p8 = (pos >>> 3) + 4, len = data[p8 - 4] | data[p8 - 3] << 8;
|
|
3366
|
-
if (noBuf) buf = _check(buf, off + len);
|
|
3367
|
-
buf.set(new u8(data.buffer, data.byteOffset + p8, len), off);
|
|
3368
|
-
pos = p8 + len << 3;
|
|
3369
|
-
off += len;
|
|
3370
|
-
continue;
|
|
3371
|
-
}
|
|
3372
|
-
if (noBuf) buf = _check(buf, off + (1 << 17));
|
|
3373
|
-
if (BTYPE == 1) {
|
|
3374
|
-
lmap = U.flmap;
|
|
3375
|
-
dmap = U.fdmap;
|
|
3376
|
-
ML = (1 << 9) - 1;
|
|
3377
|
-
MD = (1 << 5) - 1;
|
|
3378
|
-
}
|
|
3379
|
-
if (BTYPE == 2) {
|
|
3380
|
-
HLIT = bitsE(data, pos, 5) + 257;
|
|
3381
|
-
HDIST = bitsE(data, pos + 5, 5) + 1;
|
|
3382
|
-
HCLEN = bitsE(data, pos + 10, 4) + 4;
|
|
3383
|
-
pos += 14;
|
|
3384
|
-
for (var i = 0; i < 38; i += 2) {
|
|
3385
|
-
U.itree[i] = 0;
|
|
3386
|
-
U.itree[i + 1] = 0;
|
|
3387
|
-
}
|
|
3388
|
-
var tl = 1;
|
|
3389
|
-
for (var i = 0; i < HCLEN; i++) {
|
|
3390
|
-
var l = bitsE(data, pos + i * 3, 3);
|
|
3391
|
-
U.itree[(U.ordr[i] << 1) + 1] = l;
|
|
3392
|
-
if (l > tl) tl = l;
|
|
3393
|
-
}
|
|
3394
|
-
pos += 3 * HCLEN;
|
|
3395
|
-
makeCodes(U.itree, tl);
|
|
3396
|
-
codes2map(U.itree, tl, U.imap);
|
|
3397
|
-
lmap = U.lmap;
|
|
3398
|
-
dmap = U.dmap;
|
|
3399
|
-
pos = decodeTiny(U.imap, (1 << tl) - 1, HLIT + HDIST, data, pos, U.ttree);
|
|
3400
|
-
var mx0 = _copyOut(U.ttree, 0, HLIT, U.ltree);
|
|
3401
|
-
ML = (1 << mx0) - 1;
|
|
3402
|
-
var mx1 = _copyOut(U.ttree, HLIT, HDIST, U.dtree);
|
|
3403
|
-
MD = (1 << mx1) - 1;
|
|
3404
|
-
makeCodes(U.ltree, mx0);
|
|
3405
|
-
codes2map(U.ltree, mx0, lmap);
|
|
3406
|
-
makeCodes(U.dtree, mx1);
|
|
3407
|
-
codes2map(U.dtree, mx1, dmap);
|
|
3408
|
-
}
|
|
3409
|
-
while (true) {
|
|
3410
|
-
var code = lmap[get17(data, pos) & ML];
|
|
3411
|
-
pos += code & 15;
|
|
3412
|
-
var lit = code >>> 4;
|
|
3413
|
-
if (lit >>> 8 == 0) {
|
|
3414
|
-
buf[off++] = lit;
|
|
3415
|
-
} else if (lit == 256) {
|
|
3416
|
-
break;
|
|
3417
|
-
} else {
|
|
3418
|
-
var end = off + lit - 254;
|
|
3419
|
-
if (lit > 264) {
|
|
3420
|
-
var ebs = U.ldef[lit - 257];
|
|
3421
|
-
end = off + (ebs >>> 3) + bitsE(data, pos, ebs & 7);
|
|
3422
|
-
pos += ebs & 7;
|
|
3423
|
-
}
|
|
3424
|
-
var dcode = dmap[get17(data, pos) & MD];
|
|
3425
|
-
pos += dcode & 15;
|
|
3426
|
-
var dlit = dcode >>> 4;
|
|
3427
|
-
var dbs = U.ddef[dlit], dst = (dbs >>> 4) + bitsF(data, pos, dbs & 15);
|
|
3428
|
-
pos += dbs & 15;
|
|
3429
|
-
if (noBuf) buf = _check(buf, off + (1 << 17));
|
|
3430
|
-
while (off < end) {
|
|
3431
|
-
buf[off] = buf[off++ - dst];
|
|
3432
|
-
buf[off] = buf[off++ - dst];
|
|
3433
|
-
buf[off] = buf[off++ - dst];
|
|
3434
|
-
buf[off] = buf[off++ - dst];
|
|
3435
|
-
}
|
|
3436
|
-
off = end;
|
|
3437
|
-
}
|
|
3438
|
-
}
|
|
3439
|
-
}
|
|
3440
|
-
return buf.length == off ? buf : buf.slice(0, off);
|
|
3441
|
-
}
|
|
3442
|
-
function _check(buf, len) {
|
|
3443
|
-
var bl = buf.length;
|
|
3444
|
-
if (len <= bl) return buf;
|
|
3445
|
-
var nbuf = new Uint8Array(Math.max(bl << 1, len));
|
|
3446
|
-
nbuf.set(buf, 0);
|
|
3447
|
-
return nbuf;
|
|
3448
|
-
}
|
|
3449
|
-
function _decodeTiny(lmap, LL, len, data, pos, tree) {
|
|
3450
|
-
var bitsE = _bitsE, get17 = _get17;
|
|
3451
|
-
var i = 0;
|
|
3452
|
-
while (i < len) {
|
|
3453
|
-
var code = lmap[get17(data, pos) & LL];
|
|
3454
|
-
pos += code & 15;
|
|
3455
|
-
var lit = code >>> 4;
|
|
3456
|
-
if (lit <= 15) {
|
|
3457
|
-
tree[i] = lit;
|
|
3458
|
-
i++;
|
|
3459
|
-
} else {
|
|
3460
|
-
var ll = 0, n = 0;
|
|
3461
|
-
if (lit == 16) {
|
|
3462
|
-
n = 3 + bitsE(data, pos, 2);
|
|
3463
|
-
pos += 2;
|
|
3464
|
-
ll = tree[i - 1];
|
|
3465
|
-
} else if (lit == 17) {
|
|
3466
|
-
n = 3 + bitsE(data, pos, 3);
|
|
3467
|
-
pos += 3;
|
|
3468
|
-
} else if (lit == 18) {
|
|
3469
|
-
n = 11 + bitsE(data, pos, 7);
|
|
3470
|
-
pos += 7;
|
|
3471
|
-
}
|
|
3472
|
-
var ni = i + n;
|
|
3473
|
-
while (i < ni) {
|
|
3474
|
-
tree[i] = ll;
|
|
3475
|
-
i++;
|
|
3476
|
-
}
|
|
3477
|
-
}
|
|
3478
|
-
}
|
|
3479
|
-
return pos;
|
|
3480
|
-
}
|
|
3481
|
-
function _copyOut(src, off, len, tree) {
|
|
3482
|
-
var mx = 0, i = 0, tl = tree.length >>> 1;
|
|
3483
|
-
while (i < len) {
|
|
3484
|
-
var v = src[i + off];
|
|
3485
|
-
tree[i << 1] = 0;
|
|
3486
|
-
tree[(i << 1) + 1] = v;
|
|
3487
|
-
if (v > mx) mx = v;
|
|
3488
|
-
i++;
|
|
3489
|
-
}
|
|
3490
|
-
while (i < tl) {
|
|
3491
|
-
tree[i << 1] = 0;
|
|
3492
|
-
tree[(i << 1) + 1] = 0;
|
|
3493
|
-
i++;
|
|
3494
|
-
}
|
|
3495
|
-
return mx;
|
|
3496
|
-
}
|
|
3497
|
-
function makeCodes(tree, MAX_BITS) {
|
|
3498
|
-
var max_code = tree.length;
|
|
3499
|
-
var code, bits, n, i, len;
|
|
3500
|
-
var bl_count = U.bl_count;
|
|
3501
|
-
for (var i = 0; i <= MAX_BITS; i++) bl_count[i] = 0;
|
|
3502
|
-
for (i = 1; i < max_code; i += 2) bl_count[tree[i]]++;
|
|
3503
|
-
var next_code = U.next_code;
|
|
3504
|
-
code = 0;
|
|
3505
|
-
bl_count[0] = 0;
|
|
3506
|
-
for (bits = 1; bits <= MAX_BITS; bits++) {
|
|
3507
|
-
code = code + bl_count[bits - 1] << 1;
|
|
3508
|
-
next_code[bits] = code;
|
|
3509
|
-
}
|
|
3510
|
-
for (n = 0; n < max_code; n += 2) {
|
|
3511
|
-
len = tree[n + 1];
|
|
3512
|
-
if (len != 0) {
|
|
3513
|
-
tree[n] = next_code[len];
|
|
3514
|
-
next_code[len]++;
|
|
3515
|
-
}
|
|
3516
|
-
}
|
|
3517
|
-
}
|
|
3518
|
-
function codes2map(tree, MAX_BITS, map) {
|
|
3519
|
-
var max_code = tree.length;
|
|
3520
|
-
var r15 = U.rev15;
|
|
3521
|
-
for (var i = 0; i < max_code; i += 2) if (tree[i + 1] != 0) {
|
|
3522
|
-
var lit = i >> 1;
|
|
3523
|
-
var cl = tree[i + 1], val = lit << 4 | cl;
|
|
3524
|
-
var rest = MAX_BITS - cl, i0 = tree[i] << rest, i1 = i0 + (1 << rest);
|
|
3525
|
-
while (i0 != i1) {
|
|
3526
|
-
var p0 = r15[i0] >>> 15 - MAX_BITS;
|
|
3527
|
-
map[p0] = val;
|
|
3528
|
-
i0++;
|
|
3529
|
-
}
|
|
3530
|
-
}
|
|
3531
|
-
}
|
|
3532
|
-
function revCodes(tree, MAX_BITS) {
|
|
3533
|
-
var r15 = U.rev15, imb = 15 - MAX_BITS;
|
|
3534
|
-
for (var i = 0; i < tree.length; i += 2) {
|
|
3535
|
-
var i0 = tree[i] << MAX_BITS - tree[i + 1];
|
|
3536
|
-
tree[i] = r15[i0] >>> imb;
|
|
3537
|
-
}
|
|
3538
|
-
}
|
|
3539
|
-
function _bitsE(dt, pos, length) {
|
|
3540
|
-
return (dt[pos >>> 3] | dt[(pos >>> 3) + 1] << 8) >>> (pos & 7) & (1 << length) - 1;
|
|
3541
|
-
}
|
|
3542
|
-
function _bitsF(dt, pos, length) {
|
|
3543
|
-
return (dt[pos >>> 3] | dt[(pos >>> 3) + 1] << 8 | dt[(pos >>> 3) + 2] << 16) >>> (pos & 7) & (1 << length) - 1;
|
|
3544
|
-
}
|
|
3545
|
-
function _get17(dt, pos) {
|
|
3546
|
-
return (dt[pos >>> 3] | dt[(pos >>> 3) + 1] << 8 | dt[(pos >>> 3) + 2] << 16) >>> (pos & 7);
|
|
3547
|
-
}
|
|
3548
|
-
const U = function() {
|
|
3549
|
-
var u16 = Uint16Array, u32 = Uint32Array;
|
|
3550
|
-
return {
|
|
3551
|
-
next_code: new u16(16),
|
|
3552
|
-
bl_count: new u16(16),
|
|
3553
|
-
ordr: [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15],
|
|
3554
|
-
of0: [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 999, 999, 999],
|
|
3555
|
-
exb: [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0, 0],
|
|
3556
|
-
ldef: new u16(32),
|
|
3557
|
-
df0: [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 65535, 65535],
|
|
3558
|
-
dxb: [0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 0, 0],
|
|
3559
|
-
ddef: new u32(32),
|
|
3560
|
-
flmap: new u16(512),
|
|
3561
|
-
fltree: [],
|
|
3562
|
-
fdmap: new u16(32),
|
|
3563
|
-
fdtree: [],
|
|
3564
|
-
lmap: new u16(32768),
|
|
3565
|
-
ltree: [],
|
|
3566
|
-
ttree: [],
|
|
3567
|
-
dmap: new u16(32768),
|
|
3568
|
-
dtree: [],
|
|
3569
|
-
imap: new u16(512),
|
|
3570
|
-
itree: [],
|
|
3571
|
-
//rev9 : new u16( 512)
|
|
3572
|
-
rev15: new u16(1 << 15),
|
|
3573
|
-
lhst: new u32(286),
|
|
3574
|
-
dhst: new u32(30),
|
|
3575
|
-
ihst: new u32(19),
|
|
3576
|
-
lits: new u32(15e3),
|
|
3577
|
-
strt: new u16(1 << 16),
|
|
3578
|
-
prev: new u16(1 << 15)
|
|
3579
|
-
};
|
|
3580
|
-
}();
|
|
3581
|
-
(function() {
|
|
3582
|
-
var len = 1 << 15;
|
|
3583
|
-
for (var i = 0; i < len; i++) {
|
|
3584
|
-
var x = i;
|
|
3585
|
-
x = (x & 2863311530) >>> 1 | (x & 1431655765) << 1;
|
|
3586
|
-
x = (x & 3435973836) >>> 2 | (x & 858993459) << 2;
|
|
3587
|
-
x = (x & 4042322160) >>> 4 | (x & 252645135) << 4;
|
|
3588
|
-
x = (x & 4278255360) >>> 8 | (x & 16711935) << 8;
|
|
3589
|
-
U.rev15[i] = (x >>> 16 | x << 16) >>> 17;
|
|
3590
|
-
}
|
|
3591
|
-
function pushV(tgt, n, sv) {
|
|
3592
|
-
while (n-- != 0) tgt.push(0, sv);
|
|
3593
|
-
}
|
|
3594
|
-
for (var i = 0; i < 32; i++) {
|
|
3595
|
-
U.ldef[i] = U.of0[i] << 3 | U.exb[i];
|
|
3596
|
-
U.ddef[i] = U.df0[i] << 4 | U.dxb[i];
|
|
3597
|
-
}
|
|
3598
|
-
pushV(U.fltree, 144, 8);
|
|
3599
|
-
pushV(U.fltree, 255 - 143, 9);
|
|
3600
|
-
pushV(U.fltree, 279 - 255, 7);
|
|
3601
|
-
pushV(U.fltree, 287 - 279, 8);
|
|
3602
|
-
makeCodes(U.fltree, 9);
|
|
3603
|
-
codes2map(U.fltree, 9, U.flmap);
|
|
3604
|
-
revCodes(U.fltree, 9);
|
|
3605
|
-
pushV(U.fdtree, 32, 5);
|
|
3606
|
-
makeCodes(U.fdtree, 5);
|
|
3607
|
-
codes2map(U.fdtree, 5, U.fdmap);
|
|
3608
|
-
revCodes(U.fdtree, 5);
|
|
3609
|
-
pushV(U.itree, 19, 0);
|
|
3610
|
-
pushV(U.ltree, 286, 0);
|
|
3611
|
-
pushV(U.dtree, 30, 0);
|
|
3612
|
-
pushV(U.ttree, 320, 0);
|
|
3613
|
-
})();
|
|
3614
|
-
({
|
|
3615
|
-
table: function() {
|
|
3616
|
-
var tab = new Uint32Array(256);
|
|
3617
|
-
for (var n = 0; n < 256; n++) {
|
|
3618
|
-
var c = n;
|
|
3619
|
-
for (var k = 0; k < 8; k++) {
|
|
3620
|
-
if (c & 1) c = 3988292384 ^ c >>> 1;
|
|
3621
|
-
else c = c >>> 1;
|
|
3622
|
-
}
|
|
3623
|
-
tab[n] = c;
|
|
3624
|
-
}
|
|
3625
|
-
return tab;
|
|
3626
|
-
}()
|
|
3627
|
-
});
|
|
3628
|
-
function inflateRaw(file, buf) {
|
|
3629
|
-
return inflate(file, buf);
|
|
3630
|
-
}
|
|
3631
|
-
const config = {
|
|
3632
|
-
numWorkers: 1,
|
|
3633
|
-
workerURL: "",
|
|
3634
|
-
useWorkers: false
|
|
3635
|
-
};
|
|
3636
4123
|
let nextId = 0;
|
|
3637
4124
|
const waitingForWorkerQueue = [];
|
|
3638
|
-
function
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
if (e.data === "start") {
|
|
3643
|
-
worker.onerror = void 0;
|
|
3644
|
-
worker.onmessage = void 0;
|
|
3645
|
-
resolve2(worker);
|
|
3646
|
-
} else {
|
|
3647
|
-
reject(new Error(`unexpected message: ${e.data}`));
|
|
3648
|
-
}
|
|
3649
|
-
};
|
|
3650
|
-
worker.onerror = reject;
|
|
4125
|
+
async function decompressRaw(src) {
|
|
4126
|
+
const ds = new DecompressionStream("deflate-raw");
|
|
4127
|
+
const writer = ds.writable.getWriter();
|
|
4128
|
+
writer.write(src).then(() => writer.close()).catch(() => {
|
|
3651
4129
|
});
|
|
4130
|
+
const chunks = [];
|
|
4131
|
+
const reader = ds.readable.getReader();
|
|
4132
|
+
for (; ; ) {
|
|
4133
|
+
const { done, value } = await reader.read();
|
|
4134
|
+
if (done) {
|
|
4135
|
+
break;
|
|
4136
|
+
}
|
|
4137
|
+
chunks.push(value);
|
|
4138
|
+
}
|
|
4139
|
+
const size = chunks.reduce((s, c) => s + c.byteLength, 0);
|
|
4140
|
+
const result = new Uint8Array(size);
|
|
4141
|
+
let offset = 0;
|
|
4142
|
+
for (const chunk of chunks) {
|
|
4143
|
+
result.set(chunk, offset);
|
|
4144
|
+
offset += chunk.byteLength;
|
|
4145
|
+
}
|
|
4146
|
+
return result;
|
|
3652
4147
|
}
|
|
3653
|
-
function
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
(
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
return {
|
|
3660
|
-
async createWorker(url) {
|
|
3661
|
-
return new Worker2(url);
|
|
3662
|
-
},
|
|
3663
|
-
addEventListener(worker, fn) {
|
|
3664
|
-
worker.on("message", (data) => {
|
|
3665
|
-
fn({ target: worker, data });
|
|
3666
|
-
});
|
|
3667
|
-
},
|
|
3668
|
-
async terminate(worker) {
|
|
3669
|
-
await worker.terminate();
|
|
3670
|
-
}
|
|
3671
|
-
};
|
|
3672
|
-
} else {
|
|
3673
|
-
return {
|
|
3674
|
-
async createWorker(url) {
|
|
3675
|
-
try {
|
|
3676
|
-
const worker = await startWorker(url);
|
|
3677
|
-
return worker;
|
|
3678
|
-
} catch (e) {
|
|
3679
|
-
console.warn("could not load worker:", url);
|
|
3680
|
-
}
|
|
3681
|
-
let text;
|
|
3682
|
-
try {
|
|
3683
|
-
const req = await fetch(url, { mode: "cors" });
|
|
3684
|
-
if (!req.ok) {
|
|
3685
|
-
throw new Error(`could not load: ${url}`);
|
|
3686
|
-
}
|
|
3687
|
-
text = await req.text();
|
|
3688
|
-
url = URL.createObjectURL(new Blob([text], { type: "application/javascript" }));
|
|
3689
|
-
const worker = await startWorker(url);
|
|
3690
|
-
config.workerURL = url;
|
|
3691
|
-
return worker;
|
|
3692
|
-
} catch (e) {
|
|
3693
|
-
console.warn("could not load worker via fetch:", url);
|
|
3694
|
-
}
|
|
3695
|
-
if (text !== void 0) {
|
|
3696
|
-
try {
|
|
3697
|
-
url = `data:application/javascript;base64,${btoa(text)}`;
|
|
3698
|
-
const worker = await startWorker(url);
|
|
3699
|
-
config.workerURL = url;
|
|
3700
|
-
return worker;
|
|
3701
|
-
} catch (e) {
|
|
3702
|
-
console.warn("could not load worker via dataURI");
|
|
3703
|
-
}
|
|
3704
|
-
}
|
|
3705
|
-
console.warn("workers will not be used");
|
|
3706
|
-
throw new Error("can not start workers");
|
|
3707
|
-
},
|
|
3708
|
-
addEventListener(worker, fn) {
|
|
3709
|
-
worker.addEventListener("message", fn);
|
|
3710
|
-
},
|
|
3711
|
-
async terminate(worker) {
|
|
3712
|
-
worker.terminate();
|
|
3713
|
-
}
|
|
3714
|
-
};
|
|
4148
|
+
async function inflateRawLocal(src, type, resolve2, reject) {
|
|
4149
|
+
try {
|
|
4150
|
+
const dst = await decompressRaw(src);
|
|
4151
|
+
resolve2(type ? new Blob([dst], { type }) : dst.buffer);
|
|
4152
|
+
} catch (e) {
|
|
4153
|
+
reject(e);
|
|
3715
4154
|
}
|
|
3716
|
-
})();
|
|
3717
|
-
function inflateRawLocal(src, uncompressedSize, type, resolve2) {
|
|
3718
|
-
const dst = new Uint8Array(uncompressedSize);
|
|
3719
|
-
inflateRaw(src, dst);
|
|
3720
|
-
resolve2(type ? new Blob([dst], { type }) : dst.buffer);
|
|
3721
4155
|
}
|
|
3722
4156
|
async function processWaitingForWorkerQueue() {
|
|
3723
4157
|
if (waitingForWorkerQueue.length === 0) {
|
|
3724
4158
|
return;
|
|
3725
4159
|
}
|
|
3726
4160
|
while (waitingForWorkerQueue.length) {
|
|
3727
|
-
const { src,
|
|
3728
|
-
|
|
3729
|
-
|
|
3730
|
-
data = await readBlobAsUint8Array(src);
|
|
3731
|
-
}
|
|
3732
|
-
inflateRawLocal(data, uncompressedSize, type, resolve2);
|
|
4161
|
+
const { src, type, resolve: resolve2, reject } = waitingForWorkerQueue.shift();
|
|
4162
|
+
const data = isBlob(src) ? await readBlobAsUint8Array(src) : src;
|
|
4163
|
+
inflateRawLocal(data, type, resolve2, reject);
|
|
3733
4164
|
}
|
|
3734
4165
|
}
|
|
3735
4166
|
function inflateRawAsync(src, uncompressedSize, type) {
|
|
@@ -3779,6 +4210,7 @@ class ZipEntry {
|
|
|
3779
4210
|
return decodeBuffer(new Uint8Array(buffer));
|
|
3780
4211
|
}
|
|
3781
4212
|
// returns text with JSON.parse called on it. If you want more options decode arrayBuffer yourself
|
|
4213
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
3782
4214
|
async json() {
|
|
3783
4215
|
const text = await this.text();
|
|
3784
4216
|
return JSON.parse(text);
|
|
@@ -3797,7 +4229,7 @@ async function readAsBlobOrTypedArray(reader, offset, length, type) {
|
|
|
3797
4229
|
}
|
|
3798
4230
|
return await reader.read(offset, length);
|
|
3799
4231
|
}
|
|
3800
|
-
const crc
|
|
4232
|
+
const crc = {
|
|
3801
4233
|
unsigned() {
|
|
3802
4234
|
return 0;
|
|
3803
4235
|
}
|
|
@@ -3812,7 +4244,7 @@ function getUint64LE(uint8View, offset) {
|
|
|
3812
4244
|
return getUint32LE(uint8View, offset) + getUint32LE(uint8View, offset + 4) * 4294967296;
|
|
3813
4245
|
}
|
|
3814
4246
|
const utf8Decoder = new TextDecoder();
|
|
3815
|
-
function decodeBuffer(uint8View,
|
|
4247
|
+
function decodeBuffer(uint8View, _isUTF8) {
|
|
3816
4248
|
if (isSharedArrayBuffer(uint8View.buffer)) {
|
|
3817
4249
|
uint8View = new Uint8Array(uint8View);
|
|
3818
4250
|
}
|
|
@@ -3915,6 +4347,7 @@ async function readEntries(reader, centralDirectoryOffset, centralDirectorySize,
|
|
|
3915
4347
|
}
|
|
3916
4348
|
readEntryCursor += 46;
|
|
3917
4349
|
const data = allEntriesBuffer.subarray(readEntryCursor, readEntryCursor + rawEntry.fileNameLength + rawEntry.extraFieldLength + rawEntry.fileCommentLength);
|
|
4350
|
+
(rawEntry.generalPurposeBitFlag & 2048) !== 0;
|
|
3918
4351
|
rawEntry.nameBytes = data.slice(0, rawEntry.fileNameLength);
|
|
3919
4352
|
rawEntry.name = decodeBuffer(rawEntry.nameBytes);
|
|
3920
4353
|
const fileCommentStart = rawEntry.fileNameLength + rawEntry.extraFieldLength;
|
|
@@ -3969,7 +4402,7 @@ async function readEntries(reader, centralDirectoryOffset, centralDirectorySize,
|
|
|
3969
4402
|
}
|
|
3970
4403
|
const nameField = rawEntry.extraFields.find((e2) => e2.id === 28789 && e2.data.length >= 6 && // too short to be meaningful
|
|
3971
4404
|
e2.data[0] === 1 && // Version 1 byte version of this extra field, currently 1
|
|
3972
|
-
getUint32LE(e2.data, 1), crc
|
|
4405
|
+
getUint32LE(e2.data, 1), crc.unsigned());
|
|
3973
4406
|
if (nameField) {
|
|
3974
4407
|
rawEntry.fileName = decodeBuffer(nameField.data.slice(5));
|
|
3975
4408
|
}
|
|
@@ -4033,7 +4466,7 @@ async function readEntryDataAsArrayBuffer(reader, rawEntry) {
|
|
|
4033
4466
|
return isTypedArraySameAsArrayBuffer(dataView) ? dataView.buffer : dataView.slice().buffer;
|
|
4034
4467
|
}
|
|
4035
4468
|
const typedArrayOrBlob = await readAsBlobOrTypedArray(reader, fileDataStart, rawEntry.compressedSize);
|
|
4036
|
-
const result = await inflateRawAsync(typedArrayOrBlob, rawEntry.uncompressedSize);
|
|
4469
|
+
const result = await inflateRawAsync(typedArrayOrBlob instanceof Uint8Array ? typedArrayOrBlob : typedArrayOrBlob, rawEntry.uncompressedSize);
|
|
4037
4470
|
return result;
|
|
4038
4471
|
}
|
|
4039
4472
|
async function readEntryDataAsBlob(reader, rawEntry, type) {
|
|
@@ -4043,10 +4476,10 @@ async function readEntryDataAsBlob(reader, rawEntry, type) {
|
|
|
4043
4476
|
if (isBlob(typedArrayOrBlob2)) {
|
|
4044
4477
|
return typedArrayOrBlob2;
|
|
4045
4478
|
}
|
|
4046
|
-
return new Blob([
|
|
4479
|
+
return new Blob([typedArrayOrBlob2], { type });
|
|
4047
4480
|
}
|
|
4048
4481
|
const typedArrayOrBlob = await readAsBlobOrTypedArray(reader, fileDataStart, rawEntry.compressedSize);
|
|
4049
|
-
const result = await inflateRawAsync(typedArrayOrBlob, rawEntry.uncompressedSize, type);
|
|
4482
|
+
const result = await inflateRawAsync(typedArrayOrBlob instanceof Uint8Array ? typedArrayOrBlob : typedArrayOrBlob, rawEntry.uncompressedSize, type);
|
|
4050
4483
|
return result;
|
|
4051
4484
|
}
|
|
4052
4485
|
async function unzipRaw(source) {
|
|
@@ -4082,12 +4515,17 @@ async function unzip(source) {
|
|
|
4082
4515
|
entries: Object.fromEntries(entries.map((v) => [v.name, v]))
|
|
4083
4516
|
};
|
|
4084
4517
|
}
|
|
4085
|
-
function
|
|
4086
|
-
if (!("
|
|
4087
|
-
return
|
|
4518
|
+
function getRawEntry(entry) {
|
|
4519
|
+
if (!("_rawEntry" in entry)) {
|
|
4520
|
+
return void 0;
|
|
4088
4521
|
}
|
|
4089
4522
|
const rawEntry = entry._rawEntry;
|
|
4090
|
-
|
|
4523
|
+
if (typeof rawEntry === "object" && rawEntry !== null && "relativeOffsetOfLocalHeader" in rawEntry && typeof rawEntry.relativeOffsetOfLocalHeader === "number") {
|
|
4524
|
+
return {
|
|
4525
|
+
relativeOffsetOfLocalHeader: rawEntry.relativeOffsetOfLocalHeader
|
|
4526
|
+
};
|
|
4527
|
+
}
|
|
4528
|
+
return void 0;
|
|
4091
4529
|
}
|
|
4092
4530
|
class BlobReader2 {
|
|
4093
4531
|
constructor(blob) {
|
|
@@ -4125,7 +4563,7 @@ class HTTPRangeReader {
|
|
|
4125
4563
|
if (size === 0) {
|
|
4126
4564
|
return new Uint8Array(0);
|
|
4127
4565
|
}
|
|
4128
|
-
const req = await
|
|
4566
|
+
const req = await fetchRange(this.url, offset, size, this.#overrides);
|
|
4129
4567
|
assert$1(req.ok, `failed http request ${this.url}, status: ${req.status} offset: ${offset} size: ${size}: ${req.statusText}`);
|
|
4130
4568
|
return new Uint8Array(await req.arrayBuffer());
|
|
4131
4569
|
}
|
|
@@ -4144,24 +4582,25 @@ class ZipFileStore {
|
|
|
4144
4582
|
* Compute the byte offset where entry data begins in the zip file.
|
|
4145
4583
|
* This requires reading the local file header to get filename and extra field lengths.
|
|
4146
4584
|
*/
|
|
4147
|
-
async getEntryDataOffset(
|
|
4148
|
-
const localHeaderOffset =
|
|
4585
|
+
async getEntryDataOffset(rawEntry) {
|
|
4586
|
+
const localHeaderOffset = rawEntry.relativeOffsetOfLocalHeader;
|
|
4149
4587
|
const header = await this.reader.read(localHeaderOffset, 30);
|
|
4150
4588
|
const fileNameLength = header[26] + header[27] * 256;
|
|
4151
4589
|
const extraFieldLength = header[28] + header[29] * 256;
|
|
4152
4590
|
return localHeaderOffset + 30 + fileNameLength + extraFieldLength;
|
|
4153
4591
|
}
|
|
4154
4592
|
async get(key) {
|
|
4155
|
-
let entry = (await this.info).entries[
|
|
4593
|
+
let entry = (await this.info).entries[stripPrefix(key)];
|
|
4156
4594
|
if (!entry)
|
|
4157
4595
|
return;
|
|
4158
4596
|
return new Uint8Array(await entry.arrayBuffer());
|
|
4159
4597
|
}
|
|
4160
4598
|
async getRange(key, range) {
|
|
4161
|
-
const entry = (await this.info).entries[
|
|
4599
|
+
const entry = (await this.info).entries[stripPrefix(key)];
|
|
4162
4600
|
if (!entry)
|
|
4163
4601
|
return void 0;
|
|
4164
|
-
|
|
4602
|
+
const rawEntry = getRawEntry(entry);
|
|
4603
|
+
if (!rawEntry) {
|
|
4165
4604
|
throw new Error("ZipFileStore.getRange requires internal unzipit properties that are not available. This may indicate an incompatible version of unzipit.");
|
|
4166
4605
|
}
|
|
4167
4606
|
if (entry.compressionMethod !== 0) {
|
|
@@ -4173,7 +4612,7 @@ class ZipFileStore {
|
|
|
4173
4612
|
}
|
|
4174
4613
|
return bytes.slice(range.offset, range.offset + range.length);
|
|
4175
4614
|
}
|
|
4176
|
-
const dataOffset = await this.getEntryDataOffset(
|
|
4615
|
+
const dataOffset = await this.getEntryDataOffset(rawEntry);
|
|
4177
4616
|
if ("suffixLength" in range) {
|
|
4178
4617
|
const start = dataOffset + entry.size - range.suffixLength;
|
|
4179
4618
|
return this.reader.read(start, range.suffixLength);
|
|
@@ -4181,7 +4620,7 @@ class ZipFileStore {
|
|
|
4181
4620
|
return this.reader.read(dataOffset + range.offset, range.length);
|
|
4182
4621
|
}
|
|
4183
4622
|
async has(key) {
|
|
4184
|
-
return
|
|
4623
|
+
return stripPrefix(key) in (await this.info).entries;
|
|
4185
4624
|
}
|
|
4186
4625
|
static fromUrl(href, opts = {}) {
|
|
4187
4626
|
return new ZipFileStore(new HTTPRangeReader(href, opts), opts);
|
|
@@ -4294,6 +4733,8 @@ class AnnDataAutoConfig extends AbstractAutoConfig2 {
|
|
|
4294
4733
|
}
|
|
4295
4734
|
// eslint-disable-next-line class-methods-use-this
|
|
4296
4735
|
addViews(vc, dataset, layoutOption) {
|
|
4736
|
+
const geneList = vc.addView(dataset, "featureList");
|
|
4737
|
+
vc.layout(geneList);
|
|
4297
4738
|
}
|
|
4298
4739
|
}
|
|
4299
4740
|
class SpatialDataAutoConfig extends AbstractAutoConfig2 {
|
|
@@ -4490,9 +4931,15 @@ async function parsedUrlToZmetadata(parsedUrl) {
|
|
|
4490
4931
|
let promises = [];
|
|
4491
4932
|
try {
|
|
4492
4933
|
try {
|
|
4493
|
-
store = await
|
|
4934
|
+
store = await extendStore(
|
|
4935
|
+
initialStore,
|
|
4936
|
+
(s) => withConsolidatedMetadata(s)
|
|
4937
|
+
);
|
|
4494
4938
|
} catch {
|
|
4495
|
-
store = await
|
|
4939
|
+
store = await extendStore(
|
|
4940
|
+
initialStore,
|
|
4941
|
+
(s) => withConsolidatedMetadata(s, { metadataKey: "zmetadata" })
|
|
4942
|
+
);
|
|
4496
4943
|
}
|
|
4497
4944
|
const contents = store.contents();
|
|
4498
4945
|
const consolidatedRoot = await open(store, { kind: "group" });
|
|
@@ -4595,12 +5042,12 @@ var hasRequiredFetchJsonp;
|
|
|
4595
5042
|
function requireFetchJsonp() {
|
|
4596
5043
|
if (hasRequiredFetchJsonp) return fetchJsonp$2.exports;
|
|
4597
5044
|
hasRequiredFetchJsonp = 1;
|
|
4598
|
-
(function(
|
|
5045
|
+
(function(module, exports) {
|
|
4599
5046
|
(function(global, factory) {
|
|
4600
5047
|
{
|
|
4601
|
-
factory(exports,
|
|
5048
|
+
factory(exports, module);
|
|
4602
5049
|
}
|
|
4603
|
-
})(fetchJsonp$1, function(exports2,
|
|
5050
|
+
})(fetchJsonp$1, function(exports2, module2) {
|
|
4604
5051
|
var defaultOptions = {
|
|
4605
5052
|
timeout: 5e3,
|
|
4606
5053
|
jsonpCallback: "callback"
|
|
@@ -4679,7 +5126,7 @@ function requireFetchJsonp() {
|
|
|
4679
5126
|
};
|
|
4680
5127
|
});
|
|
4681
5128
|
}
|
|
4682
|
-
|
|
5129
|
+
module2.exports = fetchJsonp2;
|
|
4683
5130
|
});
|
|
4684
5131
|
})(fetchJsonp$2, fetchJsonp$2.exports);
|
|
4685
5132
|
return fetchJsonp$2.exports;
|