icechunk-js 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -10
- package/dist/index.cjs +528 -195
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +105 -9
- package/dist/index.d.ts +105 -9
- package/dist/index.js +528 -195
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -71,12 +71,12 @@ var StorageError = class extends Error {
|
|
|
71
71
|
this.name = "StorageError";
|
|
72
72
|
}
|
|
73
73
|
};
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}
|
|
74
|
+
function hasErrorName(error) {
|
|
75
|
+
return typeof error === "object" && error !== null && "name" in error;
|
|
76
|
+
}
|
|
77
|
+
function isAbortError(error) {
|
|
78
|
+
return hasErrorName(error) && error.name === "AbortError";
|
|
79
|
+
}
|
|
80
80
|
|
|
81
81
|
// src/format/constants.ts
|
|
82
82
|
var PATHS = {
|
|
@@ -198,6 +198,7 @@ function decodeObjectId8(str) {
|
|
|
198
198
|
|
|
199
199
|
// src/format/flatbuffers/repo-parser.ts
|
|
200
200
|
var import_flatbuffers = require("flatbuffers");
|
|
201
|
+
var flexbuffers = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
|
|
201
202
|
|
|
202
203
|
// src/format/flatbuffers/generated/metadata-item.ts
|
|
203
204
|
var MetadataItem = class _MetadataItem {
|
|
@@ -684,7 +685,36 @@ function parseRepo(data) {
|
|
|
684
685
|
);
|
|
685
686
|
}
|
|
686
687
|
const snapshotsLength = repo.snapshotsLength();
|
|
687
|
-
|
|
688
|
+
const virtualChunkContainers = parseVirtualChunkContainers(repo);
|
|
689
|
+
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
690
|
+
}
|
|
691
|
+
function parseVirtualChunkContainers(repo) {
|
|
692
|
+
const result = /* @__PURE__ */ new Map();
|
|
693
|
+
const configBytes = repo.configArray();
|
|
694
|
+
if (!configBytes || configBytes.length === 0) return result;
|
|
695
|
+
let config;
|
|
696
|
+
try {
|
|
697
|
+
const ab = configBytes.buffer.slice(
|
|
698
|
+
configBytes.byteOffset,
|
|
699
|
+
configBytes.byteOffset + configBytes.byteLength
|
|
700
|
+
);
|
|
701
|
+
config = flexbuffers.toObject(ab);
|
|
702
|
+
} catch {
|
|
703
|
+
return result;
|
|
704
|
+
}
|
|
705
|
+
if (!isRecord(config)) return result;
|
|
706
|
+
const containers = config.virtual_chunk_containers;
|
|
707
|
+
if (!isRecord(containers)) return result;
|
|
708
|
+
for (const container of Object.values(containers)) {
|
|
709
|
+
if (!isRecord(container)) continue;
|
|
710
|
+
const { name, url_prefix: urlPrefix } = container;
|
|
711
|
+
if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
|
|
712
|
+
result.set(name, urlPrefix);
|
|
713
|
+
}
|
|
714
|
+
return result;
|
|
715
|
+
}
|
|
716
|
+
function isRecord(value) {
|
|
717
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
688
718
|
}
|
|
689
719
|
function compareUtf8ByteOrder(aBytes, bBytes) {
|
|
690
720
|
const minLen = Math.min(aBytes.length, bBytes.length);
|
|
@@ -827,6 +857,85 @@ var LRUCache = class {
|
|
|
827
857
|
}
|
|
828
858
|
};
|
|
829
859
|
|
|
860
|
+
// src/cache/single-flight.ts
|
|
861
|
+
function singleFlight(cache) {
|
|
862
|
+
const pending = /* @__PURE__ */ new Map();
|
|
863
|
+
return {
|
|
864
|
+
load(key, fetcher, signal) {
|
|
865
|
+
if (signal?.aborted) return Promise.reject(makeAbortError());
|
|
866
|
+
const hit = cache.get(key);
|
|
867
|
+
if (hit !== void 0) return Promise.resolve(hit);
|
|
868
|
+
let entry = pending.get(key);
|
|
869
|
+
if (!entry) {
|
|
870
|
+
const controller = new AbortController();
|
|
871
|
+
const promise = invokeFetcher(fetcher, controller.signal).then((value) => {
|
|
872
|
+
cache.set(key, value);
|
|
873
|
+
return value;
|
|
874
|
+
}).finally(() => {
|
|
875
|
+
if (pending.get(key)?.promise === promise) pending.delete(key);
|
|
876
|
+
});
|
|
877
|
+
const fresh = { promise, controller, refCount: 0 };
|
|
878
|
+
pending.set(key, fresh);
|
|
879
|
+
entry = fresh;
|
|
880
|
+
}
|
|
881
|
+
const owned = entry;
|
|
882
|
+
owned.refCount++;
|
|
883
|
+
const release = () => {
|
|
884
|
+
owned.refCount--;
|
|
885
|
+
if (owned.refCount > 0) return;
|
|
886
|
+
if (pending.get(key) === owned) {
|
|
887
|
+
pending.delete(key);
|
|
888
|
+
owned.controller.abort();
|
|
889
|
+
}
|
|
890
|
+
};
|
|
891
|
+
if (!signal) {
|
|
892
|
+
return owned.promise.then(
|
|
893
|
+
(value) => {
|
|
894
|
+
release();
|
|
895
|
+
return value;
|
|
896
|
+
},
|
|
897
|
+
(error) => {
|
|
898
|
+
release();
|
|
899
|
+
throw error;
|
|
900
|
+
}
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
return new Promise((resolve, reject) => {
|
|
904
|
+
let done = false;
|
|
905
|
+
const finish = () => {
|
|
906
|
+
if (done) return false;
|
|
907
|
+
done = true;
|
|
908
|
+
signal.removeEventListener("abort", handleAbort);
|
|
909
|
+
release();
|
|
910
|
+
return true;
|
|
911
|
+
};
|
|
912
|
+
const handleAbort = () => {
|
|
913
|
+
if (finish()) reject(makeAbortError());
|
|
914
|
+
};
|
|
915
|
+
signal.addEventListener("abort", handleAbort, { once: true });
|
|
916
|
+
owned.promise.then(
|
|
917
|
+
(value) => {
|
|
918
|
+
if (finish()) resolve(value);
|
|
919
|
+
},
|
|
920
|
+
(error) => {
|
|
921
|
+
if (finish()) reject(error);
|
|
922
|
+
}
|
|
923
|
+
);
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
function invokeFetcher(fetcher, signal) {
|
|
929
|
+
try {
|
|
930
|
+
return fetcher(signal);
|
|
931
|
+
} catch (error) {
|
|
932
|
+
return Promise.reject(error);
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
function makeAbortError() {
|
|
936
|
+
return new DOMException("The operation was aborted.", "AbortError");
|
|
937
|
+
}
|
|
938
|
+
|
|
830
939
|
// src/format/flatbuffers/types.ts
|
|
831
940
|
function asObjectId12(bytes) {
|
|
832
941
|
if (bytes.length !== 12) {
|
|
@@ -1818,11 +1927,11 @@ function getChunkPayload(ref) {
|
|
|
1818
1927
|
|
|
1819
1928
|
// src/format/flatbuffers/metadata.ts
|
|
1820
1929
|
var import_msgpack = require("@msgpack/msgpack");
|
|
1821
|
-
var
|
|
1930
|
+
var flexbuffers2 = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
|
|
1822
1931
|
function deserializeMetadata(items, specVersion) {
|
|
1823
1932
|
const result = /* @__PURE__ */ Object.create(null);
|
|
1824
1933
|
for (const item of items) {
|
|
1825
|
-
result[item.name] = specVersion === 1 /* V1_0 */ ? (0, import_msgpack.decode)(item.value) :
|
|
1934
|
+
result[item.name] = specVersion === 1 /* V1_0 */ ? (0, import_msgpack.decode)(item.value) : flexbuffers2.toObject(
|
|
1826
1935
|
item.value.buffer.slice(
|
|
1827
1936
|
item.value.byteOffset,
|
|
1828
1937
|
item.value.byteOffset + item.value.byteLength
|
|
@@ -2154,17 +2263,219 @@ function parseTransactionLog(data) {
|
|
|
2154
2263
|
};
|
|
2155
2264
|
}
|
|
2156
2265
|
|
|
2266
|
+
// src/reader/range-coalescer.ts
|
|
2267
|
+
function expectedRangeLength(range) {
|
|
2268
|
+
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2269
|
+
}
|
|
2270
|
+
function makeUrlStore(opts) {
|
|
2271
|
+
const { url, fetchClient, conditionalHeaders } = opts;
|
|
2272
|
+
async function doFetch(init) {
|
|
2273
|
+
return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
|
|
2274
|
+
}
|
|
2275
|
+
return {
|
|
2276
|
+
async get() {
|
|
2277
|
+
throw new Error(
|
|
2278
|
+
`Virtual chunk URL store for ${url} only supports ranged reads`
|
|
2279
|
+
);
|
|
2280
|
+
},
|
|
2281
|
+
async getRange(_key, range, options) {
|
|
2282
|
+
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2283
|
+
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2284
|
+
const response = await doFetch({ headers, signal: options?.signal });
|
|
2285
|
+
if (response.status === 412) {
|
|
2286
|
+
throw new Error(
|
|
2287
|
+
`Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2288
|
+
);
|
|
2289
|
+
}
|
|
2290
|
+
if (response.status !== 200 && response.status !== 206) {
|
|
2291
|
+
throw new Error(
|
|
2292
|
+
`Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
|
|
2293
|
+
);
|
|
2294
|
+
}
|
|
2295
|
+
const data = new Uint8Array(await response.arrayBuffer());
|
|
2296
|
+
if (response.status === 206) {
|
|
2297
|
+
const expected = expectedRangeLength(range);
|
|
2298
|
+
if (data.length === expected) return data;
|
|
2299
|
+
throw new Error(
|
|
2300
|
+
`Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
|
|
2301
|
+
);
|
|
2302
|
+
}
|
|
2303
|
+
if ("offset" in range) {
|
|
2304
|
+
const end = range.offset + range.length;
|
|
2305
|
+
if (data.length >= end) return data.slice(range.offset, end);
|
|
2306
|
+
throw new Error(
|
|
2307
|
+
`Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2308
|
+
);
|
|
2309
|
+
}
|
|
2310
|
+
if (data.length >= range.suffixLength) {
|
|
2311
|
+
return data.slice(data.length - range.suffixLength);
|
|
2312
|
+
}
|
|
2313
|
+
throw new Error(
|
|
2314
|
+
`Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2315
|
+
);
|
|
2316
|
+
}
|
|
2317
|
+
};
|
|
2318
|
+
}
|
|
2319
|
+
function makeStorageStore(storage) {
|
|
2320
|
+
return {
|
|
2321
|
+
async get(key, options) {
|
|
2322
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2323
|
+
return storage.getObject(key, void 0, storageOptions);
|
|
2324
|
+
},
|
|
2325
|
+
async getRange(key, range, options) {
|
|
2326
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2327
|
+
if ("suffixLength" in range) {
|
|
2328
|
+
throw new Error(
|
|
2329
|
+
`Storage suffix ranges are not supported for ${key}; convert suffixLength to offset/length before reading`
|
|
2330
|
+
);
|
|
2331
|
+
}
|
|
2332
|
+
const storageRange = {
|
|
2333
|
+
start: range.offset,
|
|
2334
|
+
end: range.offset + range.length
|
|
2335
|
+
};
|
|
2336
|
+
const data = await storage.getObject(key, storageRange, storageOptions);
|
|
2337
|
+
if (data.length === range.length) return data;
|
|
2338
|
+
if (data.length >= storageRange.end) {
|
|
2339
|
+
return data.slice(storageRange.start, storageRange.end);
|
|
2340
|
+
}
|
|
2341
|
+
throw new Error(
|
|
2342
|
+
`Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
|
|
2343
|
+
);
|
|
2344
|
+
}
|
|
2345
|
+
};
|
|
2346
|
+
}
|
|
2347
|
+
|
|
2157
2348
|
// src/reader/session.ts
|
|
2349
|
+
var RANGE_COALESCE_SIZE = 32 * 1024;
|
|
2350
|
+
var RANGE_STORE_CACHE_SIZE = 256;
|
|
2351
|
+
function makeRangeStoreCacheKey(parts) {
|
|
2352
|
+
return JSON.stringify(parts);
|
|
2353
|
+
}
|
|
2158
2354
|
var ReadSession = class _ReadSession {
|
|
2159
2355
|
storage;
|
|
2160
2356
|
snapshot;
|
|
2161
2357
|
specVersion;
|
|
2162
2358
|
manifestCache;
|
|
2163
|
-
|
|
2359
|
+
/**
|
|
2360
|
+
* Single-flight loader over `manifestCache`: concurrent misses for the
|
|
2361
|
+
* same manifest share one fetch instead of racing N identical GETs.
|
|
2362
|
+
* Critical for fan-out reads that all need the same manifest (e.g. the
|
|
2363
|
+
* many parallel chunk reads issued when a renderer crosses into a new
|
|
2364
|
+
* pyramid level).
|
|
2365
|
+
*/
|
|
2366
|
+
manifestLoader;
|
|
2367
|
+
/**
|
|
2368
|
+
* Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
|
|
2369
|
+
* The cache key includes request-option identities that must not share one
|
|
2370
|
+
* coalescing queue, notably virtual fetch clients and checksum headers.
|
|
2371
|
+
*
|
|
2372
|
+
* Promise slots are inserted synchronously on first use, so
|
|
2373
|
+
* concurrent requests for the same partition share one coalescing window
|
|
2374
|
+
* instead of racing to create parallel stores.
|
|
2375
|
+
*/
|
|
2376
|
+
rangeStores;
|
|
2377
|
+
nativeStore;
|
|
2378
|
+
fetchClientIds;
|
|
2379
|
+
nextFetchClientId = 1;
|
|
2380
|
+
rangeCoalescerIds;
|
|
2381
|
+
nextRangeCoalescerId = 1;
|
|
2382
|
+
/** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
|
|
2383
|
+
virtualChunkContainers;
|
|
2384
|
+
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
2164
2385
|
this.storage = storage;
|
|
2165
2386
|
this.snapshot = snapshot;
|
|
2166
2387
|
this.specVersion = specVersion;
|
|
2167
2388
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
2389
|
+
this.manifestLoader = singleFlight(this.manifestCache);
|
|
2390
|
+
this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
|
|
2391
|
+
}
|
|
2392
|
+
getFetchClientKey(fetchClient) {
|
|
2393
|
+
if (!fetchClient) return "default";
|
|
2394
|
+
if (!this.fetchClientIds) this.fetchClientIds = /* @__PURE__ */ new WeakMap();
|
|
2395
|
+
let id = this.fetchClientIds.get(fetchClient);
|
|
2396
|
+
if (id === void 0) {
|
|
2397
|
+
id = this.nextFetchClientId;
|
|
2398
|
+
this.nextFetchClientId = id + 1;
|
|
2399
|
+
this.fetchClientIds.set(fetchClient, id);
|
|
2400
|
+
}
|
|
2401
|
+
return String(id);
|
|
2402
|
+
}
|
|
2403
|
+
getRangeCoalescerKey(withRangeCoalescing) {
|
|
2404
|
+
if (!this.rangeCoalescerIds) this.rangeCoalescerIds = /* @__PURE__ */ new WeakMap();
|
|
2405
|
+
let id = this.rangeCoalescerIds.get(withRangeCoalescing);
|
|
2406
|
+
if (id === void 0) {
|
|
2407
|
+
id = this.nextRangeCoalescerId;
|
|
2408
|
+
this.nextRangeCoalescerId = id + 1;
|
|
2409
|
+
this.rangeCoalescerIds.set(withRangeCoalescing, id);
|
|
2410
|
+
}
|
|
2411
|
+
return String(id);
|
|
2412
|
+
}
|
|
2413
|
+
getRangeStore(partitionKey, createStore, withRangeCoalescing) {
|
|
2414
|
+
if (!this.rangeStores) {
|
|
2415
|
+
this.rangeStores = new LRUCache(RANGE_STORE_CACHE_SIZE);
|
|
2416
|
+
}
|
|
2417
|
+
const stores = this.rangeStores;
|
|
2418
|
+
const cacheKey = makeRangeStoreCacheKey([
|
|
2419
|
+
...partitionKey,
|
|
2420
|
+
["coalescer", this.getRangeCoalescerKey(withRangeCoalescing)]
|
|
2421
|
+
]);
|
|
2422
|
+
const cached = stores.get(cacheKey);
|
|
2423
|
+
if (cached) return cached;
|
|
2424
|
+
const raw = createStore();
|
|
2425
|
+
const promise = Promise.resolve().then(
|
|
2426
|
+
() => withRangeCoalescing(raw, { coalesceSize: RANGE_COALESCE_SIZE })
|
|
2427
|
+
).catch((error) => {
|
|
2428
|
+
if (stores.get(cacheKey) === promise) stores.delete(cacheKey);
|
|
2429
|
+
throw error;
|
|
2430
|
+
});
|
|
2431
|
+
stores.set(cacheKey, promise);
|
|
2432
|
+
return promise;
|
|
2433
|
+
}
|
|
2434
|
+
getNativeStore(options) {
|
|
2435
|
+
if (!this.nativeStore) {
|
|
2436
|
+
this.nativeStore = makeStorageStore(this.storage);
|
|
2437
|
+
}
|
|
2438
|
+
const raw = this.nativeStore;
|
|
2439
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
2440
|
+
if (!withRangeCoalescing) {
|
|
2441
|
+
return Promise.resolve(raw);
|
|
2442
|
+
}
|
|
2443
|
+
return this.getRangeStore(["native"], () => raw, withRangeCoalescing);
|
|
2444
|
+
}
|
|
2445
|
+
getVirtualStoreForPayload(httpUrl, payload, options) {
|
|
2446
|
+
const validate = !!options?.validateChecksums;
|
|
2447
|
+
let conditionalHeaders;
|
|
2448
|
+
if (validate) {
|
|
2449
|
+
conditionalHeaders = {};
|
|
2450
|
+
if (payload.checksumEtag) {
|
|
2451
|
+
conditionalHeaders["If-Match"] = payload.checksumEtag;
|
|
2452
|
+
}
|
|
2453
|
+
if (payload.checksumLastModified > 0) {
|
|
2454
|
+
conditionalHeaders["If-Unmodified-Since"] = new Date(
|
|
2455
|
+
payload.checksumLastModified * 1e3
|
|
2456
|
+
).toUTCString();
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2459
|
+
const checksumKey = validate ? ["checked", payload.checksumEtag ?? "", payload.checksumLastModified] : ["unchecked"];
|
|
2460
|
+
const createStore = () => makeUrlStore({
|
|
2461
|
+
url: httpUrl,
|
|
2462
|
+
fetchClient: options?.fetchClient,
|
|
2463
|
+
conditionalHeaders
|
|
2464
|
+
});
|
|
2465
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
2466
|
+
if (!withRangeCoalescing) {
|
|
2467
|
+
return Promise.resolve(createStore());
|
|
2468
|
+
}
|
|
2469
|
+
return this.getRangeStore(
|
|
2470
|
+
[
|
|
2471
|
+
"virtual",
|
|
2472
|
+
httpUrl,
|
|
2473
|
+
["fetch", this.getFetchClientKey(options?.fetchClient)],
|
|
2474
|
+
checksumKey
|
|
2475
|
+
],
|
|
2476
|
+
createStore,
|
|
2477
|
+
withRangeCoalescing
|
|
2478
|
+
);
|
|
2168
2479
|
}
|
|
2169
2480
|
/**
|
|
2170
2481
|
* Open a read session for a specific snapshot.
|
|
@@ -2184,7 +2495,8 @@ var ReadSession = class _ReadSession {
|
|
|
2184
2495
|
storage,
|
|
2185
2496
|
snapshot,
|
|
2186
2497
|
specVersion,
|
|
2187
|
-
options?.maxManifestCacheSize
|
|
2498
|
+
options?.maxManifestCacheSize,
|
|
2499
|
+
options?.virtualChunkContainers
|
|
2188
2500
|
);
|
|
2189
2501
|
}
|
|
2190
2502
|
/** Load and parse a snapshot from storage */
|
|
@@ -2202,22 +2514,40 @@ var ReadSession = class _ReadSession {
|
|
|
2202
2514
|
specVersion: header.specVersion
|
|
2203
2515
|
};
|
|
2204
2516
|
}
|
|
2205
|
-
/**
|
|
2206
|
-
|
|
2517
|
+
/**
|
|
2518
|
+
* Load and parse a manifest from storage.
|
|
2519
|
+
*
|
|
2520
|
+
* Concurrent calls for the same manifest are coalesced through
|
|
2521
|
+
* `manifestLoader` so a fan-out of chunk reads — the typical case
|
|
2522
|
+
* when many tiles cross into a new pyramid level at once — issues one
|
|
2523
|
+
* HTTP GET instead of one per caller. A caller's `signal` rejects only
|
|
2524
|
+
* that caller's await; the shared fetch is aborted only once every
|
|
2525
|
+
* active waiter has aborted.
|
|
2526
|
+
*/
|
|
2527
|
+
loadManifest(manifestId, options) {
|
|
2207
2528
|
const idStr = encodeObjectId12(manifestId);
|
|
2208
|
-
|
|
2209
|
-
|
|
2529
|
+
return this.manifestLoader.load(
|
|
2530
|
+
idStr,
|
|
2531
|
+
(signal) => this.fetchManifest(idStr, signal),
|
|
2532
|
+
options?.signal
|
|
2533
|
+
);
|
|
2534
|
+
}
|
|
2535
|
+
/**
|
|
2536
|
+
* Fetch and parse a single manifest. The `signal` here is the
|
|
2537
|
+
* single-flight loader's own signal — it fires only when every waiter
|
|
2538
|
+
* has aborted, so a hung manifest fetch doesn't trap later callers
|
|
2539
|
+
* behind it.
|
|
2540
|
+
*/
|
|
2541
|
+
async fetchManifest(idStr, signal) {
|
|
2210
2542
|
const path = getManifestPath(idStr);
|
|
2211
|
-
const data = await this.storage.getObject(path, void 0,
|
|
2543
|
+
const data = await this.storage.getObject(path, void 0, { signal });
|
|
2212
2544
|
const header = parseHeader(data);
|
|
2213
2545
|
validateFileType(header, 2 /* Manifest */);
|
|
2214
2546
|
let flatbufferData = getDataAfterHeader(data);
|
|
2215
2547
|
if (header.compression === 1 /* Zstd */) {
|
|
2216
2548
|
flatbufferData = (0, import_fzstd2.decompress)(flatbufferData);
|
|
2217
2549
|
}
|
|
2218
|
-
|
|
2219
|
-
this.manifestCache.set(idStr, manifest);
|
|
2220
|
-
return manifest;
|
|
2550
|
+
return parseManifest(flatbufferData);
|
|
2221
2551
|
}
|
|
2222
2552
|
/**
|
|
2223
2553
|
* Get the snapshot ID.
|
|
@@ -2348,26 +2678,25 @@ var ReadSession = class _ReadSession {
|
|
|
2348
2678
|
* @returns Chunk data bytes or null if not found
|
|
2349
2679
|
*/
|
|
2350
2680
|
async getChunk(path, coords, options) {
|
|
2351
|
-
|
|
2681
|
+
options?.signal?.throwIfAborted();
|
|
2682
|
+
const requestOptions = toRequestOptions(options);
|
|
2352
2683
|
const node = this.getNode(path);
|
|
2353
2684
|
if (!node || node.nodeData.type !== "array") {
|
|
2354
2685
|
return null;
|
|
2355
2686
|
}
|
|
2356
2687
|
const arrayData = node.nodeData;
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
continue;
|
|
2361
|
-
}
|
|
2362
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2363
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2364
|
-
if (!chunkRef) continue;
|
|
2365
|
-
const payload = getChunkPayload(chunkRef);
|
|
2366
|
-
return this.fetchChunkPayload(payload, options);
|
|
2688
|
+
for (const manifestRef of arrayData.manifests) {
|
|
2689
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
2690
|
+
continue;
|
|
2367
2691
|
}
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2692
|
+
const manifest = await this.loadManifest(
|
|
2693
|
+
manifestRef.objectId,
|
|
2694
|
+
requestOptions
|
|
2695
|
+
);
|
|
2696
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2697
|
+
if (!chunkRef) continue;
|
|
2698
|
+
const payload = getChunkPayload(chunkRef);
|
|
2699
|
+
return this.fetchChunkPayload(payload, options);
|
|
2371
2700
|
}
|
|
2372
2701
|
return null;
|
|
2373
2702
|
}
|
|
@@ -2385,26 +2714,25 @@ var ReadSession = class _ReadSession {
|
|
|
2385
2714
|
* @returns Chunk data bytes or null if not found
|
|
2386
2715
|
*/
|
|
2387
2716
|
async getChunkRange(path, coords, range, options) {
|
|
2388
|
-
|
|
2717
|
+
options?.signal?.throwIfAborted();
|
|
2718
|
+
const requestOptions = toRequestOptions(options);
|
|
2389
2719
|
const node = this.getNode(path);
|
|
2390
2720
|
if (!node || node.nodeData.type !== "array") {
|
|
2391
2721
|
return null;
|
|
2392
2722
|
}
|
|
2393
2723
|
const arrayData = node.nodeData;
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
continue;
|
|
2398
|
-
}
|
|
2399
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2400
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2401
|
-
if (!chunkRef) continue;
|
|
2402
|
-
const payload = getChunkPayload(chunkRef);
|
|
2403
|
-
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2724
|
+
for (const manifestRef of arrayData.manifests) {
|
|
2725
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
2726
|
+
continue;
|
|
2404
2727
|
}
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2728
|
+
const manifest = await this.loadManifest(
|
|
2729
|
+
manifestRef.objectId,
|
|
2730
|
+
requestOptions
|
|
2731
|
+
);
|
|
2732
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2733
|
+
if (!chunkRef) continue;
|
|
2734
|
+
const payload = getChunkPayload(chunkRef);
|
|
2735
|
+
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2408
2736
|
}
|
|
2409
2737
|
return null;
|
|
2410
2738
|
}
|
|
@@ -2426,77 +2754,54 @@ var ReadSession = class _ReadSession {
|
|
|
2426
2754
|
return payload.data;
|
|
2427
2755
|
case "native": {
|
|
2428
2756
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2429
|
-
const
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2757
|
+
const requestedStart = payload.offset;
|
|
2758
|
+
const requestedEnd = payload.offset + payload.length;
|
|
2759
|
+
const store = await this.getNativeStore(options);
|
|
2760
|
+
const data = await store.getRange(
|
|
2761
|
+
path,
|
|
2762
|
+
{ offset: requestedStart, length: payload.length },
|
|
2763
|
+
{ signal: options?.signal }
|
|
2764
|
+
);
|
|
2765
|
+
if (!data) {
|
|
2766
|
+
throw new Error(
|
|
2767
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
2768
|
+
);
|
|
2437
2769
|
}
|
|
2770
|
+
if (data.length === payload.length) return data;
|
|
2438
2771
|
throw new Error(
|
|
2439
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
2772
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${payload.length} bytes`
|
|
2440
2773
|
);
|
|
2441
2774
|
}
|
|
2442
2775
|
case "virtual": {
|
|
2443
|
-
const
|
|
2776
|
+
const absoluteLocation = expandVccUrl(
|
|
2444
2777
|
payload.location,
|
|
2778
|
+
this.virtualChunkContainers
|
|
2779
|
+
);
|
|
2780
|
+
const httpUrl = translateToHttpUrl(
|
|
2781
|
+
absoluteLocation,
|
|
2445
2782
|
options?.azureAccount
|
|
2446
2783
|
);
|
|
2447
|
-
const
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
}
|
|
2459
|
-
}
|
|
2460
|
-
const fetchInit = {
|
|
2461
|
-
headers,
|
|
2462
|
-
signal: options?.signal
|
|
2463
|
-
};
|
|
2464
|
-
let response;
|
|
2465
|
-
try {
|
|
2466
|
-
const client = options?.fetchClient;
|
|
2467
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2468
|
-
} catch (error) {
|
|
2469
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2470
|
-
throw new AbortError();
|
|
2471
|
-
}
|
|
2472
|
-
throw error;
|
|
2473
|
-
}
|
|
2474
|
-
if (response.status === 412) {
|
|
2784
|
+
const store = await this.getVirtualStoreForPayload(
|
|
2785
|
+
httpUrl,
|
|
2786
|
+
payload,
|
|
2787
|
+
options
|
|
2788
|
+
);
|
|
2789
|
+
const data = await store.getRange(
|
|
2790
|
+
"/",
|
|
2791
|
+
{ offset: payload.offset, length: payload.length },
|
|
2792
|
+
{ signal: options?.signal }
|
|
2793
|
+
);
|
|
2794
|
+
if (!data) {
|
|
2475
2795
|
throw new Error(
|
|
2476
|
-
`
|
|
2796
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2477
2797
|
);
|
|
2478
2798
|
}
|
|
2479
|
-
if (
|
|
2799
|
+
if (data.length !== payload.length) {
|
|
2480
2800
|
throw new Error(
|
|
2481
|
-
`
|
|
2801
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2482
2802
|
);
|
|
2483
2803
|
}
|
|
2484
|
-
|
|
2485
|
-
if (response.status === 206) {
|
|
2486
|
-
if (data.length !== payload.length) {
|
|
2487
|
-
throw new Error(
|
|
2488
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2489
|
-
);
|
|
2490
|
-
}
|
|
2491
|
-
return data;
|
|
2492
|
-
}
|
|
2493
|
-
const absoluteEnd = payload.offset + payload.length;
|
|
2494
|
-
if (data.length >= absoluteEnd) {
|
|
2495
|
-
return data.slice(payload.offset, absoluteEnd);
|
|
2496
|
-
}
|
|
2497
|
-
throw new Error(
|
|
2498
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2499
|
-
);
|
|
2804
|
+
return data;
|
|
2500
2805
|
}
|
|
2501
2806
|
}
|
|
2502
2807
|
}
|
|
@@ -2516,80 +2821,57 @@ var ReadSession = class _ReadSession {
|
|
|
2516
2821
|
return payload.data.slice(rangeStart, rangeEnd);
|
|
2517
2822
|
case "native": {
|
|
2518
2823
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2519
|
-
const
|
|
2520
|
-
start: payload.offset + rangeStart,
|
|
2521
|
-
end: payload.offset + rangeEnd
|
|
2522
|
-
};
|
|
2824
|
+
const requestedStart = payload.offset + rangeStart;
|
|
2523
2825
|
const expectedSize = rangeEnd - rangeStart;
|
|
2524
|
-
const
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2826
|
+
const requestedEnd = requestedStart + expectedSize;
|
|
2827
|
+
const store = await this.getNativeStore(options);
|
|
2828
|
+
const data = await store.getRange(
|
|
2829
|
+
path,
|
|
2830
|
+
{ offset: requestedStart, length: expectedSize },
|
|
2831
|
+
{ signal: options?.signal }
|
|
2832
|
+
);
|
|
2833
|
+
if (!data) {
|
|
2834
|
+
throw new Error(
|
|
2835
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
2836
|
+
);
|
|
2528
2837
|
}
|
|
2838
|
+
if (data.length === expectedSize) return data;
|
|
2529
2839
|
throw new Error(
|
|
2530
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
2840
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${expectedSize} bytes`
|
|
2531
2841
|
);
|
|
2532
2842
|
}
|
|
2533
2843
|
case "virtual": {
|
|
2534
2844
|
const absoluteStart = payload.offset + rangeStart;
|
|
2535
|
-
const
|
|
2536
|
-
const
|
|
2845
|
+
const expectedSize = rangeEnd - rangeStart;
|
|
2846
|
+
const absoluteLocation = expandVccUrl(
|
|
2537
2847
|
payload.location,
|
|
2848
|
+
this.virtualChunkContainers
|
|
2849
|
+
);
|
|
2850
|
+
const httpUrl = translateToHttpUrl(
|
|
2851
|
+
absoluteLocation,
|
|
2538
2852
|
options?.azureAccount
|
|
2539
2853
|
);
|
|
2540
|
-
const
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
}
|
|
2552
|
-
}
|
|
2553
|
-
const fetchInit = {
|
|
2554
|
-
headers,
|
|
2555
|
-
signal: options?.signal
|
|
2556
|
-
};
|
|
2557
|
-
let response;
|
|
2558
|
-
try {
|
|
2559
|
-
const client = options?.fetchClient;
|
|
2560
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2561
|
-
} catch (error) {
|
|
2562
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2563
|
-
throw new AbortError();
|
|
2564
|
-
}
|
|
2565
|
-
throw error;
|
|
2566
|
-
}
|
|
2567
|
-
if (response.status === 412) {
|
|
2854
|
+
const store = await this.getVirtualStoreForPayload(
|
|
2855
|
+
httpUrl,
|
|
2856
|
+
payload,
|
|
2857
|
+
options
|
|
2858
|
+
);
|
|
2859
|
+
const data = await store.getRange(
|
|
2860
|
+
"/",
|
|
2861
|
+
{ offset: absoluteStart, length: expectedSize },
|
|
2862
|
+
{ signal: options?.signal }
|
|
2863
|
+
);
|
|
2864
|
+
if (!data) {
|
|
2568
2865
|
throw new Error(
|
|
2569
|
-
`
|
|
2866
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2570
2867
|
);
|
|
2571
2868
|
}
|
|
2572
|
-
if (
|
|
2869
|
+
if (data.length !== expectedSize) {
|
|
2573
2870
|
throw new Error(
|
|
2574
|
-
`
|
|
2871
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2575
2872
|
);
|
|
2576
2873
|
}
|
|
2577
|
-
|
|
2578
|
-
const data = new Uint8Array(await response.arrayBuffer());
|
|
2579
|
-
if (response.status === 206) {
|
|
2580
|
-
if (data.length !== expectedSize) {
|
|
2581
|
-
throw new Error(
|
|
2582
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2583
|
-
);
|
|
2584
|
-
}
|
|
2585
|
-
return data;
|
|
2586
|
-
}
|
|
2587
|
-
if (data.length >= absoluteEnd) {
|
|
2588
|
-
return data.slice(absoluteStart, absoluteEnd);
|
|
2589
|
-
}
|
|
2590
|
-
throw new Error(
|
|
2591
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2592
|
-
);
|
|
2874
|
+
return data;
|
|
2593
2875
|
}
|
|
2594
2876
|
}
|
|
2595
2877
|
}
|
|
@@ -2613,6 +2895,19 @@ var ReadSession = class _ReadSession {
|
|
|
2613
2895
|
}
|
|
2614
2896
|
};
|
|
2615
2897
|
var utf8Encoder = new TextEncoder();
|
|
2898
|
+
function toRequestOptions(options) {
|
|
2899
|
+
if (!options) return void 0;
|
|
2900
|
+
return {
|
|
2901
|
+
...options.signal && { signal: options.signal },
|
|
2902
|
+
...options.fetchClient && { fetchClient: options.fetchClient },
|
|
2903
|
+
...options.validateChecksums !== void 0 && {
|
|
2904
|
+
validateChecksums: options.validateChecksums
|
|
2905
|
+
},
|
|
2906
|
+
...options.azureAccount !== void 0 && {
|
|
2907
|
+
azureAccount: options.azureAccount
|
|
2908
|
+
}
|
|
2909
|
+
};
|
|
2910
|
+
}
|
|
2616
2911
|
function compareUtf8Bytes(a, b) {
|
|
2617
2912
|
const bytesA = utf8Encoder.encode(a);
|
|
2618
2913
|
const bytesB = utf8Encoder.encode(b);
|
|
@@ -2624,6 +2919,28 @@ function compareUtf8Bytes(a, b) {
|
|
|
2624
2919
|
}
|
|
2625
2920
|
return bytesA.length - bytesB.length;
|
|
2626
2921
|
}
|
|
2922
|
+
var VCC_SCHEME = "vcc://";
|
|
2923
|
+
function expandVccUrl(location, containers) {
|
|
2924
|
+
if (!location.startsWith(VCC_SCHEME)) return location;
|
|
2925
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
2926
|
+
const slash = rest.indexOf("/");
|
|
2927
|
+
if (slash === -1) {
|
|
2928
|
+
throw new Error(
|
|
2929
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
2930
|
+
);
|
|
2931
|
+
}
|
|
2932
|
+
const name = rest.slice(0, slash);
|
|
2933
|
+
const relativePath = rest.slice(slash + 1);
|
|
2934
|
+
const urlPrefix = containers.get(name);
|
|
2935
|
+
if (urlPrefix === void 0) {
|
|
2936
|
+
if (containers.size === 0) return location;
|
|
2937
|
+
throw new Error(
|
|
2938
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
2939
|
+
);
|
|
2940
|
+
}
|
|
2941
|
+
const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
|
|
2942
|
+
return normalizedPrefix + relativePath;
|
|
2943
|
+
}
|
|
2627
2944
|
function translateToHttpUrl(url, azureAccount) {
|
|
2628
2945
|
if (url.startsWith("s3://")) {
|
|
2629
2946
|
const rest = url.slice(5);
|
|
@@ -2699,7 +3016,7 @@ var Repository = class _Repository {
|
|
|
2699
3016
|
* @param options - Optional request options (signal for cancellation)
|
|
2700
3017
|
* @returns ParsedRepo if v2 format, null if v1 format
|
|
2701
3018
|
* @throws Error if repo file exists but fails to parse
|
|
2702
|
-
* @throws AbortError if the operation was aborted
|
|
3019
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
2703
3020
|
*/
|
|
2704
3021
|
async loadRepoInfo(options) {
|
|
2705
3022
|
if (this.repoInfoAttempted) {
|
|
@@ -2715,7 +3032,7 @@ var Repository = class _Repository {
|
|
|
2715
3032
|
this.repoInfo = parseRepo(data);
|
|
2716
3033
|
return this.repoInfo;
|
|
2717
3034
|
} catch (error) {
|
|
2718
|
-
if (error
|
|
3035
|
+
if (isAbortError(error)) {
|
|
2719
3036
|
this.repoInfoAttempted = false;
|
|
2720
3037
|
throw error;
|
|
2721
3038
|
}
|
|
@@ -2780,7 +3097,7 @@ var Repository = class _Repository {
|
|
|
2780
3097
|
}
|
|
2781
3098
|
}
|
|
2782
3099
|
} catch (error) {
|
|
2783
|
-
if (error
|
|
3100
|
+
if (isAbortError(error)) {
|
|
2784
3101
|
throw error;
|
|
2785
3102
|
}
|
|
2786
3103
|
if (legacyPath) {
|
|
@@ -2813,7 +3130,7 @@ var Repository = class _Repository {
|
|
|
2813
3130
|
}
|
|
2814
3131
|
}
|
|
2815
3132
|
} catch (error) {
|
|
2816
|
-
if (error
|
|
3133
|
+
if (isAbortError(error)) {
|
|
2817
3134
|
throw error;
|
|
2818
3135
|
}
|
|
2819
3136
|
return legacyPath || null;
|
|
@@ -2854,7 +3171,7 @@ var Repository = class _Repository {
|
|
|
2854
3171
|
}
|
|
2855
3172
|
}
|
|
2856
3173
|
} catch (error) {
|
|
2857
|
-
if (error
|
|
3174
|
+
if (isAbortError(error)) {
|
|
2858
3175
|
throw error;
|
|
2859
3176
|
}
|
|
2860
3177
|
throw new Error("Cannot list branches: storage does not support listing");
|
|
@@ -2896,7 +3213,7 @@ var Repository = class _Repository {
|
|
|
2896
3213
|
}
|
|
2897
3214
|
}
|
|
2898
3215
|
} catch (error) {
|
|
2899
|
-
if (error
|
|
3216
|
+
if (isAbortError(error)) {
|
|
2900
3217
|
throw error;
|
|
2901
3218
|
}
|
|
2902
3219
|
throw new Error("Cannot list tags: storage does not support listing");
|
|
@@ -2926,7 +3243,10 @@ var Repository = class _Repository {
|
|
|
2926
3243
|
if (!snapshotId2) {
|
|
2927
3244
|
throw new Error(`Branch not found: ${name}`);
|
|
2928
3245
|
}
|
|
2929
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3246
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3247
|
+
...options,
|
|
3248
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3249
|
+
});
|
|
2930
3250
|
}
|
|
2931
3251
|
const refDirPath = getBranchRefDirPath(name);
|
|
2932
3252
|
const refPath = await this.findLatestRefFile(
|
|
@@ -2953,7 +3273,10 @@ var Repository = class _Repository {
|
|
|
2953
3273
|
if (!snapshotId2) {
|
|
2954
3274
|
throw new Error(`Tag not found: ${name}`);
|
|
2955
3275
|
}
|
|
2956
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3276
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3277
|
+
...options,
|
|
3278
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3279
|
+
});
|
|
2957
3280
|
}
|
|
2958
3281
|
const refDirPath = getTagRefDirPath(name);
|
|
2959
3282
|
const legacyPath = getTagRefPath(name);
|
|
@@ -2976,7 +3299,12 @@ var Repository = class _Repository {
|
|
|
2976
3299
|
*/
|
|
2977
3300
|
async checkoutSnapshot(snapshotId, options) {
|
|
2978
3301
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
2979
|
-
|
|
3302
|
+
const repoInfo = await this.loadRepoInfo(options);
|
|
3303
|
+
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
3304
|
+
return ReadSession.open(this.storage, id, {
|
|
3305
|
+
...options,
|
|
3306
|
+
...virtualChunkContainers && { virtualChunkContainers }
|
|
3307
|
+
});
|
|
2980
3308
|
}
|
|
2981
3309
|
/**
|
|
2982
3310
|
* Walk the snapshot history chain starting from the given session.
|
|
@@ -3055,9 +3383,7 @@ var HttpStorage = class {
|
|
|
3055
3383
|
return headers;
|
|
3056
3384
|
}
|
|
3057
3385
|
async getObject(path, range, options) {
|
|
3058
|
-
|
|
3059
|
-
throw new AbortError();
|
|
3060
|
-
}
|
|
3386
|
+
options?.signal?.throwIfAborted();
|
|
3061
3387
|
const url = this.getUrl(path);
|
|
3062
3388
|
const headers = this.getHeaders(range);
|
|
3063
3389
|
let response;
|
|
@@ -3070,9 +3396,7 @@ var HttpStorage = class {
|
|
|
3070
3396
|
signal: options?.signal
|
|
3071
3397
|
});
|
|
3072
3398
|
} catch (error) {
|
|
3073
|
-
if (error
|
|
3074
|
-
throw new AbortError();
|
|
3075
|
-
}
|
|
3399
|
+
if (isAbortError(error)) throw error;
|
|
3076
3400
|
throw new StorageError(
|
|
3077
3401
|
`Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3078
3402
|
error instanceof Error ? error : void 0
|
|
@@ -3090,9 +3414,7 @@ var HttpStorage = class {
|
|
|
3090
3414
|
return new Uint8Array(arrayBuffer);
|
|
3091
3415
|
}
|
|
3092
3416
|
async exists(path, options) {
|
|
3093
|
-
|
|
3094
|
-
throw new AbortError();
|
|
3095
|
-
}
|
|
3417
|
+
options?.signal?.throwIfAborted();
|
|
3096
3418
|
const url = this.getUrl(path);
|
|
3097
3419
|
try {
|
|
3098
3420
|
const response = await fetch(url, {
|
|
@@ -3103,9 +3425,7 @@ var HttpStorage = class {
|
|
|
3103
3425
|
});
|
|
3104
3426
|
return response.ok;
|
|
3105
3427
|
} catch (error) {
|
|
3106
|
-
if (error
|
|
3107
|
-
throw new AbortError();
|
|
3108
|
-
}
|
|
3428
|
+
if (isAbortError(error)) throw error;
|
|
3109
3429
|
return false;
|
|
3110
3430
|
}
|
|
3111
3431
|
}
|
|
@@ -3123,8 +3443,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3123
3443
|
fetchClient;
|
|
3124
3444
|
validateChecksums;
|
|
3125
3445
|
azureAccount;
|
|
3446
|
+
withRangeCoalescing;
|
|
3126
3447
|
basePath = "";
|
|
3127
|
-
constructor(session, fetchClient, validateChecksums, azureAccount) {
|
|
3448
|
+
constructor(session, fetchClient, validateChecksums, azureAccount, withRangeCoalescing) {
|
|
3128
3449
|
if (!(session instanceof ReadSession)) {
|
|
3129
3450
|
throw new Error(
|
|
3130
3451
|
"IcechunkStore constructor is private. Use IcechunkStore.open() instead."
|
|
@@ -3134,6 +3455,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3134
3455
|
this.fetchClient = fetchClient;
|
|
3135
3456
|
this.validateChecksums = validateChecksums ?? false;
|
|
3136
3457
|
this.azureAccount = azureAccount;
|
|
3458
|
+
this.withRangeCoalescing = withRangeCoalescing;
|
|
3137
3459
|
}
|
|
3138
3460
|
static async open(arg, options = {}) {
|
|
3139
3461
|
if (arg instanceof ReadSession) {
|
|
@@ -3141,7 +3463,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3141
3463
|
arg,
|
|
3142
3464
|
options.fetchClient,
|
|
3143
3465
|
options.validateChecksums,
|
|
3144
|
-
options.azureAccount
|
|
3466
|
+
options.azureAccount,
|
|
3467
|
+
options.withRangeCoalescing
|
|
3145
3468
|
);
|
|
3146
3469
|
}
|
|
3147
3470
|
const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
|
|
@@ -3169,7 +3492,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3169
3492
|
session,
|
|
3170
3493
|
options.fetchClient,
|
|
3171
3494
|
options.validateChecksums,
|
|
3172
|
-
options.azureAccount
|
|
3495
|
+
options.azureAccount,
|
|
3496
|
+
options.withRangeCoalescing
|
|
3173
3497
|
);
|
|
3174
3498
|
}
|
|
3175
3499
|
/**
|
|
@@ -3185,7 +3509,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3185
3509
|
* @returns Data bytes or undefined if not found
|
|
3186
3510
|
*/
|
|
3187
3511
|
async get(key, opts) {
|
|
3188
|
-
|
|
3512
|
+
opts?.signal?.throwIfAborted();
|
|
3189
3513
|
const parsed = parseZarrKey(key);
|
|
3190
3514
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3191
3515
|
try {
|
|
@@ -3197,11 +3521,15 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3197
3521
|
signal: opts?.signal,
|
|
3198
3522
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3199
3523
|
validateChecksums: this.validateChecksums,
|
|
3200
|
-
azureAccount: this.azureAccount
|
|
3524
|
+
azureAccount: this.azureAccount,
|
|
3525
|
+
...this.withRangeCoalescing && {
|
|
3526
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
3527
|
+
}
|
|
3201
3528
|
});
|
|
3202
3529
|
return chunk ?? void 0;
|
|
3203
|
-
} catch {
|
|
3204
|
-
return void 0;
|
|
3530
|
+
} catch (err) {
|
|
3531
|
+
if (err instanceof NotFoundError) return void 0;
|
|
3532
|
+
throw err;
|
|
3205
3533
|
}
|
|
3206
3534
|
}
|
|
3207
3535
|
/**
|
|
@@ -3217,7 +3545,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3217
3545
|
* @returns Data bytes or undefined if not found
|
|
3218
3546
|
*/
|
|
3219
3547
|
async getRange(key, range, opts) {
|
|
3220
|
-
|
|
3548
|
+
opts?.signal?.throwIfAborted();
|
|
3221
3549
|
const parsed = parseZarrKey(key);
|
|
3222
3550
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3223
3551
|
try {
|
|
@@ -3230,7 +3558,10 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3230
3558
|
signal: opts?.signal,
|
|
3231
3559
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3232
3560
|
validateChecksums: this.validateChecksums,
|
|
3233
|
-
azureAccount: this.azureAccount
|
|
3561
|
+
azureAccount: this.azureAccount,
|
|
3562
|
+
...this.withRangeCoalescing && {
|
|
3563
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
3564
|
+
}
|
|
3234
3565
|
}
|
|
3235
3566
|
);
|
|
3236
3567
|
return data2 ?? void 0;
|
|
@@ -3241,8 +3572,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3241
3572
|
return data.slice(-range.suffixLength);
|
|
3242
3573
|
}
|
|
3243
3574
|
return data.slice(range.offset, range.offset + range.length);
|
|
3244
|
-
} catch {
|
|
3245
|
-
return void 0;
|
|
3575
|
+
} catch (err) {
|
|
3576
|
+
if (err instanceof NotFoundError) return void 0;
|
|
3577
|
+
throw err;
|
|
3246
3578
|
}
|
|
3247
3579
|
}
|
|
3248
3580
|
/** Prepend basePath to a parsed path. */
|
|
@@ -3266,7 +3598,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3266
3598
|
this.session,
|
|
3267
3599
|
this.fetchClient,
|
|
3268
3600
|
this.validateChecksums,
|
|
3269
|
-
this.azureAccount
|
|
3601
|
+
this.azureAccount,
|
|
3602
|
+
this.withRangeCoalescing
|
|
3270
3603
|
);
|
|
3271
3604
|
const cleanPath = path.replace(/^\/+|\/+$/g, "");
|
|
3272
3605
|
scoped.basePath = this.basePath ? `${this.basePath}/${cleanPath}` : cleanPath;
|