icechunk-js 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -10
- package/dist/index.cjs +528 -195
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +105 -9
- package/dist/index.d.ts +105 -9
- package/dist/index.js +528 -195
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -18,12 +18,12 @@ var StorageError = class extends Error {
|
|
|
18
18
|
this.name = "StorageError";
|
|
19
19
|
}
|
|
20
20
|
};
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
}
|
|
21
|
+
function hasErrorName(error) {
|
|
22
|
+
return typeof error === "object" && error !== null && "name" in error;
|
|
23
|
+
}
|
|
24
|
+
function isAbortError(error) {
|
|
25
|
+
return hasErrorName(error) && error.name === "AbortError";
|
|
26
|
+
}
|
|
27
27
|
|
|
28
28
|
// src/format/constants.ts
|
|
29
29
|
var PATHS = {
|
|
@@ -145,6 +145,7 @@ function decodeObjectId8(str) {
|
|
|
145
145
|
|
|
146
146
|
// src/format/flatbuffers/repo-parser.ts
|
|
147
147
|
import { ByteBuffer } from "flatbuffers";
|
|
148
|
+
import * as flexbuffers from "flatbuffers/js/flexbuffers.js";
|
|
148
149
|
|
|
149
150
|
// src/format/flatbuffers/generated/metadata-item.ts
|
|
150
151
|
var MetadataItem = class _MetadataItem {
|
|
@@ -631,7 +632,36 @@ function parseRepo(data) {
|
|
|
631
632
|
);
|
|
632
633
|
}
|
|
633
634
|
const snapshotsLength = repo.snapshotsLength();
|
|
634
|
-
|
|
635
|
+
const virtualChunkContainers = parseVirtualChunkContainers(repo);
|
|
636
|
+
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
637
|
+
}
|
|
638
|
+
function parseVirtualChunkContainers(repo) {
|
|
639
|
+
const result = /* @__PURE__ */ new Map();
|
|
640
|
+
const configBytes = repo.configArray();
|
|
641
|
+
if (!configBytes || configBytes.length === 0) return result;
|
|
642
|
+
let config;
|
|
643
|
+
try {
|
|
644
|
+
const ab = configBytes.buffer.slice(
|
|
645
|
+
configBytes.byteOffset,
|
|
646
|
+
configBytes.byteOffset + configBytes.byteLength
|
|
647
|
+
);
|
|
648
|
+
config = flexbuffers.toObject(ab);
|
|
649
|
+
} catch {
|
|
650
|
+
return result;
|
|
651
|
+
}
|
|
652
|
+
if (!isRecord(config)) return result;
|
|
653
|
+
const containers = config.virtual_chunk_containers;
|
|
654
|
+
if (!isRecord(containers)) return result;
|
|
655
|
+
for (const container of Object.values(containers)) {
|
|
656
|
+
if (!isRecord(container)) continue;
|
|
657
|
+
const { name, url_prefix: urlPrefix } = container;
|
|
658
|
+
if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
|
|
659
|
+
result.set(name, urlPrefix);
|
|
660
|
+
}
|
|
661
|
+
return result;
|
|
662
|
+
}
|
|
663
|
+
function isRecord(value) {
|
|
664
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
635
665
|
}
|
|
636
666
|
function compareUtf8ByteOrder(aBytes, bBytes) {
|
|
637
667
|
const minLen = Math.min(aBytes.length, bBytes.length);
|
|
@@ -774,6 +804,85 @@ var LRUCache = class {
|
|
|
774
804
|
}
|
|
775
805
|
};
|
|
776
806
|
|
|
807
|
+
// src/cache/single-flight.ts
|
|
808
|
+
function singleFlight(cache) {
|
|
809
|
+
const pending = /* @__PURE__ */ new Map();
|
|
810
|
+
return {
|
|
811
|
+
load(key, fetcher, signal) {
|
|
812
|
+
if (signal?.aborted) return Promise.reject(makeAbortError());
|
|
813
|
+
const hit = cache.get(key);
|
|
814
|
+
if (hit !== void 0) return Promise.resolve(hit);
|
|
815
|
+
let entry = pending.get(key);
|
|
816
|
+
if (!entry) {
|
|
817
|
+
const controller = new AbortController();
|
|
818
|
+
const promise = invokeFetcher(fetcher, controller.signal).then((value) => {
|
|
819
|
+
cache.set(key, value);
|
|
820
|
+
return value;
|
|
821
|
+
}).finally(() => {
|
|
822
|
+
if (pending.get(key)?.promise === promise) pending.delete(key);
|
|
823
|
+
});
|
|
824
|
+
const fresh = { promise, controller, refCount: 0 };
|
|
825
|
+
pending.set(key, fresh);
|
|
826
|
+
entry = fresh;
|
|
827
|
+
}
|
|
828
|
+
const owned = entry;
|
|
829
|
+
owned.refCount++;
|
|
830
|
+
const release = () => {
|
|
831
|
+
owned.refCount--;
|
|
832
|
+
if (owned.refCount > 0) return;
|
|
833
|
+
if (pending.get(key) === owned) {
|
|
834
|
+
pending.delete(key);
|
|
835
|
+
owned.controller.abort();
|
|
836
|
+
}
|
|
837
|
+
};
|
|
838
|
+
if (!signal) {
|
|
839
|
+
return owned.promise.then(
|
|
840
|
+
(value) => {
|
|
841
|
+
release();
|
|
842
|
+
return value;
|
|
843
|
+
},
|
|
844
|
+
(error) => {
|
|
845
|
+
release();
|
|
846
|
+
throw error;
|
|
847
|
+
}
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
return new Promise((resolve, reject) => {
|
|
851
|
+
let done = false;
|
|
852
|
+
const finish = () => {
|
|
853
|
+
if (done) return false;
|
|
854
|
+
done = true;
|
|
855
|
+
signal.removeEventListener("abort", handleAbort);
|
|
856
|
+
release();
|
|
857
|
+
return true;
|
|
858
|
+
};
|
|
859
|
+
const handleAbort = () => {
|
|
860
|
+
if (finish()) reject(makeAbortError());
|
|
861
|
+
};
|
|
862
|
+
signal.addEventListener("abort", handleAbort, { once: true });
|
|
863
|
+
owned.promise.then(
|
|
864
|
+
(value) => {
|
|
865
|
+
if (finish()) resolve(value);
|
|
866
|
+
},
|
|
867
|
+
(error) => {
|
|
868
|
+
if (finish()) reject(error);
|
|
869
|
+
}
|
|
870
|
+
);
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
function invokeFetcher(fetcher, signal) {
|
|
876
|
+
try {
|
|
877
|
+
return fetcher(signal);
|
|
878
|
+
} catch (error) {
|
|
879
|
+
return Promise.reject(error);
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
function makeAbortError() {
|
|
883
|
+
return new DOMException("The operation was aborted.", "AbortError");
|
|
884
|
+
}
|
|
885
|
+
|
|
777
886
|
// src/format/flatbuffers/types.ts
|
|
778
887
|
function asObjectId12(bytes) {
|
|
779
888
|
if (bytes.length !== 12) {
|
|
@@ -1765,11 +1874,11 @@ function getChunkPayload(ref) {
|
|
|
1765
1874
|
|
|
1766
1875
|
// src/format/flatbuffers/metadata.ts
|
|
1767
1876
|
import { decode } from "@msgpack/msgpack";
|
|
1768
|
-
import * as
|
|
1877
|
+
import * as flexbuffers2 from "flatbuffers/js/flexbuffers.js";
|
|
1769
1878
|
function deserializeMetadata(items, specVersion) {
|
|
1770
1879
|
const result = /* @__PURE__ */ Object.create(null);
|
|
1771
1880
|
for (const item of items) {
|
|
1772
|
-
result[item.name] = specVersion === 1 /* V1_0 */ ? decode(item.value) :
|
|
1881
|
+
result[item.name] = specVersion === 1 /* V1_0 */ ? decode(item.value) : flexbuffers2.toObject(
|
|
1773
1882
|
item.value.buffer.slice(
|
|
1774
1883
|
item.value.byteOffset,
|
|
1775
1884
|
item.value.byteOffset + item.value.byteLength
|
|
@@ -2101,17 +2210,219 @@ function parseTransactionLog(data) {
|
|
|
2101
2210
|
};
|
|
2102
2211
|
}
|
|
2103
2212
|
|
|
2213
|
+
// src/reader/range-coalescer.ts
|
|
2214
|
+
function expectedRangeLength(range) {
|
|
2215
|
+
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2216
|
+
}
|
|
2217
|
+
function makeUrlStore(opts) {
|
|
2218
|
+
const { url, fetchClient, conditionalHeaders } = opts;
|
|
2219
|
+
async function doFetch(init) {
|
|
2220
|
+
return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
|
|
2221
|
+
}
|
|
2222
|
+
return {
|
|
2223
|
+
async get() {
|
|
2224
|
+
throw new Error(
|
|
2225
|
+
`Virtual chunk URL store for ${url} only supports ranged reads`
|
|
2226
|
+
);
|
|
2227
|
+
},
|
|
2228
|
+
async getRange(_key, range, options) {
|
|
2229
|
+
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2230
|
+
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2231
|
+
const response = await doFetch({ headers, signal: options?.signal });
|
|
2232
|
+
if (response.status === 412) {
|
|
2233
|
+
throw new Error(
|
|
2234
|
+
`Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2235
|
+
);
|
|
2236
|
+
}
|
|
2237
|
+
if (response.status !== 200 && response.status !== 206) {
|
|
2238
|
+
throw new Error(
|
|
2239
|
+
`Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
|
|
2240
|
+
);
|
|
2241
|
+
}
|
|
2242
|
+
const data = new Uint8Array(await response.arrayBuffer());
|
|
2243
|
+
if (response.status === 206) {
|
|
2244
|
+
const expected = expectedRangeLength(range);
|
|
2245
|
+
if (data.length === expected) return data;
|
|
2246
|
+
throw new Error(
|
|
2247
|
+
`Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
|
|
2248
|
+
);
|
|
2249
|
+
}
|
|
2250
|
+
if ("offset" in range) {
|
|
2251
|
+
const end = range.offset + range.length;
|
|
2252
|
+
if (data.length >= end) return data.slice(range.offset, end);
|
|
2253
|
+
throw new Error(
|
|
2254
|
+
`Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2255
|
+
);
|
|
2256
|
+
}
|
|
2257
|
+
if (data.length >= range.suffixLength) {
|
|
2258
|
+
return data.slice(data.length - range.suffixLength);
|
|
2259
|
+
}
|
|
2260
|
+
throw new Error(
|
|
2261
|
+
`Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2262
|
+
);
|
|
2263
|
+
}
|
|
2264
|
+
};
|
|
2265
|
+
}
|
|
2266
|
+
function makeStorageStore(storage) {
|
|
2267
|
+
return {
|
|
2268
|
+
async get(key, options) {
|
|
2269
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2270
|
+
return storage.getObject(key, void 0, storageOptions);
|
|
2271
|
+
},
|
|
2272
|
+
async getRange(key, range, options) {
|
|
2273
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2274
|
+
if ("suffixLength" in range) {
|
|
2275
|
+
throw new Error(
|
|
2276
|
+
`Storage suffix ranges are not supported for ${key}; convert suffixLength to offset/length before reading`
|
|
2277
|
+
);
|
|
2278
|
+
}
|
|
2279
|
+
const storageRange = {
|
|
2280
|
+
start: range.offset,
|
|
2281
|
+
end: range.offset + range.length
|
|
2282
|
+
};
|
|
2283
|
+
const data = await storage.getObject(key, storageRange, storageOptions);
|
|
2284
|
+
if (data.length === range.length) return data;
|
|
2285
|
+
if (data.length >= storageRange.end) {
|
|
2286
|
+
return data.slice(storageRange.start, storageRange.end);
|
|
2287
|
+
}
|
|
2288
|
+
throw new Error(
|
|
2289
|
+
`Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
|
|
2290
|
+
);
|
|
2291
|
+
}
|
|
2292
|
+
};
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2104
2295
|
// src/reader/session.ts
|
|
2296
|
+
var RANGE_COALESCE_SIZE = 32 * 1024;
|
|
2297
|
+
var RANGE_STORE_CACHE_SIZE = 256;
|
|
2298
|
+
function makeRangeStoreCacheKey(parts) {
|
|
2299
|
+
return JSON.stringify(parts);
|
|
2300
|
+
}
|
|
2105
2301
|
var ReadSession = class _ReadSession {
|
|
2106
2302
|
storage;
|
|
2107
2303
|
snapshot;
|
|
2108
2304
|
specVersion;
|
|
2109
2305
|
manifestCache;
|
|
2110
|
-
|
|
2306
|
+
/**
|
|
2307
|
+
* Single-flight loader over `manifestCache`: concurrent misses for the
|
|
2308
|
+
* same manifest share one fetch instead of racing N identical GETs.
|
|
2309
|
+
* Critical for fan-out reads that all need the same manifest (e.g. the
|
|
2310
|
+
* many parallel chunk reads issued when a renderer crosses into a new
|
|
2311
|
+
* pyramid level).
|
|
2312
|
+
*/
|
|
2313
|
+
manifestLoader;
|
|
2314
|
+
/**
|
|
2315
|
+
* Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
|
|
2316
|
+
* The cache key includes request-option identities that must not share one
|
|
2317
|
+
* coalescing queue, notably virtual fetch clients and checksum headers.
|
|
2318
|
+
*
|
|
2319
|
+
* Promise slots are inserted synchronously on first use, so
|
|
2320
|
+
* concurrent requests for the same partition share one coalescing window
|
|
2321
|
+
* instead of racing to create parallel stores.
|
|
2322
|
+
*/
|
|
2323
|
+
rangeStores;
|
|
2324
|
+
nativeStore;
|
|
2325
|
+
fetchClientIds;
|
|
2326
|
+
nextFetchClientId = 1;
|
|
2327
|
+
rangeCoalescerIds;
|
|
2328
|
+
nextRangeCoalescerId = 1;
|
|
2329
|
+
/** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
|
|
2330
|
+
virtualChunkContainers;
|
|
2331
|
+
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
2111
2332
|
this.storage = storage;
|
|
2112
2333
|
this.snapshot = snapshot;
|
|
2113
2334
|
this.specVersion = specVersion;
|
|
2114
2335
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
2336
|
+
this.manifestLoader = singleFlight(this.manifestCache);
|
|
2337
|
+
this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
|
|
2338
|
+
}
|
|
2339
|
+
getFetchClientKey(fetchClient) {
|
|
2340
|
+
if (!fetchClient) return "default";
|
|
2341
|
+
if (!this.fetchClientIds) this.fetchClientIds = /* @__PURE__ */ new WeakMap();
|
|
2342
|
+
let id = this.fetchClientIds.get(fetchClient);
|
|
2343
|
+
if (id === void 0) {
|
|
2344
|
+
id = this.nextFetchClientId;
|
|
2345
|
+
this.nextFetchClientId = id + 1;
|
|
2346
|
+
this.fetchClientIds.set(fetchClient, id);
|
|
2347
|
+
}
|
|
2348
|
+
return String(id);
|
|
2349
|
+
}
|
|
2350
|
+
getRangeCoalescerKey(withRangeCoalescing) {
|
|
2351
|
+
if (!this.rangeCoalescerIds) this.rangeCoalescerIds = /* @__PURE__ */ new WeakMap();
|
|
2352
|
+
let id = this.rangeCoalescerIds.get(withRangeCoalescing);
|
|
2353
|
+
if (id === void 0) {
|
|
2354
|
+
id = this.nextRangeCoalescerId;
|
|
2355
|
+
this.nextRangeCoalescerId = id + 1;
|
|
2356
|
+
this.rangeCoalescerIds.set(withRangeCoalescing, id);
|
|
2357
|
+
}
|
|
2358
|
+
return String(id);
|
|
2359
|
+
}
|
|
2360
|
+
getRangeStore(partitionKey, createStore, withRangeCoalescing) {
|
|
2361
|
+
if (!this.rangeStores) {
|
|
2362
|
+
this.rangeStores = new LRUCache(RANGE_STORE_CACHE_SIZE);
|
|
2363
|
+
}
|
|
2364
|
+
const stores = this.rangeStores;
|
|
2365
|
+
const cacheKey = makeRangeStoreCacheKey([
|
|
2366
|
+
...partitionKey,
|
|
2367
|
+
["coalescer", this.getRangeCoalescerKey(withRangeCoalescing)]
|
|
2368
|
+
]);
|
|
2369
|
+
const cached = stores.get(cacheKey);
|
|
2370
|
+
if (cached) return cached;
|
|
2371
|
+
const raw = createStore();
|
|
2372
|
+
const promise = Promise.resolve().then(
|
|
2373
|
+
() => withRangeCoalescing(raw, { coalesceSize: RANGE_COALESCE_SIZE })
|
|
2374
|
+
).catch((error) => {
|
|
2375
|
+
if (stores.get(cacheKey) === promise) stores.delete(cacheKey);
|
|
2376
|
+
throw error;
|
|
2377
|
+
});
|
|
2378
|
+
stores.set(cacheKey, promise);
|
|
2379
|
+
return promise;
|
|
2380
|
+
}
|
|
2381
|
+
getNativeStore(options) {
|
|
2382
|
+
if (!this.nativeStore) {
|
|
2383
|
+
this.nativeStore = makeStorageStore(this.storage);
|
|
2384
|
+
}
|
|
2385
|
+
const raw = this.nativeStore;
|
|
2386
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
2387
|
+
if (!withRangeCoalescing) {
|
|
2388
|
+
return Promise.resolve(raw);
|
|
2389
|
+
}
|
|
2390
|
+
return this.getRangeStore(["native"], () => raw, withRangeCoalescing);
|
|
2391
|
+
}
|
|
2392
|
+
getVirtualStoreForPayload(httpUrl, payload, options) {
|
|
2393
|
+
const validate = !!options?.validateChecksums;
|
|
2394
|
+
let conditionalHeaders;
|
|
2395
|
+
if (validate) {
|
|
2396
|
+
conditionalHeaders = {};
|
|
2397
|
+
if (payload.checksumEtag) {
|
|
2398
|
+
conditionalHeaders["If-Match"] = payload.checksumEtag;
|
|
2399
|
+
}
|
|
2400
|
+
if (payload.checksumLastModified > 0) {
|
|
2401
|
+
conditionalHeaders["If-Unmodified-Since"] = new Date(
|
|
2402
|
+
payload.checksumLastModified * 1e3
|
|
2403
|
+
).toUTCString();
|
|
2404
|
+
}
|
|
2405
|
+
}
|
|
2406
|
+
const checksumKey = validate ? ["checked", payload.checksumEtag ?? "", payload.checksumLastModified] : ["unchecked"];
|
|
2407
|
+
const createStore = () => makeUrlStore({
|
|
2408
|
+
url: httpUrl,
|
|
2409
|
+
fetchClient: options?.fetchClient,
|
|
2410
|
+
conditionalHeaders
|
|
2411
|
+
});
|
|
2412
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
2413
|
+
if (!withRangeCoalescing) {
|
|
2414
|
+
return Promise.resolve(createStore());
|
|
2415
|
+
}
|
|
2416
|
+
return this.getRangeStore(
|
|
2417
|
+
[
|
|
2418
|
+
"virtual",
|
|
2419
|
+
httpUrl,
|
|
2420
|
+
["fetch", this.getFetchClientKey(options?.fetchClient)],
|
|
2421
|
+
checksumKey
|
|
2422
|
+
],
|
|
2423
|
+
createStore,
|
|
2424
|
+
withRangeCoalescing
|
|
2425
|
+
);
|
|
2115
2426
|
}
|
|
2116
2427
|
/**
|
|
2117
2428
|
* Open a read session for a specific snapshot.
|
|
@@ -2131,7 +2442,8 @@ var ReadSession = class _ReadSession {
|
|
|
2131
2442
|
storage,
|
|
2132
2443
|
snapshot,
|
|
2133
2444
|
specVersion,
|
|
2134
|
-
options?.maxManifestCacheSize
|
|
2445
|
+
options?.maxManifestCacheSize,
|
|
2446
|
+
options?.virtualChunkContainers
|
|
2135
2447
|
);
|
|
2136
2448
|
}
|
|
2137
2449
|
/** Load and parse a snapshot from storage */
|
|
@@ -2149,22 +2461,40 @@ var ReadSession = class _ReadSession {
|
|
|
2149
2461
|
specVersion: header.specVersion
|
|
2150
2462
|
};
|
|
2151
2463
|
}
|
|
2152
|
-
/**
|
|
2153
|
-
|
|
2464
|
+
/**
|
|
2465
|
+
* Load and parse a manifest from storage.
|
|
2466
|
+
*
|
|
2467
|
+
* Concurrent calls for the same manifest are coalesced through
|
|
2468
|
+
* `manifestLoader` so a fan-out of chunk reads — the typical case
|
|
2469
|
+
* when many tiles cross into a new pyramid level at once — issues one
|
|
2470
|
+
* HTTP GET instead of one per caller. A caller's `signal` rejects only
|
|
2471
|
+
* that caller's await; the shared fetch is aborted only once every
|
|
2472
|
+
* active waiter has aborted.
|
|
2473
|
+
*/
|
|
2474
|
+
loadManifest(manifestId, options) {
|
|
2154
2475
|
const idStr = encodeObjectId12(manifestId);
|
|
2155
|
-
|
|
2156
|
-
|
|
2476
|
+
return this.manifestLoader.load(
|
|
2477
|
+
idStr,
|
|
2478
|
+
(signal) => this.fetchManifest(idStr, signal),
|
|
2479
|
+
options?.signal
|
|
2480
|
+
);
|
|
2481
|
+
}
|
|
2482
|
+
/**
|
|
2483
|
+
* Fetch and parse a single manifest. The `signal` here is the
|
|
2484
|
+
* single-flight loader's own signal — it fires only when every waiter
|
|
2485
|
+
* has aborted, so a hung manifest fetch doesn't trap later callers
|
|
2486
|
+
* behind it.
|
|
2487
|
+
*/
|
|
2488
|
+
async fetchManifest(idStr, signal) {
|
|
2157
2489
|
const path = getManifestPath(idStr);
|
|
2158
|
-
const data = await this.storage.getObject(path, void 0,
|
|
2490
|
+
const data = await this.storage.getObject(path, void 0, { signal });
|
|
2159
2491
|
const header = parseHeader(data);
|
|
2160
2492
|
validateFileType(header, 2 /* Manifest */);
|
|
2161
2493
|
let flatbufferData = getDataAfterHeader(data);
|
|
2162
2494
|
if (header.compression === 1 /* Zstd */) {
|
|
2163
2495
|
flatbufferData = decompress2(flatbufferData);
|
|
2164
2496
|
}
|
|
2165
|
-
|
|
2166
|
-
this.manifestCache.set(idStr, manifest);
|
|
2167
|
-
return manifest;
|
|
2497
|
+
return parseManifest(flatbufferData);
|
|
2168
2498
|
}
|
|
2169
2499
|
/**
|
|
2170
2500
|
* Get the snapshot ID.
|
|
@@ -2295,26 +2625,25 @@ var ReadSession = class _ReadSession {
|
|
|
2295
2625
|
* @returns Chunk data bytes or null if not found
|
|
2296
2626
|
*/
|
|
2297
2627
|
async getChunk(path, coords, options) {
|
|
2298
|
-
|
|
2628
|
+
options?.signal?.throwIfAborted();
|
|
2629
|
+
const requestOptions = toRequestOptions(options);
|
|
2299
2630
|
const node = this.getNode(path);
|
|
2300
2631
|
if (!node || node.nodeData.type !== "array") {
|
|
2301
2632
|
return null;
|
|
2302
2633
|
}
|
|
2303
2634
|
const arrayData = node.nodeData;
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
continue;
|
|
2308
|
-
}
|
|
2309
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2310
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2311
|
-
if (!chunkRef) continue;
|
|
2312
|
-
const payload = getChunkPayload(chunkRef);
|
|
2313
|
-
return this.fetchChunkPayload(payload, options);
|
|
2635
|
+
for (const manifestRef of arrayData.manifests) {
|
|
2636
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
2637
|
+
continue;
|
|
2314
2638
|
}
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2639
|
+
const manifest = await this.loadManifest(
|
|
2640
|
+
manifestRef.objectId,
|
|
2641
|
+
requestOptions
|
|
2642
|
+
);
|
|
2643
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2644
|
+
if (!chunkRef) continue;
|
|
2645
|
+
const payload = getChunkPayload(chunkRef);
|
|
2646
|
+
return this.fetchChunkPayload(payload, options);
|
|
2318
2647
|
}
|
|
2319
2648
|
return null;
|
|
2320
2649
|
}
|
|
@@ -2332,26 +2661,25 @@ var ReadSession = class _ReadSession {
|
|
|
2332
2661
|
* @returns Chunk data bytes or null if not found
|
|
2333
2662
|
*/
|
|
2334
2663
|
async getChunkRange(path, coords, range, options) {
|
|
2335
|
-
|
|
2664
|
+
options?.signal?.throwIfAborted();
|
|
2665
|
+
const requestOptions = toRequestOptions(options);
|
|
2336
2666
|
const node = this.getNode(path);
|
|
2337
2667
|
if (!node || node.nodeData.type !== "array") {
|
|
2338
2668
|
return null;
|
|
2339
2669
|
}
|
|
2340
2670
|
const arrayData = node.nodeData;
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
continue;
|
|
2345
|
-
}
|
|
2346
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2347
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2348
|
-
if (!chunkRef) continue;
|
|
2349
|
-
const payload = getChunkPayload(chunkRef);
|
|
2350
|
-
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2671
|
+
for (const manifestRef of arrayData.manifests) {
|
|
2672
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
2673
|
+
continue;
|
|
2351
2674
|
}
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2675
|
+
const manifest = await this.loadManifest(
|
|
2676
|
+
manifestRef.objectId,
|
|
2677
|
+
requestOptions
|
|
2678
|
+
);
|
|
2679
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2680
|
+
if (!chunkRef) continue;
|
|
2681
|
+
const payload = getChunkPayload(chunkRef);
|
|
2682
|
+
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2355
2683
|
}
|
|
2356
2684
|
return null;
|
|
2357
2685
|
}
|
|
@@ -2373,77 +2701,54 @@ var ReadSession = class _ReadSession {
|
|
|
2373
2701
|
return payload.data;
|
|
2374
2702
|
case "native": {
|
|
2375
2703
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2376
|
-
const
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2704
|
+
const requestedStart = payload.offset;
|
|
2705
|
+
const requestedEnd = payload.offset + payload.length;
|
|
2706
|
+
const store = await this.getNativeStore(options);
|
|
2707
|
+
const data = await store.getRange(
|
|
2708
|
+
path,
|
|
2709
|
+
{ offset: requestedStart, length: payload.length },
|
|
2710
|
+
{ signal: options?.signal }
|
|
2711
|
+
);
|
|
2712
|
+
if (!data) {
|
|
2713
|
+
throw new Error(
|
|
2714
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
2715
|
+
);
|
|
2384
2716
|
}
|
|
2717
|
+
if (data.length === payload.length) return data;
|
|
2385
2718
|
throw new Error(
|
|
2386
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
2719
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${payload.length} bytes`
|
|
2387
2720
|
);
|
|
2388
2721
|
}
|
|
2389
2722
|
case "virtual": {
|
|
2390
|
-
const
|
|
2723
|
+
const absoluteLocation = expandVccUrl(
|
|
2391
2724
|
payload.location,
|
|
2725
|
+
this.virtualChunkContainers
|
|
2726
|
+
);
|
|
2727
|
+
const httpUrl = translateToHttpUrl(
|
|
2728
|
+
absoluteLocation,
|
|
2392
2729
|
options?.azureAccount
|
|
2393
2730
|
);
|
|
2394
|
-
const
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
}
|
|
2406
|
-
}
|
|
2407
|
-
const fetchInit = {
|
|
2408
|
-
headers,
|
|
2409
|
-
signal: options?.signal
|
|
2410
|
-
};
|
|
2411
|
-
let response;
|
|
2412
|
-
try {
|
|
2413
|
-
const client = options?.fetchClient;
|
|
2414
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2415
|
-
} catch (error) {
|
|
2416
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2417
|
-
throw new AbortError();
|
|
2418
|
-
}
|
|
2419
|
-
throw error;
|
|
2420
|
-
}
|
|
2421
|
-
if (response.status === 412) {
|
|
2731
|
+
const store = await this.getVirtualStoreForPayload(
|
|
2732
|
+
httpUrl,
|
|
2733
|
+
payload,
|
|
2734
|
+
options
|
|
2735
|
+
);
|
|
2736
|
+
const data = await store.getRange(
|
|
2737
|
+
"/",
|
|
2738
|
+
{ offset: payload.offset, length: payload.length },
|
|
2739
|
+
{ signal: options?.signal }
|
|
2740
|
+
);
|
|
2741
|
+
if (!data) {
|
|
2422
2742
|
throw new Error(
|
|
2423
|
-
`
|
|
2743
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2424
2744
|
);
|
|
2425
2745
|
}
|
|
2426
|
-
if (
|
|
2746
|
+
if (data.length !== payload.length) {
|
|
2427
2747
|
throw new Error(
|
|
2428
|
-
`
|
|
2748
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2429
2749
|
);
|
|
2430
2750
|
}
|
|
2431
|
-
|
|
2432
|
-
if (response.status === 206) {
|
|
2433
|
-
if (data.length !== payload.length) {
|
|
2434
|
-
throw new Error(
|
|
2435
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2436
|
-
);
|
|
2437
|
-
}
|
|
2438
|
-
return data;
|
|
2439
|
-
}
|
|
2440
|
-
const absoluteEnd = payload.offset + payload.length;
|
|
2441
|
-
if (data.length >= absoluteEnd) {
|
|
2442
|
-
return data.slice(payload.offset, absoluteEnd);
|
|
2443
|
-
}
|
|
2444
|
-
throw new Error(
|
|
2445
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2446
|
-
);
|
|
2751
|
+
return data;
|
|
2447
2752
|
}
|
|
2448
2753
|
}
|
|
2449
2754
|
}
|
|
@@ -2463,80 +2768,57 @@ var ReadSession = class _ReadSession {
|
|
|
2463
2768
|
return payload.data.slice(rangeStart, rangeEnd);
|
|
2464
2769
|
case "native": {
|
|
2465
2770
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2466
|
-
const
|
|
2467
|
-
start: payload.offset + rangeStart,
|
|
2468
|
-
end: payload.offset + rangeEnd
|
|
2469
|
-
};
|
|
2771
|
+
const requestedStart = payload.offset + rangeStart;
|
|
2470
2772
|
const expectedSize = rangeEnd - rangeStart;
|
|
2471
|
-
const
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2773
|
+
const requestedEnd = requestedStart + expectedSize;
|
|
2774
|
+
const store = await this.getNativeStore(options);
|
|
2775
|
+
const data = await store.getRange(
|
|
2776
|
+
path,
|
|
2777
|
+
{ offset: requestedStart, length: expectedSize },
|
|
2778
|
+
{ signal: options?.signal }
|
|
2779
|
+
);
|
|
2780
|
+
if (!data) {
|
|
2781
|
+
throw new Error(
|
|
2782
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
2783
|
+
);
|
|
2475
2784
|
}
|
|
2785
|
+
if (data.length === expectedSize) return data;
|
|
2476
2786
|
throw new Error(
|
|
2477
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
2787
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${expectedSize} bytes`
|
|
2478
2788
|
);
|
|
2479
2789
|
}
|
|
2480
2790
|
case "virtual": {
|
|
2481
2791
|
const absoluteStart = payload.offset + rangeStart;
|
|
2482
|
-
const
|
|
2483
|
-
const
|
|
2792
|
+
const expectedSize = rangeEnd - rangeStart;
|
|
2793
|
+
const absoluteLocation = expandVccUrl(
|
|
2484
2794
|
payload.location,
|
|
2795
|
+
this.virtualChunkContainers
|
|
2796
|
+
);
|
|
2797
|
+
const httpUrl = translateToHttpUrl(
|
|
2798
|
+
absoluteLocation,
|
|
2485
2799
|
options?.azureAccount
|
|
2486
2800
|
);
|
|
2487
|
-
const
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
}
|
|
2499
|
-
}
|
|
2500
|
-
const fetchInit = {
|
|
2501
|
-
headers,
|
|
2502
|
-
signal: options?.signal
|
|
2503
|
-
};
|
|
2504
|
-
let response;
|
|
2505
|
-
try {
|
|
2506
|
-
const client = options?.fetchClient;
|
|
2507
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2508
|
-
} catch (error) {
|
|
2509
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2510
|
-
throw new AbortError();
|
|
2511
|
-
}
|
|
2512
|
-
throw error;
|
|
2513
|
-
}
|
|
2514
|
-
if (response.status === 412) {
|
|
2801
|
+
const store = await this.getVirtualStoreForPayload(
|
|
2802
|
+
httpUrl,
|
|
2803
|
+
payload,
|
|
2804
|
+
options
|
|
2805
|
+
);
|
|
2806
|
+
const data = await store.getRange(
|
|
2807
|
+
"/",
|
|
2808
|
+
{ offset: absoluteStart, length: expectedSize },
|
|
2809
|
+
{ signal: options?.signal }
|
|
2810
|
+
);
|
|
2811
|
+
if (!data) {
|
|
2515
2812
|
throw new Error(
|
|
2516
|
-
`
|
|
2813
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2517
2814
|
);
|
|
2518
2815
|
}
|
|
2519
|
-
if (
|
|
2816
|
+
if (data.length !== expectedSize) {
|
|
2520
2817
|
throw new Error(
|
|
2521
|
-
`
|
|
2818
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2522
2819
|
);
|
|
2523
2820
|
}
|
|
2524
|
-
|
|
2525
|
-
const data = new Uint8Array(await response.arrayBuffer());
|
|
2526
|
-
if (response.status === 206) {
|
|
2527
|
-
if (data.length !== expectedSize) {
|
|
2528
|
-
throw new Error(
|
|
2529
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2530
|
-
);
|
|
2531
|
-
}
|
|
2532
|
-
return data;
|
|
2533
|
-
}
|
|
2534
|
-
if (data.length >= absoluteEnd) {
|
|
2535
|
-
return data.slice(absoluteStart, absoluteEnd);
|
|
2536
|
-
}
|
|
2537
|
-
throw new Error(
|
|
2538
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2539
|
-
);
|
|
2821
|
+
return data;
|
|
2540
2822
|
}
|
|
2541
2823
|
}
|
|
2542
2824
|
}
|
|
@@ -2560,6 +2842,19 @@ var ReadSession = class _ReadSession {
|
|
|
2560
2842
|
}
|
|
2561
2843
|
};
|
|
2562
2844
|
var utf8Encoder = new TextEncoder();
|
|
2845
|
+
function toRequestOptions(options) {
|
|
2846
|
+
if (!options) return void 0;
|
|
2847
|
+
return {
|
|
2848
|
+
...options.signal && { signal: options.signal },
|
|
2849
|
+
...options.fetchClient && { fetchClient: options.fetchClient },
|
|
2850
|
+
...options.validateChecksums !== void 0 && {
|
|
2851
|
+
validateChecksums: options.validateChecksums
|
|
2852
|
+
},
|
|
2853
|
+
...options.azureAccount !== void 0 && {
|
|
2854
|
+
azureAccount: options.azureAccount
|
|
2855
|
+
}
|
|
2856
|
+
};
|
|
2857
|
+
}
|
|
2563
2858
|
function compareUtf8Bytes(a, b) {
|
|
2564
2859
|
const bytesA = utf8Encoder.encode(a);
|
|
2565
2860
|
const bytesB = utf8Encoder.encode(b);
|
|
@@ -2571,6 +2866,28 @@ function compareUtf8Bytes(a, b) {
|
|
|
2571
2866
|
}
|
|
2572
2867
|
return bytesA.length - bytesB.length;
|
|
2573
2868
|
}
|
|
2869
|
+
var VCC_SCHEME = "vcc://";
|
|
2870
|
+
function expandVccUrl(location, containers) {
|
|
2871
|
+
if (!location.startsWith(VCC_SCHEME)) return location;
|
|
2872
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
2873
|
+
const slash = rest.indexOf("/");
|
|
2874
|
+
if (slash === -1) {
|
|
2875
|
+
throw new Error(
|
|
2876
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
2877
|
+
);
|
|
2878
|
+
}
|
|
2879
|
+
const name = rest.slice(0, slash);
|
|
2880
|
+
const relativePath = rest.slice(slash + 1);
|
|
2881
|
+
const urlPrefix = containers.get(name);
|
|
2882
|
+
if (urlPrefix === void 0) {
|
|
2883
|
+
if (containers.size === 0) return location;
|
|
2884
|
+
throw new Error(
|
|
2885
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
2886
|
+
);
|
|
2887
|
+
}
|
|
2888
|
+
const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
|
|
2889
|
+
return normalizedPrefix + relativePath;
|
|
2890
|
+
}
|
|
2574
2891
|
function translateToHttpUrl(url, azureAccount) {
|
|
2575
2892
|
if (url.startsWith("s3://")) {
|
|
2576
2893
|
const rest = url.slice(5);
|
|
@@ -2646,7 +2963,7 @@ var Repository = class _Repository {
|
|
|
2646
2963
|
* @param options - Optional request options (signal for cancellation)
|
|
2647
2964
|
* @returns ParsedRepo if v2 format, null if v1 format
|
|
2648
2965
|
* @throws Error if repo file exists but fails to parse
|
|
2649
|
-
* @throws AbortError if the operation was aborted
|
|
2966
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
2650
2967
|
*/
|
|
2651
2968
|
async loadRepoInfo(options) {
|
|
2652
2969
|
if (this.repoInfoAttempted) {
|
|
@@ -2662,7 +2979,7 @@ var Repository = class _Repository {
|
|
|
2662
2979
|
this.repoInfo = parseRepo(data);
|
|
2663
2980
|
return this.repoInfo;
|
|
2664
2981
|
} catch (error) {
|
|
2665
|
-
if (error
|
|
2982
|
+
if (isAbortError(error)) {
|
|
2666
2983
|
this.repoInfoAttempted = false;
|
|
2667
2984
|
throw error;
|
|
2668
2985
|
}
|
|
@@ -2727,7 +3044,7 @@ var Repository = class _Repository {
|
|
|
2727
3044
|
}
|
|
2728
3045
|
}
|
|
2729
3046
|
} catch (error) {
|
|
2730
|
-
if (error
|
|
3047
|
+
if (isAbortError(error)) {
|
|
2731
3048
|
throw error;
|
|
2732
3049
|
}
|
|
2733
3050
|
if (legacyPath) {
|
|
@@ -2760,7 +3077,7 @@ var Repository = class _Repository {
|
|
|
2760
3077
|
}
|
|
2761
3078
|
}
|
|
2762
3079
|
} catch (error) {
|
|
2763
|
-
if (error
|
|
3080
|
+
if (isAbortError(error)) {
|
|
2764
3081
|
throw error;
|
|
2765
3082
|
}
|
|
2766
3083
|
return legacyPath || null;
|
|
@@ -2801,7 +3118,7 @@ var Repository = class _Repository {
|
|
|
2801
3118
|
}
|
|
2802
3119
|
}
|
|
2803
3120
|
} catch (error) {
|
|
2804
|
-
if (error
|
|
3121
|
+
if (isAbortError(error)) {
|
|
2805
3122
|
throw error;
|
|
2806
3123
|
}
|
|
2807
3124
|
throw new Error("Cannot list branches: storage does not support listing");
|
|
@@ -2843,7 +3160,7 @@ var Repository = class _Repository {
|
|
|
2843
3160
|
}
|
|
2844
3161
|
}
|
|
2845
3162
|
} catch (error) {
|
|
2846
|
-
if (error
|
|
3163
|
+
if (isAbortError(error)) {
|
|
2847
3164
|
throw error;
|
|
2848
3165
|
}
|
|
2849
3166
|
throw new Error("Cannot list tags: storage does not support listing");
|
|
@@ -2873,7 +3190,10 @@ var Repository = class _Repository {
|
|
|
2873
3190
|
if (!snapshotId2) {
|
|
2874
3191
|
throw new Error(`Branch not found: ${name}`);
|
|
2875
3192
|
}
|
|
2876
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3193
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3194
|
+
...options,
|
|
3195
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3196
|
+
});
|
|
2877
3197
|
}
|
|
2878
3198
|
const refDirPath = getBranchRefDirPath(name);
|
|
2879
3199
|
const refPath = await this.findLatestRefFile(
|
|
@@ -2900,7 +3220,10 @@ var Repository = class _Repository {
|
|
|
2900
3220
|
if (!snapshotId2) {
|
|
2901
3221
|
throw new Error(`Tag not found: ${name}`);
|
|
2902
3222
|
}
|
|
2903
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3223
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3224
|
+
...options,
|
|
3225
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3226
|
+
});
|
|
2904
3227
|
}
|
|
2905
3228
|
const refDirPath = getTagRefDirPath(name);
|
|
2906
3229
|
const legacyPath = getTagRefPath(name);
|
|
@@ -2923,7 +3246,12 @@ var Repository = class _Repository {
|
|
|
2923
3246
|
*/
|
|
2924
3247
|
async checkoutSnapshot(snapshotId, options) {
|
|
2925
3248
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
2926
|
-
|
|
3249
|
+
const repoInfo = await this.loadRepoInfo(options);
|
|
3250
|
+
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
3251
|
+
return ReadSession.open(this.storage, id, {
|
|
3252
|
+
...options,
|
|
3253
|
+
...virtualChunkContainers && { virtualChunkContainers }
|
|
3254
|
+
});
|
|
2927
3255
|
}
|
|
2928
3256
|
/**
|
|
2929
3257
|
* Walk the snapshot history chain starting from the given session.
|
|
@@ -3002,9 +3330,7 @@ var HttpStorage = class {
|
|
|
3002
3330
|
return headers;
|
|
3003
3331
|
}
|
|
3004
3332
|
async getObject(path, range, options) {
|
|
3005
|
-
|
|
3006
|
-
throw new AbortError();
|
|
3007
|
-
}
|
|
3333
|
+
options?.signal?.throwIfAborted();
|
|
3008
3334
|
const url = this.getUrl(path);
|
|
3009
3335
|
const headers = this.getHeaders(range);
|
|
3010
3336
|
let response;
|
|
@@ -3017,9 +3343,7 @@ var HttpStorage = class {
|
|
|
3017
3343
|
signal: options?.signal
|
|
3018
3344
|
});
|
|
3019
3345
|
} catch (error) {
|
|
3020
|
-
if (error
|
|
3021
|
-
throw new AbortError();
|
|
3022
|
-
}
|
|
3346
|
+
if (isAbortError(error)) throw error;
|
|
3023
3347
|
throw new StorageError(
|
|
3024
3348
|
`Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3025
3349
|
error instanceof Error ? error : void 0
|
|
@@ -3037,9 +3361,7 @@ var HttpStorage = class {
|
|
|
3037
3361
|
return new Uint8Array(arrayBuffer);
|
|
3038
3362
|
}
|
|
3039
3363
|
async exists(path, options) {
|
|
3040
|
-
|
|
3041
|
-
throw new AbortError();
|
|
3042
|
-
}
|
|
3364
|
+
options?.signal?.throwIfAborted();
|
|
3043
3365
|
const url = this.getUrl(path);
|
|
3044
3366
|
try {
|
|
3045
3367
|
const response = await fetch(url, {
|
|
@@ -3050,9 +3372,7 @@ var HttpStorage = class {
|
|
|
3050
3372
|
});
|
|
3051
3373
|
return response.ok;
|
|
3052
3374
|
} catch (error) {
|
|
3053
|
-
if (error
|
|
3054
|
-
throw new AbortError();
|
|
3055
|
-
}
|
|
3375
|
+
if (isAbortError(error)) throw error;
|
|
3056
3376
|
return false;
|
|
3057
3377
|
}
|
|
3058
3378
|
}
|
|
@@ -3070,8 +3390,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3070
3390
|
fetchClient;
|
|
3071
3391
|
validateChecksums;
|
|
3072
3392
|
azureAccount;
|
|
3393
|
+
withRangeCoalescing;
|
|
3073
3394
|
basePath = "";
|
|
3074
|
-
constructor(session, fetchClient, validateChecksums, azureAccount) {
|
|
3395
|
+
constructor(session, fetchClient, validateChecksums, azureAccount, withRangeCoalescing) {
|
|
3075
3396
|
if (!(session instanceof ReadSession)) {
|
|
3076
3397
|
throw new Error(
|
|
3077
3398
|
"IcechunkStore constructor is private. Use IcechunkStore.open() instead."
|
|
@@ -3081,6 +3402,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3081
3402
|
this.fetchClient = fetchClient;
|
|
3082
3403
|
this.validateChecksums = validateChecksums ?? false;
|
|
3083
3404
|
this.azureAccount = azureAccount;
|
|
3405
|
+
this.withRangeCoalescing = withRangeCoalescing;
|
|
3084
3406
|
}
|
|
3085
3407
|
static async open(arg, options = {}) {
|
|
3086
3408
|
if (arg instanceof ReadSession) {
|
|
@@ -3088,7 +3410,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3088
3410
|
arg,
|
|
3089
3411
|
options.fetchClient,
|
|
3090
3412
|
options.validateChecksums,
|
|
3091
|
-
options.azureAccount
|
|
3413
|
+
options.azureAccount,
|
|
3414
|
+
options.withRangeCoalescing
|
|
3092
3415
|
);
|
|
3093
3416
|
}
|
|
3094
3417
|
const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
|
|
@@ -3116,7 +3439,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3116
3439
|
session,
|
|
3117
3440
|
options.fetchClient,
|
|
3118
3441
|
options.validateChecksums,
|
|
3119
|
-
options.azureAccount
|
|
3442
|
+
options.azureAccount,
|
|
3443
|
+
options.withRangeCoalescing
|
|
3120
3444
|
);
|
|
3121
3445
|
}
|
|
3122
3446
|
/**
|
|
@@ -3132,7 +3456,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3132
3456
|
* @returns Data bytes or undefined if not found
|
|
3133
3457
|
*/
|
|
3134
3458
|
async get(key, opts) {
|
|
3135
|
-
|
|
3459
|
+
opts?.signal?.throwIfAborted();
|
|
3136
3460
|
const parsed = parseZarrKey(key);
|
|
3137
3461
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3138
3462
|
try {
|
|
@@ -3144,11 +3468,15 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3144
3468
|
signal: opts?.signal,
|
|
3145
3469
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3146
3470
|
validateChecksums: this.validateChecksums,
|
|
3147
|
-
azureAccount: this.azureAccount
|
|
3471
|
+
azureAccount: this.azureAccount,
|
|
3472
|
+
...this.withRangeCoalescing && {
|
|
3473
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
3474
|
+
}
|
|
3148
3475
|
});
|
|
3149
3476
|
return chunk ?? void 0;
|
|
3150
|
-
} catch {
|
|
3151
|
-
return void 0;
|
|
3477
|
+
} catch (err) {
|
|
3478
|
+
if (err instanceof NotFoundError) return void 0;
|
|
3479
|
+
throw err;
|
|
3152
3480
|
}
|
|
3153
3481
|
}
|
|
3154
3482
|
/**
|
|
@@ -3164,7 +3492,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3164
3492
|
* @returns Data bytes or undefined if not found
|
|
3165
3493
|
*/
|
|
3166
3494
|
async getRange(key, range, opts) {
|
|
3167
|
-
|
|
3495
|
+
opts?.signal?.throwIfAborted();
|
|
3168
3496
|
const parsed = parseZarrKey(key);
|
|
3169
3497
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3170
3498
|
try {
|
|
@@ -3177,7 +3505,10 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3177
3505
|
signal: opts?.signal,
|
|
3178
3506
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3179
3507
|
validateChecksums: this.validateChecksums,
|
|
3180
|
-
azureAccount: this.azureAccount
|
|
3508
|
+
azureAccount: this.azureAccount,
|
|
3509
|
+
...this.withRangeCoalescing && {
|
|
3510
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
3511
|
+
}
|
|
3181
3512
|
}
|
|
3182
3513
|
);
|
|
3183
3514
|
return data2 ?? void 0;
|
|
@@ -3188,8 +3519,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3188
3519
|
return data.slice(-range.suffixLength);
|
|
3189
3520
|
}
|
|
3190
3521
|
return data.slice(range.offset, range.offset + range.length);
|
|
3191
|
-
} catch {
|
|
3192
|
-
return void 0;
|
|
3522
|
+
} catch (err) {
|
|
3523
|
+
if (err instanceof NotFoundError) return void 0;
|
|
3524
|
+
throw err;
|
|
3193
3525
|
}
|
|
3194
3526
|
}
|
|
3195
3527
|
/** Prepend basePath to a parsed path. */
|
|
@@ -3213,7 +3545,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3213
3545
|
this.session,
|
|
3214
3546
|
this.fetchClient,
|
|
3215
3547
|
this.validateChecksums,
|
|
3216
|
-
this.azureAccount
|
|
3548
|
+
this.azureAccount,
|
|
3549
|
+
this.withRangeCoalescing
|
|
3217
3550
|
);
|
|
3218
3551
|
const cleanPath = path.replace(/^\/+|\/+$/g, "");
|
|
3219
3552
|
scoped.basePath = this.basePath ? `${this.basePath}/${cleanPath}` : cleanPath;
|