icechunk-js 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -14
- package/dist/index.cjs +543 -231
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +110 -12
- package/dist/index.d.ts +110 -12
- package/dist/index.js +543 -231
- package/dist/index.js.map +1 -1
- package/package.json +10 -10
package/dist/index.cjs
CHANGED
|
@@ -71,12 +71,12 @@ var StorageError = class extends Error {
|
|
|
71
71
|
this.name = "StorageError";
|
|
72
72
|
}
|
|
73
73
|
};
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}
|
|
74
|
+
function hasErrorName(error) {
|
|
75
|
+
return typeof error === "object" && error !== null && "name" in error;
|
|
76
|
+
}
|
|
77
|
+
function isAbortError(error) {
|
|
78
|
+
return hasErrorName(error) && error.name === "AbortError";
|
|
79
|
+
}
|
|
80
80
|
|
|
81
81
|
// src/format/constants.ts
|
|
82
82
|
var PATHS = {
|
|
@@ -198,6 +198,7 @@ function decodeObjectId8(str) {
|
|
|
198
198
|
|
|
199
199
|
// src/format/flatbuffers/repo-parser.ts
|
|
200
200
|
var import_flatbuffers = require("flatbuffers");
|
|
201
|
+
var flexbuffers = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
|
|
201
202
|
|
|
202
203
|
// src/format/flatbuffers/generated/metadata-item.ts
|
|
203
204
|
var MetadataItem = class _MetadataItem {
|
|
@@ -684,7 +685,36 @@ function parseRepo(data) {
|
|
|
684
685
|
);
|
|
685
686
|
}
|
|
686
687
|
const snapshotsLength = repo.snapshotsLength();
|
|
687
|
-
|
|
688
|
+
const virtualChunkContainers = parseVirtualChunkContainers(repo);
|
|
689
|
+
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
690
|
+
}
|
|
691
|
+
function parseVirtualChunkContainers(repo) {
|
|
692
|
+
const result = /* @__PURE__ */ new Map();
|
|
693
|
+
const configBytes = repo.configArray();
|
|
694
|
+
if (!configBytes || configBytes.length === 0) return result;
|
|
695
|
+
let config;
|
|
696
|
+
try {
|
|
697
|
+
const ab = configBytes.buffer.slice(
|
|
698
|
+
configBytes.byteOffset,
|
|
699
|
+
configBytes.byteOffset + configBytes.byteLength
|
|
700
|
+
);
|
|
701
|
+
config = flexbuffers.toObject(ab);
|
|
702
|
+
} catch {
|
|
703
|
+
return result;
|
|
704
|
+
}
|
|
705
|
+
if (!isRecord(config)) return result;
|
|
706
|
+
const containers = config.virtual_chunk_containers;
|
|
707
|
+
if (!isRecord(containers)) return result;
|
|
708
|
+
for (const container of Object.values(containers)) {
|
|
709
|
+
if (!isRecord(container)) continue;
|
|
710
|
+
const { name, url_prefix: urlPrefix } = container;
|
|
711
|
+
if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
|
|
712
|
+
result.set(name, urlPrefix);
|
|
713
|
+
}
|
|
714
|
+
return result;
|
|
715
|
+
}
|
|
716
|
+
function isRecord(value) {
|
|
717
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
688
718
|
}
|
|
689
719
|
function compareUtf8ByteOrder(aBytes, bBytes) {
|
|
690
720
|
const minLen = Math.min(aBytes.length, bBytes.length);
|
|
@@ -827,6 +857,85 @@ var LRUCache = class {
|
|
|
827
857
|
}
|
|
828
858
|
};
|
|
829
859
|
|
|
860
|
+
// src/cache/single-flight.ts
|
|
861
|
+
function singleFlight(cache) {
|
|
862
|
+
const pending = /* @__PURE__ */ new Map();
|
|
863
|
+
return {
|
|
864
|
+
load(key, fetcher, signal) {
|
|
865
|
+
if (signal?.aborted) return Promise.reject(makeAbortError());
|
|
866
|
+
const hit = cache.get(key);
|
|
867
|
+
if (hit !== void 0) return Promise.resolve(hit);
|
|
868
|
+
let entry = pending.get(key);
|
|
869
|
+
if (!entry) {
|
|
870
|
+
const controller = new AbortController();
|
|
871
|
+
const promise = invokeFetcher(fetcher, controller.signal).then((value) => {
|
|
872
|
+
cache.set(key, value);
|
|
873
|
+
return value;
|
|
874
|
+
}).finally(() => {
|
|
875
|
+
if (pending.get(key)?.promise === promise) pending.delete(key);
|
|
876
|
+
});
|
|
877
|
+
const fresh = { promise, controller, refCount: 0 };
|
|
878
|
+
pending.set(key, fresh);
|
|
879
|
+
entry = fresh;
|
|
880
|
+
}
|
|
881
|
+
const owned = entry;
|
|
882
|
+
owned.refCount++;
|
|
883
|
+
const release = () => {
|
|
884
|
+
owned.refCount--;
|
|
885
|
+
if (owned.refCount > 0) return;
|
|
886
|
+
if (pending.get(key) === owned) {
|
|
887
|
+
pending.delete(key);
|
|
888
|
+
owned.controller.abort();
|
|
889
|
+
}
|
|
890
|
+
};
|
|
891
|
+
if (!signal) {
|
|
892
|
+
return owned.promise.then(
|
|
893
|
+
(value) => {
|
|
894
|
+
release();
|
|
895
|
+
return value;
|
|
896
|
+
},
|
|
897
|
+
(error) => {
|
|
898
|
+
release();
|
|
899
|
+
throw error;
|
|
900
|
+
}
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
return new Promise((resolve, reject) => {
|
|
904
|
+
let done = false;
|
|
905
|
+
const finish = () => {
|
|
906
|
+
if (done) return false;
|
|
907
|
+
done = true;
|
|
908
|
+
signal.removeEventListener("abort", handleAbort);
|
|
909
|
+
release();
|
|
910
|
+
return true;
|
|
911
|
+
};
|
|
912
|
+
const handleAbort = () => {
|
|
913
|
+
if (finish()) reject(makeAbortError());
|
|
914
|
+
};
|
|
915
|
+
signal.addEventListener("abort", handleAbort, { once: true });
|
|
916
|
+
owned.promise.then(
|
|
917
|
+
(value) => {
|
|
918
|
+
if (finish()) resolve(value);
|
|
919
|
+
},
|
|
920
|
+
(error) => {
|
|
921
|
+
if (finish()) reject(error);
|
|
922
|
+
}
|
|
923
|
+
);
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
function invokeFetcher(fetcher, signal) {
|
|
929
|
+
try {
|
|
930
|
+
return fetcher(signal);
|
|
931
|
+
} catch (error) {
|
|
932
|
+
return Promise.reject(error);
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
function makeAbortError() {
|
|
936
|
+
return new DOMException("The operation was aborted.", "AbortError");
|
|
937
|
+
}
|
|
938
|
+
|
|
830
939
|
// src/format/flatbuffers/types.ts
|
|
831
940
|
function asObjectId12(bytes) {
|
|
832
941
|
if (bytes.length !== 12) {
|
|
@@ -1818,11 +1927,11 @@ function getChunkPayload(ref) {
|
|
|
1818
1927
|
|
|
1819
1928
|
// src/format/flatbuffers/metadata.ts
|
|
1820
1929
|
var import_msgpack = require("@msgpack/msgpack");
|
|
1821
|
-
var
|
|
1930
|
+
var flexbuffers2 = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
|
|
1822
1931
|
function deserializeMetadata(items, specVersion) {
|
|
1823
1932
|
const result = /* @__PURE__ */ Object.create(null);
|
|
1824
1933
|
for (const item of items) {
|
|
1825
|
-
result[item.name] = specVersion === 1 /* V1_0 */ ? (0, import_msgpack.decode)(item.value) :
|
|
1934
|
+
result[item.name] = specVersion === 1 /* V1_0 */ ? (0, import_msgpack.decode)(item.value) : flexbuffers2.toObject(
|
|
1826
1935
|
item.value.buffer.slice(
|
|
1827
1936
|
item.value.byteOffset,
|
|
1828
1937
|
item.value.byteOffset + item.value.byteLength
|
|
@@ -2154,17 +2263,219 @@ function parseTransactionLog(data) {
|
|
|
2154
2263
|
};
|
|
2155
2264
|
}
|
|
2156
2265
|
|
|
2266
|
+
// src/reader/range-coalescer.ts
|
|
2267
|
+
function expectedRangeLength(range) {
|
|
2268
|
+
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2269
|
+
}
|
|
2270
|
+
function makeUrlStore(opts) {
|
|
2271
|
+
const { url, fetchClient, conditionalHeaders } = opts;
|
|
2272
|
+
async function doFetch(init) {
|
|
2273
|
+
return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
|
|
2274
|
+
}
|
|
2275
|
+
return {
|
|
2276
|
+
async get() {
|
|
2277
|
+
throw new Error(
|
|
2278
|
+
`Virtual chunk URL store for ${url} only supports ranged reads`
|
|
2279
|
+
);
|
|
2280
|
+
},
|
|
2281
|
+
async getRange(_key, range, options) {
|
|
2282
|
+
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2283
|
+
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2284
|
+
const response = await doFetch({ headers, signal: options?.signal });
|
|
2285
|
+
if (response.status === 412) {
|
|
2286
|
+
throw new Error(
|
|
2287
|
+
`Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2288
|
+
);
|
|
2289
|
+
}
|
|
2290
|
+
if (response.status !== 200 && response.status !== 206) {
|
|
2291
|
+
throw new Error(
|
|
2292
|
+
`Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
|
|
2293
|
+
);
|
|
2294
|
+
}
|
|
2295
|
+
const data = new Uint8Array(await response.arrayBuffer());
|
|
2296
|
+
if (response.status === 206) {
|
|
2297
|
+
const expected = expectedRangeLength(range);
|
|
2298
|
+
if (data.length === expected) return data;
|
|
2299
|
+
throw new Error(
|
|
2300
|
+
`Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
|
|
2301
|
+
);
|
|
2302
|
+
}
|
|
2303
|
+
if ("offset" in range) {
|
|
2304
|
+
const end = range.offset + range.length;
|
|
2305
|
+
if (data.length >= end) return data.slice(range.offset, end);
|
|
2306
|
+
throw new Error(
|
|
2307
|
+
`Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2308
|
+
);
|
|
2309
|
+
}
|
|
2310
|
+
if (data.length >= range.suffixLength) {
|
|
2311
|
+
return data.slice(data.length - range.suffixLength);
|
|
2312
|
+
}
|
|
2313
|
+
throw new Error(
|
|
2314
|
+
`Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2315
|
+
);
|
|
2316
|
+
}
|
|
2317
|
+
};
|
|
2318
|
+
}
|
|
2319
|
+
function makeStorageStore(storage) {
|
|
2320
|
+
return {
|
|
2321
|
+
async get(key, options) {
|
|
2322
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2323
|
+
return storage.getObject(key, void 0, storageOptions);
|
|
2324
|
+
},
|
|
2325
|
+
async getRange(key, range, options) {
|
|
2326
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2327
|
+
if ("suffixLength" in range) {
|
|
2328
|
+
throw new Error(
|
|
2329
|
+
`Storage suffix ranges are not supported for ${key}; convert suffixLength to offset/length before reading`
|
|
2330
|
+
);
|
|
2331
|
+
}
|
|
2332
|
+
const storageRange = {
|
|
2333
|
+
start: range.offset,
|
|
2334
|
+
end: range.offset + range.length
|
|
2335
|
+
};
|
|
2336
|
+
const data = await storage.getObject(key, storageRange, storageOptions);
|
|
2337
|
+
if (data.length === range.length) return data;
|
|
2338
|
+
if (data.length >= storageRange.end) {
|
|
2339
|
+
return data.slice(storageRange.start, storageRange.end);
|
|
2340
|
+
}
|
|
2341
|
+
throw new Error(
|
|
2342
|
+
`Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
|
|
2343
|
+
);
|
|
2344
|
+
}
|
|
2345
|
+
};
|
|
2346
|
+
}
|
|
2347
|
+
|
|
2157
2348
|
// src/reader/session.ts
|
|
2349
|
+
var RANGE_COALESCE_SIZE = 32 * 1024;
|
|
2350
|
+
var RANGE_STORE_CACHE_SIZE = 256;
|
|
2351
|
+
function makeRangeStoreCacheKey(parts) {
|
|
2352
|
+
return JSON.stringify(parts);
|
|
2353
|
+
}
|
|
2158
2354
|
var ReadSession = class _ReadSession {
|
|
2159
2355
|
storage;
|
|
2160
2356
|
snapshot;
|
|
2161
2357
|
specVersion;
|
|
2162
2358
|
manifestCache;
|
|
2163
|
-
|
|
2359
|
+
/**
|
|
2360
|
+
* Single-flight loader over `manifestCache`: concurrent misses for the
|
|
2361
|
+
* same manifest share one fetch instead of racing N identical GETs.
|
|
2362
|
+
* Critical for fan-out reads that all need the same manifest (e.g. the
|
|
2363
|
+
* many parallel chunk reads issued when a renderer crosses into a new
|
|
2364
|
+
* pyramid level).
|
|
2365
|
+
*/
|
|
2366
|
+
manifestLoader;
|
|
2367
|
+
/**
|
|
2368
|
+
* Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
|
|
2369
|
+
* The cache key includes request-option identities that must not share one
|
|
2370
|
+
* coalescing queue, notably virtual fetch clients and checksum headers.
|
|
2371
|
+
*
|
|
2372
|
+
* Promise slots are inserted synchronously on first use, so
|
|
2373
|
+
* concurrent requests for the same partition share one coalescing window
|
|
2374
|
+
* instead of racing to create parallel stores.
|
|
2375
|
+
*/
|
|
2376
|
+
rangeStores;
|
|
2377
|
+
nativeStore;
|
|
2378
|
+
fetchClientIds;
|
|
2379
|
+
nextFetchClientId = 1;
|
|
2380
|
+
rangeCoalescerIds;
|
|
2381
|
+
nextRangeCoalescerId = 1;
|
|
2382
|
+
/** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
|
|
2383
|
+
virtualChunkContainers;
|
|
2384
|
+
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
2164
2385
|
this.storage = storage;
|
|
2165
2386
|
this.snapshot = snapshot;
|
|
2166
2387
|
this.specVersion = specVersion;
|
|
2167
2388
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
2389
|
+
this.manifestLoader = singleFlight(this.manifestCache);
|
|
2390
|
+
this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
|
|
2391
|
+
}
|
|
2392
|
+
getFetchClientKey(fetchClient) {
|
|
2393
|
+
if (!fetchClient) return "default";
|
|
2394
|
+
if (!this.fetchClientIds) this.fetchClientIds = /* @__PURE__ */ new WeakMap();
|
|
2395
|
+
let id = this.fetchClientIds.get(fetchClient);
|
|
2396
|
+
if (id === void 0) {
|
|
2397
|
+
id = this.nextFetchClientId;
|
|
2398
|
+
this.nextFetchClientId = id + 1;
|
|
2399
|
+
this.fetchClientIds.set(fetchClient, id);
|
|
2400
|
+
}
|
|
2401
|
+
return String(id);
|
|
2402
|
+
}
|
|
2403
|
+
getRangeCoalescerKey(withRangeCoalescing) {
|
|
2404
|
+
if (!this.rangeCoalescerIds) this.rangeCoalescerIds = /* @__PURE__ */ new WeakMap();
|
|
2405
|
+
let id = this.rangeCoalescerIds.get(withRangeCoalescing);
|
|
2406
|
+
if (id === void 0) {
|
|
2407
|
+
id = this.nextRangeCoalescerId;
|
|
2408
|
+
this.nextRangeCoalescerId = id + 1;
|
|
2409
|
+
this.rangeCoalescerIds.set(withRangeCoalescing, id);
|
|
2410
|
+
}
|
|
2411
|
+
return String(id);
|
|
2412
|
+
}
|
|
2413
|
+
getRangeStore(partitionKey, createStore, withRangeCoalescing) {
|
|
2414
|
+
if (!this.rangeStores) {
|
|
2415
|
+
this.rangeStores = new LRUCache(RANGE_STORE_CACHE_SIZE);
|
|
2416
|
+
}
|
|
2417
|
+
const stores = this.rangeStores;
|
|
2418
|
+
const cacheKey = makeRangeStoreCacheKey([
|
|
2419
|
+
...partitionKey,
|
|
2420
|
+
["coalescer", this.getRangeCoalescerKey(withRangeCoalescing)]
|
|
2421
|
+
]);
|
|
2422
|
+
const cached = stores.get(cacheKey);
|
|
2423
|
+
if (cached) return cached;
|
|
2424
|
+
const raw = createStore();
|
|
2425
|
+
const promise = Promise.resolve().then(
|
|
2426
|
+
() => withRangeCoalescing(raw, { coalesceSize: RANGE_COALESCE_SIZE })
|
|
2427
|
+
).catch((error) => {
|
|
2428
|
+
if (stores.get(cacheKey) === promise) stores.delete(cacheKey);
|
|
2429
|
+
throw error;
|
|
2430
|
+
});
|
|
2431
|
+
stores.set(cacheKey, promise);
|
|
2432
|
+
return promise;
|
|
2433
|
+
}
|
|
2434
|
+
getNativeStore(options) {
|
|
2435
|
+
if (!this.nativeStore) {
|
|
2436
|
+
this.nativeStore = makeStorageStore(this.storage);
|
|
2437
|
+
}
|
|
2438
|
+
const raw = this.nativeStore;
|
|
2439
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
2440
|
+
if (!withRangeCoalescing) {
|
|
2441
|
+
return Promise.resolve(raw);
|
|
2442
|
+
}
|
|
2443
|
+
return this.getRangeStore(["native"], () => raw, withRangeCoalescing);
|
|
2444
|
+
}
|
|
2445
|
+
getVirtualStoreForPayload(httpUrl, payload, options) {
|
|
2446
|
+
const validate = !!options?.validateChecksums;
|
|
2447
|
+
let conditionalHeaders;
|
|
2448
|
+
if (validate) {
|
|
2449
|
+
conditionalHeaders = {};
|
|
2450
|
+
if (payload.checksumEtag) {
|
|
2451
|
+
conditionalHeaders["If-Match"] = payload.checksumEtag;
|
|
2452
|
+
}
|
|
2453
|
+
if (payload.checksumLastModified > 0) {
|
|
2454
|
+
conditionalHeaders["If-Unmodified-Since"] = new Date(
|
|
2455
|
+
payload.checksumLastModified * 1e3
|
|
2456
|
+
).toUTCString();
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2459
|
+
const checksumKey = validate ? ["checked", payload.checksumEtag ?? "", payload.checksumLastModified] : ["unchecked"];
|
|
2460
|
+
const createStore = () => makeUrlStore({
|
|
2461
|
+
url: httpUrl,
|
|
2462
|
+
fetchClient: options?.fetchClient,
|
|
2463
|
+
conditionalHeaders
|
|
2464
|
+
});
|
|
2465
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
2466
|
+
if (!withRangeCoalescing) {
|
|
2467
|
+
return Promise.resolve(createStore());
|
|
2468
|
+
}
|
|
2469
|
+
return this.getRangeStore(
|
|
2470
|
+
[
|
|
2471
|
+
"virtual",
|
|
2472
|
+
httpUrl,
|
|
2473
|
+
["fetch", this.getFetchClientKey(options?.fetchClient)],
|
|
2474
|
+
checksumKey
|
|
2475
|
+
],
|
|
2476
|
+
createStore,
|
|
2477
|
+
withRangeCoalescing
|
|
2478
|
+
);
|
|
2168
2479
|
}
|
|
2169
2480
|
/**
|
|
2170
2481
|
* Open a read session for a specific snapshot.
|
|
@@ -2184,7 +2495,8 @@ var ReadSession = class _ReadSession {
|
|
|
2184
2495
|
storage,
|
|
2185
2496
|
snapshot,
|
|
2186
2497
|
specVersion,
|
|
2187
|
-
options?.maxManifestCacheSize
|
|
2498
|
+
options?.maxManifestCacheSize,
|
|
2499
|
+
options?.virtualChunkContainers
|
|
2188
2500
|
);
|
|
2189
2501
|
}
|
|
2190
2502
|
/** Load and parse a snapshot from storage */
|
|
@@ -2202,22 +2514,40 @@ var ReadSession = class _ReadSession {
|
|
|
2202
2514
|
specVersion: header.specVersion
|
|
2203
2515
|
};
|
|
2204
2516
|
}
|
|
2205
|
-
/**
|
|
2206
|
-
|
|
2517
|
+
/**
|
|
2518
|
+
* Load and parse a manifest from storage.
|
|
2519
|
+
*
|
|
2520
|
+
* Concurrent calls for the same manifest are coalesced through
|
|
2521
|
+
* `manifestLoader` so a fan-out of chunk reads — the typical case
|
|
2522
|
+
* when many tiles cross into a new pyramid level at once — issues one
|
|
2523
|
+
* HTTP GET instead of one per caller. A caller's `signal` rejects only
|
|
2524
|
+
* that caller's await; the shared fetch is aborted only once every
|
|
2525
|
+
* active waiter has aborted.
|
|
2526
|
+
*/
|
|
2527
|
+
loadManifest(manifestId, options) {
|
|
2207
2528
|
const idStr = encodeObjectId12(manifestId);
|
|
2208
|
-
|
|
2209
|
-
|
|
2529
|
+
return this.manifestLoader.load(
|
|
2530
|
+
idStr,
|
|
2531
|
+
(signal) => this.fetchManifest(idStr, signal),
|
|
2532
|
+
options?.signal
|
|
2533
|
+
);
|
|
2534
|
+
}
|
|
2535
|
+
/**
|
|
2536
|
+
* Fetch and parse a single manifest. The `signal` here is the
|
|
2537
|
+
* single-flight loader's own signal — it fires only when every waiter
|
|
2538
|
+
* has aborted, so a hung manifest fetch doesn't trap later callers
|
|
2539
|
+
* behind it.
|
|
2540
|
+
*/
|
|
2541
|
+
async fetchManifest(idStr, signal) {
|
|
2210
2542
|
const path = getManifestPath(idStr);
|
|
2211
|
-
const data = await this.storage.getObject(path, void 0,
|
|
2543
|
+
const data = await this.storage.getObject(path, void 0, { signal });
|
|
2212
2544
|
const header = parseHeader(data);
|
|
2213
2545
|
validateFileType(header, 2 /* Manifest */);
|
|
2214
2546
|
let flatbufferData = getDataAfterHeader(data);
|
|
2215
2547
|
if (header.compression === 1 /* Zstd */) {
|
|
2216
2548
|
flatbufferData = (0, import_fzstd2.decompress)(flatbufferData);
|
|
2217
2549
|
}
|
|
2218
|
-
|
|
2219
|
-
this.manifestCache.set(idStr, manifest);
|
|
2220
|
-
return manifest;
|
|
2550
|
+
return parseManifest(flatbufferData);
|
|
2221
2551
|
}
|
|
2222
2552
|
/**
|
|
2223
2553
|
* Get the snapshot ID.
|
|
@@ -2348,26 +2678,25 @@ var ReadSession = class _ReadSession {
|
|
|
2348
2678
|
* @returns Chunk data bytes or null if not found
|
|
2349
2679
|
*/
|
|
2350
2680
|
async getChunk(path, coords, options) {
|
|
2351
|
-
|
|
2681
|
+
options?.signal?.throwIfAborted();
|
|
2682
|
+
const requestOptions = toRequestOptions(options);
|
|
2352
2683
|
const node = this.getNode(path);
|
|
2353
2684
|
if (!node || node.nodeData.type !== "array") {
|
|
2354
2685
|
return null;
|
|
2355
2686
|
}
|
|
2356
2687
|
const arrayData = node.nodeData;
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
continue;
|
|
2361
|
-
}
|
|
2362
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2363
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2364
|
-
if (!chunkRef) continue;
|
|
2365
|
-
const payload = getChunkPayload(chunkRef);
|
|
2366
|
-
return this.fetchChunkPayload(payload, options);
|
|
2688
|
+
for (const manifestRef of arrayData.manifests) {
|
|
2689
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
2690
|
+
continue;
|
|
2367
2691
|
}
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2692
|
+
const manifest = await this.loadManifest(
|
|
2693
|
+
manifestRef.objectId,
|
|
2694
|
+
requestOptions
|
|
2695
|
+
);
|
|
2696
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2697
|
+
if (!chunkRef) continue;
|
|
2698
|
+
const payload = getChunkPayload(chunkRef);
|
|
2699
|
+
return this.fetchChunkPayload(payload, options);
|
|
2371
2700
|
}
|
|
2372
2701
|
return null;
|
|
2373
2702
|
}
|
|
@@ -2385,26 +2714,25 @@ var ReadSession = class _ReadSession {
|
|
|
2385
2714
|
* @returns Chunk data bytes or null if not found
|
|
2386
2715
|
*/
|
|
2387
2716
|
async getChunkRange(path, coords, range, options) {
|
|
2388
|
-
|
|
2717
|
+
options?.signal?.throwIfAborted();
|
|
2718
|
+
const requestOptions = toRequestOptions(options);
|
|
2389
2719
|
const node = this.getNode(path);
|
|
2390
2720
|
if (!node || node.nodeData.type !== "array") {
|
|
2391
2721
|
return null;
|
|
2392
2722
|
}
|
|
2393
2723
|
const arrayData = node.nodeData;
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
continue;
|
|
2398
|
-
}
|
|
2399
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2400
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2401
|
-
if (!chunkRef) continue;
|
|
2402
|
-
const payload = getChunkPayload(chunkRef);
|
|
2403
|
-
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2724
|
+
for (const manifestRef of arrayData.manifests) {
|
|
2725
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
2726
|
+
continue;
|
|
2404
2727
|
}
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2728
|
+
const manifest = await this.loadManifest(
|
|
2729
|
+
manifestRef.objectId,
|
|
2730
|
+
requestOptions
|
|
2731
|
+
);
|
|
2732
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2733
|
+
if (!chunkRef) continue;
|
|
2734
|
+
const payload = getChunkPayload(chunkRef);
|
|
2735
|
+
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2408
2736
|
}
|
|
2409
2737
|
return null;
|
|
2410
2738
|
}
|
|
@@ -2426,77 +2754,54 @@ var ReadSession = class _ReadSession {
|
|
|
2426
2754
|
return payload.data;
|
|
2427
2755
|
case "native": {
|
|
2428
2756
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2429
|
-
const
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2757
|
+
const requestedStart = payload.offset;
|
|
2758
|
+
const requestedEnd = payload.offset + payload.length;
|
|
2759
|
+
const store = await this.getNativeStore(options);
|
|
2760
|
+
const data = await store.getRange(
|
|
2761
|
+
path,
|
|
2762
|
+
{ offset: requestedStart, length: payload.length },
|
|
2763
|
+
{ signal: options?.signal }
|
|
2764
|
+
);
|
|
2765
|
+
if (!data) {
|
|
2766
|
+
throw new Error(
|
|
2767
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
2768
|
+
);
|
|
2437
2769
|
}
|
|
2770
|
+
if (data.length === payload.length) return data;
|
|
2438
2771
|
throw new Error(
|
|
2439
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
2772
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${payload.length} bytes`
|
|
2440
2773
|
);
|
|
2441
2774
|
}
|
|
2442
2775
|
case "virtual": {
|
|
2443
|
-
const
|
|
2776
|
+
const absoluteLocation = expandVccUrl(
|
|
2444
2777
|
payload.location,
|
|
2778
|
+
this.virtualChunkContainers
|
|
2779
|
+
);
|
|
2780
|
+
const httpUrl = translateToHttpUrl(
|
|
2781
|
+
absoluteLocation,
|
|
2445
2782
|
options?.azureAccount
|
|
2446
2783
|
);
|
|
2447
|
-
const
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
}
|
|
2459
|
-
}
|
|
2460
|
-
const fetchInit = {
|
|
2461
|
-
headers,
|
|
2462
|
-
signal: options?.signal
|
|
2463
|
-
};
|
|
2464
|
-
let response;
|
|
2465
|
-
try {
|
|
2466
|
-
const client = options?.fetchClient;
|
|
2467
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2468
|
-
} catch (error) {
|
|
2469
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2470
|
-
throw new AbortError();
|
|
2471
|
-
}
|
|
2472
|
-
throw error;
|
|
2473
|
-
}
|
|
2474
|
-
if (response.status === 412) {
|
|
2784
|
+
const store = await this.getVirtualStoreForPayload(
|
|
2785
|
+
httpUrl,
|
|
2786
|
+
payload,
|
|
2787
|
+
options
|
|
2788
|
+
);
|
|
2789
|
+
const data = await store.getRange(
|
|
2790
|
+
"/",
|
|
2791
|
+
{ offset: payload.offset, length: payload.length },
|
|
2792
|
+
{ signal: options?.signal }
|
|
2793
|
+
);
|
|
2794
|
+
if (!data) {
|
|
2475
2795
|
throw new Error(
|
|
2476
|
-
`
|
|
2796
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2477
2797
|
);
|
|
2478
2798
|
}
|
|
2479
|
-
if (
|
|
2799
|
+
if (data.length !== payload.length) {
|
|
2480
2800
|
throw new Error(
|
|
2481
|
-
`
|
|
2801
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2482
2802
|
);
|
|
2483
2803
|
}
|
|
2484
|
-
|
|
2485
|
-
if (response.status === 206) {
|
|
2486
|
-
if (data.length !== payload.length) {
|
|
2487
|
-
throw new Error(
|
|
2488
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2489
|
-
);
|
|
2490
|
-
}
|
|
2491
|
-
return data;
|
|
2492
|
-
}
|
|
2493
|
-
const absoluteEnd = payload.offset + payload.length;
|
|
2494
|
-
if (data.length >= absoluteEnd) {
|
|
2495
|
-
return data.slice(payload.offset, absoluteEnd);
|
|
2496
|
-
}
|
|
2497
|
-
throw new Error(
|
|
2498
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2499
|
-
);
|
|
2804
|
+
return data;
|
|
2500
2805
|
}
|
|
2501
2806
|
}
|
|
2502
2807
|
}
|
|
@@ -2516,80 +2821,57 @@ var ReadSession = class _ReadSession {
|
|
|
2516
2821
|
return payload.data.slice(rangeStart, rangeEnd);
|
|
2517
2822
|
case "native": {
|
|
2518
2823
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2519
|
-
const
|
|
2520
|
-
start: payload.offset + rangeStart,
|
|
2521
|
-
end: payload.offset + rangeEnd
|
|
2522
|
-
};
|
|
2824
|
+
const requestedStart = payload.offset + rangeStart;
|
|
2523
2825
|
const expectedSize = rangeEnd - rangeStart;
|
|
2524
|
-
const
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2826
|
+
const requestedEnd = requestedStart + expectedSize;
|
|
2827
|
+
const store = await this.getNativeStore(options);
|
|
2828
|
+
const data = await store.getRange(
|
|
2829
|
+
path,
|
|
2830
|
+
{ offset: requestedStart, length: expectedSize },
|
|
2831
|
+
{ signal: options?.signal }
|
|
2832
|
+
);
|
|
2833
|
+
if (!data) {
|
|
2834
|
+
throw new Error(
|
|
2835
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
2836
|
+
);
|
|
2528
2837
|
}
|
|
2838
|
+
if (data.length === expectedSize) return data;
|
|
2529
2839
|
throw new Error(
|
|
2530
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
2840
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${expectedSize} bytes`
|
|
2531
2841
|
);
|
|
2532
2842
|
}
|
|
2533
2843
|
case "virtual": {
|
|
2534
2844
|
const absoluteStart = payload.offset + rangeStart;
|
|
2535
|
-
const
|
|
2536
|
-
const
|
|
2845
|
+
const expectedSize = rangeEnd - rangeStart;
|
|
2846
|
+
const absoluteLocation = expandVccUrl(
|
|
2537
2847
|
payload.location,
|
|
2848
|
+
this.virtualChunkContainers
|
|
2849
|
+
);
|
|
2850
|
+
const httpUrl = translateToHttpUrl(
|
|
2851
|
+
absoluteLocation,
|
|
2538
2852
|
options?.azureAccount
|
|
2539
2853
|
);
|
|
2540
|
-
const
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
}
|
|
2552
|
-
}
|
|
2553
|
-
const fetchInit = {
|
|
2554
|
-
headers,
|
|
2555
|
-
signal: options?.signal
|
|
2556
|
-
};
|
|
2557
|
-
let response;
|
|
2558
|
-
try {
|
|
2559
|
-
const client = options?.fetchClient;
|
|
2560
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2561
|
-
} catch (error) {
|
|
2562
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2563
|
-
throw new AbortError();
|
|
2564
|
-
}
|
|
2565
|
-
throw error;
|
|
2566
|
-
}
|
|
2567
|
-
if (response.status === 412) {
|
|
2854
|
+
const store = await this.getVirtualStoreForPayload(
|
|
2855
|
+
httpUrl,
|
|
2856
|
+
payload,
|
|
2857
|
+
options
|
|
2858
|
+
);
|
|
2859
|
+
const data = await store.getRange(
|
|
2860
|
+
"/",
|
|
2861
|
+
{ offset: absoluteStart, length: expectedSize },
|
|
2862
|
+
{ signal: options?.signal }
|
|
2863
|
+
);
|
|
2864
|
+
if (!data) {
|
|
2568
2865
|
throw new Error(
|
|
2569
|
-
`
|
|
2866
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2570
2867
|
);
|
|
2571
2868
|
}
|
|
2572
|
-
if (
|
|
2869
|
+
if (data.length !== expectedSize) {
|
|
2573
2870
|
throw new Error(
|
|
2574
|
-
`
|
|
2871
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2575
2872
|
);
|
|
2576
2873
|
}
|
|
2577
|
-
|
|
2578
|
-
const data = new Uint8Array(await response.arrayBuffer());
|
|
2579
|
-
if (response.status === 206) {
|
|
2580
|
-
if (data.length !== expectedSize) {
|
|
2581
|
-
throw new Error(
|
|
2582
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2583
|
-
);
|
|
2584
|
-
}
|
|
2585
|
-
return data;
|
|
2586
|
-
}
|
|
2587
|
-
if (data.length >= absoluteEnd) {
|
|
2588
|
-
return data.slice(absoluteStart, absoluteEnd);
|
|
2589
|
-
}
|
|
2590
|
-
throw new Error(
|
|
2591
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2592
|
-
);
|
|
2874
|
+
return data;
|
|
2593
2875
|
}
|
|
2594
2876
|
}
|
|
2595
2877
|
}
|
|
@@ -2613,6 +2895,19 @@ var ReadSession = class _ReadSession {
|
|
|
2613
2895
|
}
|
|
2614
2896
|
};
|
|
2615
2897
|
var utf8Encoder = new TextEncoder();
|
|
2898
|
+
function toRequestOptions(options) {
|
|
2899
|
+
if (!options) return void 0;
|
|
2900
|
+
return {
|
|
2901
|
+
...options.signal && { signal: options.signal },
|
|
2902
|
+
...options.fetchClient && { fetchClient: options.fetchClient },
|
|
2903
|
+
...options.validateChecksums !== void 0 && {
|
|
2904
|
+
validateChecksums: options.validateChecksums
|
|
2905
|
+
},
|
|
2906
|
+
...options.azureAccount !== void 0 && {
|
|
2907
|
+
azureAccount: options.azureAccount
|
|
2908
|
+
}
|
|
2909
|
+
};
|
|
2910
|
+
}
|
|
2616
2911
|
function compareUtf8Bytes(a, b) {
|
|
2617
2912
|
const bytesA = utf8Encoder.encode(a);
|
|
2618
2913
|
const bytesB = utf8Encoder.encode(b);
|
|
@@ -2624,6 +2919,28 @@ function compareUtf8Bytes(a, b) {
|
|
|
2624
2919
|
}
|
|
2625
2920
|
return bytesA.length - bytesB.length;
|
|
2626
2921
|
}
|
|
2922
|
+
var VCC_SCHEME = "vcc://";
|
|
2923
|
+
function expandVccUrl(location, containers) {
|
|
2924
|
+
if (!location.startsWith(VCC_SCHEME)) return location;
|
|
2925
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
2926
|
+
const slash = rest.indexOf("/");
|
|
2927
|
+
if (slash === -1) {
|
|
2928
|
+
throw new Error(
|
|
2929
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
2930
|
+
);
|
|
2931
|
+
}
|
|
2932
|
+
const name = rest.slice(0, slash);
|
|
2933
|
+
const relativePath = rest.slice(slash + 1);
|
|
2934
|
+
const urlPrefix = containers.get(name);
|
|
2935
|
+
if (urlPrefix === void 0) {
|
|
2936
|
+
if (containers.size === 0) return location;
|
|
2937
|
+
throw new Error(
|
|
2938
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
2939
|
+
);
|
|
2940
|
+
}
|
|
2941
|
+
const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
|
|
2942
|
+
return normalizedPrefix + relativePath;
|
|
2943
|
+
}
|
|
2627
2944
|
function translateToHttpUrl(url, azureAccount) {
|
|
2628
2945
|
if (url.startsWith("s3://")) {
|
|
2629
2946
|
const rest = url.slice(5);
|
|
@@ -2699,7 +3016,7 @@ var Repository = class _Repository {
|
|
|
2699
3016
|
* @param options - Optional request options (signal for cancellation)
|
|
2700
3017
|
* @returns ParsedRepo if v2 format, null if v1 format
|
|
2701
3018
|
* @throws Error if repo file exists but fails to parse
|
|
2702
|
-
* @throws AbortError if the operation was aborted
|
|
3019
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
2703
3020
|
*/
|
|
2704
3021
|
async loadRepoInfo(options) {
|
|
2705
3022
|
if (this.repoInfoAttempted) {
|
|
@@ -2715,7 +3032,7 @@ var Repository = class _Repository {
|
|
|
2715
3032
|
this.repoInfo = parseRepo(data);
|
|
2716
3033
|
return this.repoInfo;
|
|
2717
3034
|
} catch (error) {
|
|
2718
|
-
if (error
|
|
3035
|
+
if (isAbortError(error)) {
|
|
2719
3036
|
this.repoInfoAttempted = false;
|
|
2720
3037
|
throw error;
|
|
2721
3038
|
}
|
|
@@ -2780,7 +3097,7 @@ var Repository = class _Repository {
|
|
|
2780
3097
|
}
|
|
2781
3098
|
}
|
|
2782
3099
|
} catch (error) {
|
|
2783
|
-
if (error
|
|
3100
|
+
if (isAbortError(error)) {
|
|
2784
3101
|
throw error;
|
|
2785
3102
|
}
|
|
2786
3103
|
if (legacyPath) {
|
|
@@ -2796,7 +3113,7 @@ var Repository = class _Repository {
|
|
|
2796
3113
|
* version has the highest filename. Deletion tombstones are checked.
|
|
2797
3114
|
*
|
|
2798
3115
|
* When listing is not supported (e.g., HTTP storage), only the legacy
|
|
2799
|
-
* ref.json path is
|
|
3116
|
+
* ref.json path is returned (caller must handle NotFoundError on read).
|
|
2800
3117
|
*
|
|
2801
3118
|
* @param dirPrefix - Directory prefix to search
|
|
2802
3119
|
* @param legacyPath - Optional legacy ref.json path to try if listing fails
|
|
@@ -2813,7 +3130,7 @@ var Repository = class _Repository {
|
|
|
2813
3130
|
}
|
|
2814
3131
|
}
|
|
2815
3132
|
} catch (error) {
|
|
2816
|
-
if (error
|
|
3133
|
+
if (isAbortError(error)) {
|
|
2817
3134
|
throw error;
|
|
2818
3135
|
}
|
|
2819
3136
|
return legacyPath || null;
|
|
@@ -2832,7 +3149,9 @@ var Repository = class _Repository {
|
|
|
2832
3149
|
/**
|
|
2833
3150
|
* List all branches in the repository.
|
|
2834
3151
|
*
|
|
2835
|
-
*
|
|
3152
|
+
* Unlike tags, branches in icechunk v1 have no tombstone mechanism —
|
|
3153
|
+
* deletion removes the ref file outright. A branch is present iff it has
|
|
3154
|
+
* at least one ref file in its directory.
|
|
2836
3155
|
*
|
|
2837
3156
|
* @returns Array of branch names
|
|
2838
3157
|
*/
|
|
@@ -2841,49 +3160,23 @@ var Repository = class _Repository {
|
|
|
2841
3160
|
if (repoInfo) {
|
|
2842
3161
|
return listBranchesFromRepo(repoInfo);
|
|
2843
3162
|
}
|
|
2844
|
-
const
|
|
3163
|
+
const branches = /* @__PURE__ */ new Set();
|
|
2845
3164
|
try {
|
|
2846
3165
|
for await (const path of this.storage.listPrefix(
|
|
2847
3166
|
`${PATHS.REFS}/branch.`
|
|
2848
3167
|
)) {
|
|
2849
|
-
const
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
);
|
|
2853
|
-
if (deletedMatch) {
|
|
2854
|
-
const branchName = deletedMatch[1];
|
|
2855
|
-
const refFile = `refs/branch.${branchName}/${deletedMatch[2]}`;
|
|
2856
|
-
if (!branchFiles.has(branchName)) {
|
|
2857
|
-
branchFiles.set(branchName, {
|
|
2858
|
-
jsonFiles: [],
|
|
2859
|
-
deletedFiles: /* @__PURE__ */ new Set()
|
|
2860
|
-
});
|
|
2861
|
-
}
|
|
2862
|
-
branchFiles.get(branchName).deletedFiles.add(refFile);
|
|
2863
|
-
} else if (jsonMatch) {
|
|
2864
|
-
const branchName = jsonMatch[1];
|
|
2865
|
-
if (!branchFiles.has(branchName)) {
|
|
2866
|
-
branchFiles.set(branchName, {
|
|
2867
|
-
jsonFiles: [],
|
|
2868
|
-
deletedFiles: /* @__PURE__ */ new Set()
|
|
2869
|
-
});
|
|
2870
|
-
}
|
|
2871
|
-
branchFiles.get(branchName).jsonFiles.push(path);
|
|
3168
|
+
const match = path.match(/^refs\/branch\.(.+)\/([^/]+\.json)$/);
|
|
3169
|
+
if (match) {
|
|
3170
|
+
branches.add(match[1]);
|
|
2872
3171
|
}
|
|
2873
3172
|
}
|
|
2874
|
-
} catch {
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
const result = [];
|
|
2878
|
-
for (const [branchName, { jsonFiles, deletedFiles }] of branchFiles) {
|
|
2879
|
-
if (jsonFiles.length === 0) continue;
|
|
2880
|
-
jsonFiles.sort();
|
|
2881
|
-
const latestRef = jsonFiles[jsonFiles.length - 1];
|
|
2882
|
-
if (!deletedFiles.has(latestRef)) {
|
|
2883
|
-
result.push(branchName);
|
|
3173
|
+
} catch (error) {
|
|
3174
|
+
if (isAbortError(error)) {
|
|
3175
|
+
throw error;
|
|
2884
3176
|
}
|
|
3177
|
+
throw new Error("Cannot list branches: storage does not support listing");
|
|
2885
3178
|
}
|
|
2886
|
-
return
|
|
3179
|
+
return [...branches];
|
|
2887
3180
|
}
|
|
2888
3181
|
/**
|
|
2889
3182
|
* List all tags in the repository.
|
|
@@ -2919,7 +3212,10 @@ var Repository = class _Repository {
|
|
|
2919
3212
|
tagFiles.get(tagName).jsonFiles.push(path);
|
|
2920
3213
|
}
|
|
2921
3214
|
}
|
|
2922
|
-
} catch {
|
|
3215
|
+
} catch (error) {
|
|
3216
|
+
if (isAbortError(error)) {
|
|
3217
|
+
throw error;
|
|
3218
|
+
}
|
|
2923
3219
|
throw new Error("Cannot list tags: storage does not support listing");
|
|
2924
3220
|
}
|
|
2925
3221
|
const result = [];
|
|
@@ -2947,7 +3243,10 @@ var Repository = class _Repository {
|
|
|
2947
3243
|
if (!snapshotId2) {
|
|
2948
3244
|
throw new Error(`Branch not found: ${name}`);
|
|
2949
3245
|
}
|
|
2950
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3246
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3247
|
+
...options,
|
|
3248
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3249
|
+
});
|
|
2951
3250
|
}
|
|
2952
3251
|
const refDirPath = getBranchRefDirPath(name);
|
|
2953
3252
|
const refPath = await this.findLatestRefFile(
|
|
@@ -2974,7 +3273,10 @@ var Repository = class _Repository {
|
|
|
2974
3273
|
if (!snapshotId2) {
|
|
2975
3274
|
throw new Error(`Tag not found: ${name}`);
|
|
2976
3275
|
}
|
|
2977
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3276
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3277
|
+
...options,
|
|
3278
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3279
|
+
});
|
|
2978
3280
|
}
|
|
2979
3281
|
const refDirPath = getTagRefDirPath(name);
|
|
2980
3282
|
const legacyPath = getTagRefPath(name);
|
|
@@ -2997,7 +3299,12 @@ var Repository = class _Repository {
|
|
|
2997
3299
|
*/
|
|
2998
3300
|
async checkoutSnapshot(snapshotId, options) {
|
|
2999
3301
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
3000
|
-
|
|
3302
|
+
const repoInfo = await this.loadRepoInfo(options);
|
|
3303
|
+
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
3304
|
+
return ReadSession.open(this.storage, id, {
|
|
3305
|
+
...options,
|
|
3306
|
+
...virtualChunkContainers && { virtualChunkContainers }
|
|
3307
|
+
});
|
|
3001
3308
|
}
|
|
3002
3309
|
/**
|
|
3003
3310
|
* Walk the snapshot history chain starting from the given session.
|
|
@@ -3076,9 +3383,7 @@ var HttpStorage = class {
|
|
|
3076
3383
|
return headers;
|
|
3077
3384
|
}
|
|
3078
3385
|
async getObject(path, range, options) {
|
|
3079
|
-
|
|
3080
|
-
throw new AbortError();
|
|
3081
|
-
}
|
|
3386
|
+
options?.signal?.throwIfAborted();
|
|
3082
3387
|
const url = this.getUrl(path);
|
|
3083
3388
|
const headers = this.getHeaders(range);
|
|
3084
3389
|
let response;
|
|
@@ -3091,9 +3396,7 @@ var HttpStorage = class {
|
|
|
3091
3396
|
signal: options?.signal
|
|
3092
3397
|
});
|
|
3093
3398
|
} catch (error) {
|
|
3094
|
-
if (error
|
|
3095
|
-
throw new AbortError();
|
|
3096
|
-
}
|
|
3399
|
+
if (isAbortError(error)) throw error;
|
|
3097
3400
|
throw new StorageError(
|
|
3098
3401
|
`Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3099
3402
|
error instanceof Error ? error : void 0
|
|
@@ -3111,9 +3414,7 @@ var HttpStorage = class {
|
|
|
3111
3414
|
return new Uint8Array(arrayBuffer);
|
|
3112
3415
|
}
|
|
3113
3416
|
async exists(path, options) {
|
|
3114
|
-
|
|
3115
|
-
throw new AbortError();
|
|
3116
|
-
}
|
|
3417
|
+
options?.signal?.throwIfAborted();
|
|
3117
3418
|
const url = this.getUrl(path);
|
|
3118
3419
|
try {
|
|
3119
3420
|
const response = await fetch(url, {
|
|
@@ -3124,9 +3425,7 @@ var HttpStorage = class {
|
|
|
3124
3425
|
});
|
|
3125
3426
|
return response.ok;
|
|
3126
3427
|
} catch (error) {
|
|
3127
|
-
if (error
|
|
3128
|
-
throw new AbortError();
|
|
3129
|
-
}
|
|
3428
|
+
if (isAbortError(error)) throw error;
|
|
3130
3429
|
return false;
|
|
3131
3430
|
}
|
|
3132
3431
|
}
|
|
@@ -3144,8 +3443,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3144
3443
|
fetchClient;
|
|
3145
3444
|
validateChecksums;
|
|
3146
3445
|
azureAccount;
|
|
3446
|
+
withRangeCoalescing;
|
|
3147
3447
|
basePath = "";
|
|
3148
|
-
constructor(session, fetchClient, validateChecksums, azureAccount) {
|
|
3448
|
+
constructor(session, fetchClient, validateChecksums, azureAccount, withRangeCoalescing) {
|
|
3149
3449
|
if (!(session instanceof ReadSession)) {
|
|
3150
3450
|
throw new Error(
|
|
3151
3451
|
"IcechunkStore constructor is private. Use IcechunkStore.open() instead."
|
|
@@ -3155,6 +3455,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3155
3455
|
this.fetchClient = fetchClient;
|
|
3156
3456
|
this.validateChecksums = validateChecksums ?? false;
|
|
3157
3457
|
this.azureAccount = azureAccount;
|
|
3458
|
+
this.withRangeCoalescing = withRangeCoalescing;
|
|
3158
3459
|
}
|
|
3159
3460
|
static async open(arg, options = {}) {
|
|
3160
3461
|
if (arg instanceof ReadSession) {
|
|
@@ -3162,7 +3463,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3162
3463
|
arg,
|
|
3163
3464
|
options.fetchClient,
|
|
3164
3465
|
options.validateChecksums,
|
|
3165
|
-
options.azureAccount
|
|
3466
|
+
options.azureAccount,
|
|
3467
|
+
options.withRangeCoalescing
|
|
3166
3468
|
);
|
|
3167
3469
|
}
|
|
3168
3470
|
const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
|
|
@@ -3190,7 +3492,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3190
3492
|
session,
|
|
3191
3493
|
options.fetchClient,
|
|
3192
3494
|
options.validateChecksums,
|
|
3193
|
-
options.azureAccount
|
|
3495
|
+
options.azureAccount,
|
|
3496
|
+
options.withRangeCoalescing
|
|
3194
3497
|
);
|
|
3195
3498
|
}
|
|
3196
3499
|
/**
|
|
@@ -3206,7 +3509,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3206
3509
|
* @returns Data bytes or undefined if not found
|
|
3207
3510
|
*/
|
|
3208
3511
|
async get(key, opts) {
|
|
3209
|
-
|
|
3512
|
+
opts?.signal?.throwIfAborted();
|
|
3210
3513
|
const parsed = parseZarrKey(key);
|
|
3211
3514
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3212
3515
|
try {
|
|
@@ -3218,11 +3521,15 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3218
3521
|
signal: opts?.signal,
|
|
3219
3522
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3220
3523
|
validateChecksums: this.validateChecksums,
|
|
3221
|
-
azureAccount: this.azureAccount
|
|
3524
|
+
azureAccount: this.azureAccount,
|
|
3525
|
+
...this.withRangeCoalescing && {
|
|
3526
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
3527
|
+
}
|
|
3222
3528
|
});
|
|
3223
3529
|
return chunk ?? void 0;
|
|
3224
|
-
} catch {
|
|
3225
|
-
return void 0;
|
|
3530
|
+
} catch (err) {
|
|
3531
|
+
if (err instanceof NotFoundError) return void 0;
|
|
3532
|
+
throw err;
|
|
3226
3533
|
}
|
|
3227
3534
|
}
|
|
3228
3535
|
/**
|
|
@@ -3238,7 +3545,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3238
3545
|
* @returns Data bytes or undefined if not found
|
|
3239
3546
|
*/
|
|
3240
3547
|
async getRange(key, range, opts) {
|
|
3241
|
-
|
|
3548
|
+
opts?.signal?.throwIfAborted();
|
|
3242
3549
|
const parsed = parseZarrKey(key);
|
|
3243
3550
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3244
3551
|
try {
|
|
@@ -3251,7 +3558,10 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3251
3558
|
signal: opts?.signal,
|
|
3252
3559
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3253
3560
|
validateChecksums: this.validateChecksums,
|
|
3254
|
-
azureAccount: this.azureAccount
|
|
3561
|
+
azureAccount: this.azureAccount,
|
|
3562
|
+
...this.withRangeCoalescing && {
|
|
3563
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
3564
|
+
}
|
|
3255
3565
|
}
|
|
3256
3566
|
);
|
|
3257
3567
|
return data2 ?? void 0;
|
|
@@ -3262,8 +3572,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3262
3572
|
return data.slice(-range.suffixLength);
|
|
3263
3573
|
}
|
|
3264
3574
|
return data.slice(range.offset, range.offset + range.length);
|
|
3265
|
-
} catch {
|
|
3266
|
-
return void 0;
|
|
3575
|
+
} catch (err) {
|
|
3576
|
+
if (err instanceof NotFoundError) return void 0;
|
|
3577
|
+
throw err;
|
|
3267
3578
|
}
|
|
3268
3579
|
}
|
|
3269
3580
|
/** Prepend basePath to a parsed path. */
|
|
@@ -3287,7 +3598,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3287
3598
|
this.session,
|
|
3288
3599
|
this.fetchClient,
|
|
3289
3600
|
this.validateChecksums,
|
|
3290
|
-
this.azureAccount
|
|
3601
|
+
this.azureAccount,
|
|
3602
|
+
this.withRangeCoalescing
|
|
3291
3603
|
);
|
|
3292
3604
|
const cleanPath = path.replace(/^\/+|\/+$/g, "");
|
|
3293
3605
|
scoped.basePath = this.basePath ? `${this.basePath}/${cleanPath}` : cleanPath;
|