icechunk-js 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -14
- package/dist/index.cjs +543 -231
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +110 -12
- package/dist/index.d.ts +110 -12
- package/dist/index.js +543 -231
- package/dist/index.js.map +1 -1
- package/package.json +10 -10
package/dist/index.js
CHANGED
|
@@ -18,12 +18,12 @@ var StorageError = class extends Error {
|
|
|
18
18
|
this.name = "StorageError";
|
|
19
19
|
}
|
|
20
20
|
};
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
}
|
|
21
|
+
function hasErrorName(error) {
|
|
22
|
+
return typeof error === "object" && error !== null && "name" in error;
|
|
23
|
+
}
|
|
24
|
+
function isAbortError(error) {
|
|
25
|
+
return hasErrorName(error) && error.name === "AbortError";
|
|
26
|
+
}
|
|
27
27
|
|
|
28
28
|
// src/format/constants.ts
|
|
29
29
|
var PATHS = {
|
|
@@ -145,6 +145,7 @@ function decodeObjectId8(str) {
|
|
|
145
145
|
|
|
146
146
|
// src/format/flatbuffers/repo-parser.ts
|
|
147
147
|
import { ByteBuffer } from "flatbuffers";
|
|
148
|
+
import * as flexbuffers from "flatbuffers/js/flexbuffers.js";
|
|
148
149
|
|
|
149
150
|
// src/format/flatbuffers/generated/metadata-item.ts
|
|
150
151
|
var MetadataItem = class _MetadataItem {
|
|
@@ -631,7 +632,36 @@ function parseRepo(data) {
|
|
|
631
632
|
);
|
|
632
633
|
}
|
|
633
634
|
const snapshotsLength = repo.snapshotsLength();
|
|
634
|
-
|
|
635
|
+
const virtualChunkContainers = parseVirtualChunkContainers(repo);
|
|
636
|
+
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
637
|
+
}
|
|
638
|
+
function parseVirtualChunkContainers(repo) {
|
|
639
|
+
const result = /* @__PURE__ */ new Map();
|
|
640
|
+
const configBytes = repo.configArray();
|
|
641
|
+
if (!configBytes || configBytes.length === 0) return result;
|
|
642
|
+
let config;
|
|
643
|
+
try {
|
|
644
|
+
const ab = configBytes.buffer.slice(
|
|
645
|
+
configBytes.byteOffset,
|
|
646
|
+
configBytes.byteOffset + configBytes.byteLength
|
|
647
|
+
);
|
|
648
|
+
config = flexbuffers.toObject(ab);
|
|
649
|
+
} catch {
|
|
650
|
+
return result;
|
|
651
|
+
}
|
|
652
|
+
if (!isRecord(config)) return result;
|
|
653
|
+
const containers = config.virtual_chunk_containers;
|
|
654
|
+
if (!isRecord(containers)) return result;
|
|
655
|
+
for (const container of Object.values(containers)) {
|
|
656
|
+
if (!isRecord(container)) continue;
|
|
657
|
+
const { name, url_prefix: urlPrefix } = container;
|
|
658
|
+
if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
|
|
659
|
+
result.set(name, urlPrefix);
|
|
660
|
+
}
|
|
661
|
+
return result;
|
|
662
|
+
}
|
|
663
|
+
function isRecord(value) {
|
|
664
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
635
665
|
}
|
|
636
666
|
function compareUtf8ByteOrder(aBytes, bBytes) {
|
|
637
667
|
const minLen = Math.min(aBytes.length, bBytes.length);
|
|
@@ -774,6 +804,85 @@ var LRUCache = class {
|
|
|
774
804
|
}
|
|
775
805
|
};
|
|
776
806
|
|
|
807
|
+
// src/cache/single-flight.ts
|
|
808
|
+
function singleFlight(cache) {
|
|
809
|
+
const pending = /* @__PURE__ */ new Map();
|
|
810
|
+
return {
|
|
811
|
+
load(key, fetcher, signal) {
|
|
812
|
+
if (signal?.aborted) return Promise.reject(makeAbortError());
|
|
813
|
+
const hit = cache.get(key);
|
|
814
|
+
if (hit !== void 0) return Promise.resolve(hit);
|
|
815
|
+
let entry = pending.get(key);
|
|
816
|
+
if (!entry) {
|
|
817
|
+
const controller = new AbortController();
|
|
818
|
+
const promise = invokeFetcher(fetcher, controller.signal).then((value) => {
|
|
819
|
+
cache.set(key, value);
|
|
820
|
+
return value;
|
|
821
|
+
}).finally(() => {
|
|
822
|
+
if (pending.get(key)?.promise === promise) pending.delete(key);
|
|
823
|
+
});
|
|
824
|
+
const fresh = { promise, controller, refCount: 0 };
|
|
825
|
+
pending.set(key, fresh);
|
|
826
|
+
entry = fresh;
|
|
827
|
+
}
|
|
828
|
+
const owned = entry;
|
|
829
|
+
owned.refCount++;
|
|
830
|
+
const release = () => {
|
|
831
|
+
owned.refCount--;
|
|
832
|
+
if (owned.refCount > 0) return;
|
|
833
|
+
if (pending.get(key) === owned) {
|
|
834
|
+
pending.delete(key);
|
|
835
|
+
owned.controller.abort();
|
|
836
|
+
}
|
|
837
|
+
};
|
|
838
|
+
if (!signal) {
|
|
839
|
+
return owned.promise.then(
|
|
840
|
+
(value) => {
|
|
841
|
+
release();
|
|
842
|
+
return value;
|
|
843
|
+
},
|
|
844
|
+
(error) => {
|
|
845
|
+
release();
|
|
846
|
+
throw error;
|
|
847
|
+
}
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
return new Promise((resolve, reject) => {
|
|
851
|
+
let done = false;
|
|
852
|
+
const finish = () => {
|
|
853
|
+
if (done) return false;
|
|
854
|
+
done = true;
|
|
855
|
+
signal.removeEventListener("abort", handleAbort);
|
|
856
|
+
release();
|
|
857
|
+
return true;
|
|
858
|
+
};
|
|
859
|
+
const handleAbort = () => {
|
|
860
|
+
if (finish()) reject(makeAbortError());
|
|
861
|
+
};
|
|
862
|
+
signal.addEventListener("abort", handleAbort, { once: true });
|
|
863
|
+
owned.promise.then(
|
|
864
|
+
(value) => {
|
|
865
|
+
if (finish()) resolve(value);
|
|
866
|
+
},
|
|
867
|
+
(error) => {
|
|
868
|
+
if (finish()) reject(error);
|
|
869
|
+
}
|
|
870
|
+
);
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
function invokeFetcher(fetcher, signal) {
|
|
876
|
+
try {
|
|
877
|
+
return fetcher(signal);
|
|
878
|
+
} catch (error) {
|
|
879
|
+
return Promise.reject(error);
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
function makeAbortError() {
|
|
883
|
+
return new DOMException("The operation was aborted.", "AbortError");
|
|
884
|
+
}
|
|
885
|
+
|
|
777
886
|
// src/format/flatbuffers/types.ts
|
|
778
887
|
function asObjectId12(bytes) {
|
|
779
888
|
if (bytes.length !== 12) {
|
|
@@ -1765,11 +1874,11 @@ function getChunkPayload(ref) {
|
|
|
1765
1874
|
|
|
1766
1875
|
// src/format/flatbuffers/metadata.ts
|
|
1767
1876
|
import { decode } from "@msgpack/msgpack";
|
|
1768
|
-
import * as
|
|
1877
|
+
import * as flexbuffers2 from "flatbuffers/js/flexbuffers.js";
|
|
1769
1878
|
function deserializeMetadata(items, specVersion) {
|
|
1770
1879
|
const result = /* @__PURE__ */ Object.create(null);
|
|
1771
1880
|
for (const item of items) {
|
|
1772
|
-
result[item.name] = specVersion === 1 /* V1_0 */ ? decode(item.value) :
|
|
1881
|
+
result[item.name] = specVersion === 1 /* V1_0 */ ? decode(item.value) : flexbuffers2.toObject(
|
|
1773
1882
|
item.value.buffer.slice(
|
|
1774
1883
|
item.value.byteOffset,
|
|
1775
1884
|
item.value.byteOffset + item.value.byteLength
|
|
@@ -2101,17 +2210,219 @@ function parseTransactionLog(data) {
|
|
|
2101
2210
|
};
|
|
2102
2211
|
}
|
|
2103
2212
|
|
|
2213
|
+
// src/reader/range-coalescer.ts
|
|
2214
|
+
function expectedRangeLength(range) {
|
|
2215
|
+
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2216
|
+
}
|
|
2217
|
+
function makeUrlStore(opts) {
|
|
2218
|
+
const { url, fetchClient, conditionalHeaders } = opts;
|
|
2219
|
+
async function doFetch(init) {
|
|
2220
|
+
return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
|
|
2221
|
+
}
|
|
2222
|
+
return {
|
|
2223
|
+
async get() {
|
|
2224
|
+
throw new Error(
|
|
2225
|
+
`Virtual chunk URL store for ${url} only supports ranged reads`
|
|
2226
|
+
);
|
|
2227
|
+
},
|
|
2228
|
+
async getRange(_key, range, options) {
|
|
2229
|
+
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2230
|
+
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2231
|
+
const response = await doFetch({ headers, signal: options?.signal });
|
|
2232
|
+
if (response.status === 412) {
|
|
2233
|
+
throw new Error(
|
|
2234
|
+
`Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2235
|
+
);
|
|
2236
|
+
}
|
|
2237
|
+
if (response.status !== 200 && response.status !== 206) {
|
|
2238
|
+
throw new Error(
|
|
2239
|
+
`Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
|
|
2240
|
+
);
|
|
2241
|
+
}
|
|
2242
|
+
const data = new Uint8Array(await response.arrayBuffer());
|
|
2243
|
+
if (response.status === 206) {
|
|
2244
|
+
const expected = expectedRangeLength(range);
|
|
2245
|
+
if (data.length === expected) return data;
|
|
2246
|
+
throw new Error(
|
|
2247
|
+
`Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
|
|
2248
|
+
);
|
|
2249
|
+
}
|
|
2250
|
+
if ("offset" in range) {
|
|
2251
|
+
const end = range.offset + range.length;
|
|
2252
|
+
if (data.length >= end) return data.slice(range.offset, end);
|
|
2253
|
+
throw new Error(
|
|
2254
|
+
`Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2255
|
+
);
|
|
2256
|
+
}
|
|
2257
|
+
if (data.length >= range.suffixLength) {
|
|
2258
|
+
return data.slice(data.length - range.suffixLength);
|
|
2259
|
+
}
|
|
2260
|
+
throw new Error(
|
|
2261
|
+
`Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2262
|
+
);
|
|
2263
|
+
}
|
|
2264
|
+
};
|
|
2265
|
+
}
|
|
2266
|
+
function makeStorageStore(storage) {
|
|
2267
|
+
return {
|
|
2268
|
+
async get(key, options) {
|
|
2269
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2270
|
+
return storage.getObject(key, void 0, storageOptions);
|
|
2271
|
+
},
|
|
2272
|
+
async getRange(key, range, options) {
|
|
2273
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2274
|
+
if ("suffixLength" in range) {
|
|
2275
|
+
throw new Error(
|
|
2276
|
+
`Storage suffix ranges are not supported for ${key}; convert suffixLength to offset/length before reading`
|
|
2277
|
+
);
|
|
2278
|
+
}
|
|
2279
|
+
const storageRange = {
|
|
2280
|
+
start: range.offset,
|
|
2281
|
+
end: range.offset + range.length
|
|
2282
|
+
};
|
|
2283
|
+
const data = await storage.getObject(key, storageRange, storageOptions);
|
|
2284
|
+
if (data.length === range.length) return data;
|
|
2285
|
+
if (data.length >= storageRange.end) {
|
|
2286
|
+
return data.slice(storageRange.start, storageRange.end);
|
|
2287
|
+
}
|
|
2288
|
+
throw new Error(
|
|
2289
|
+
`Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
|
|
2290
|
+
);
|
|
2291
|
+
}
|
|
2292
|
+
};
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2104
2295
|
// src/reader/session.ts
|
|
2296
|
+
var RANGE_COALESCE_SIZE = 32 * 1024;
|
|
2297
|
+
var RANGE_STORE_CACHE_SIZE = 256;
|
|
2298
|
+
function makeRangeStoreCacheKey(parts) {
|
|
2299
|
+
return JSON.stringify(parts);
|
|
2300
|
+
}
|
|
2105
2301
|
var ReadSession = class _ReadSession {
|
|
2106
2302
|
storage;
|
|
2107
2303
|
snapshot;
|
|
2108
2304
|
specVersion;
|
|
2109
2305
|
manifestCache;
|
|
2110
|
-
|
|
2306
|
+
/**
|
|
2307
|
+
* Single-flight loader over `manifestCache`: concurrent misses for the
|
|
2308
|
+
* same manifest share one fetch instead of racing N identical GETs.
|
|
2309
|
+
* Critical for fan-out reads that all need the same manifest (e.g. the
|
|
2310
|
+
* many parallel chunk reads issued when a renderer crosses into a new
|
|
2311
|
+
* pyramid level).
|
|
2312
|
+
*/
|
|
2313
|
+
manifestLoader;
|
|
2314
|
+
/**
|
|
2315
|
+
* Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
|
|
2316
|
+
* The cache key includes request-option identities that must not share one
|
|
2317
|
+
* coalescing queue, notably virtual fetch clients and checksum headers.
|
|
2318
|
+
*
|
|
2319
|
+
* Promise slots are inserted synchronously on first use, so
|
|
2320
|
+
* concurrent requests for the same partition share one coalescing window
|
|
2321
|
+
* instead of racing to create parallel stores.
|
|
2322
|
+
*/
|
|
2323
|
+
rangeStores;
|
|
2324
|
+
nativeStore;
|
|
2325
|
+
fetchClientIds;
|
|
2326
|
+
nextFetchClientId = 1;
|
|
2327
|
+
rangeCoalescerIds;
|
|
2328
|
+
nextRangeCoalescerId = 1;
|
|
2329
|
+
/** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
|
|
2330
|
+
virtualChunkContainers;
|
|
2331
|
+
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
2111
2332
|
this.storage = storage;
|
|
2112
2333
|
this.snapshot = snapshot;
|
|
2113
2334
|
this.specVersion = specVersion;
|
|
2114
2335
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
2336
|
+
this.manifestLoader = singleFlight(this.manifestCache);
|
|
2337
|
+
this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
|
|
2338
|
+
}
|
|
2339
|
+
getFetchClientKey(fetchClient) {
|
|
2340
|
+
if (!fetchClient) return "default";
|
|
2341
|
+
if (!this.fetchClientIds) this.fetchClientIds = /* @__PURE__ */ new WeakMap();
|
|
2342
|
+
let id = this.fetchClientIds.get(fetchClient);
|
|
2343
|
+
if (id === void 0) {
|
|
2344
|
+
id = this.nextFetchClientId;
|
|
2345
|
+
this.nextFetchClientId = id + 1;
|
|
2346
|
+
this.fetchClientIds.set(fetchClient, id);
|
|
2347
|
+
}
|
|
2348
|
+
return String(id);
|
|
2349
|
+
}
|
|
2350
|
+
getRangeCoalescerKey(withRangeCoalescing) {
|
|
2351
|
+
if (!this.rangeCoalescerIds) this.rangeCoalescerIds = /* @__PURE__ */ new WeakMap();
|
|
2352
|
+
let id = this.rangeCoalescerIds.get(withRangeCoalescing);
|
|
2353
|
+
if (id === void 0) {
|
|
2354
|
+
id = this.nextRangeCoalescerId;
|
|
2355
|
+
this.nextRangeCoalescerId = id + 1;
|
|
2356
|
+
this.rangeCoalescerIds.set(withRangeCoalescing, id);
|
|
2357
|
+
}
|
|
2358
|
+
return String(id);
|
|
2359
|
+
}
|
|
2360
|
+
getRangeStore(partitionKey, createStore, withRangeCoalescing) {
|
|
2361
|
+
if (!this.rangeStores) {
|
|
2362
|
+
this.rangeStores = new LRUCache(RANGE_STORE_CACHE_SIZE);
|
|
2363
|
+
}
|
|
2364
|
+
const stores = this.rangeStores;
|
|
2365
|
+
const cacheKey = makeRangeStoreCacheKey([
|
|
2366
|
+
...partitionKey,
|
|
2367
|
+
["coalescer", this.getRangeCoalescerKey(withRangeCoalescing)]
|
|
2368
|
+
]);
|
|
2369
|
+
const cached = stores.get(cacheKey);
|
|
2370
|
+
if (cached) return cached;
|
|
2371
|
+
const raw = createStore();
|
|
2372
|
+
const promise = Promise.resolve().then(
|
|
2373
|
+
() => withRangeCoalescing(raw, { coalesceSize: RANGE_COALESCE_SIZE })
|
|
2374
|
+
).catch((error) => {
|
|
2375
|
+
if (stores.get(cacheKey) === promise) stores.delete(cacheKey);
|
|
2376
|
+
throw error;
|
|
2377
|
+
});
|
|
2378
|
+
stores.set(cacheKey, promise);
|
|
2379
|
+
return promise;
|
|
2380
|
+
}
|
|
2381
|
+
getNativeStore(options) {
|
|
2382
|
+
if (!this.nativeStore) {
|
|
2383
|
+
this.nativeStore = makeStorageStore(this.storage);
|
|
2384
|
+
}
|
|
2385
|
+
const raw = this.nativeStore;
|
|
2386
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
2387
|
+
if (!withRangeCoalescing) {
|
|
2388
|
+
return Promise.resolve(raw);
|
|
2389
|
+
}
|
|
2390
|
+
return this.getRangeStore(["native"], () => raw, withRangeCoalescing);
|
|
2391
|
+
}
|
|
2392
|
+
getVirtualStoreForPayload(httpUrl, payload, options) {
|
|
2393
|
+
const validate = !!options?.validateChecksums;
|
|
2394
|
+
let conditionalHeaders;
|
|
2395
|
+
if (validate) {
|
|
2396
|
+
conditionalHeaders = {};
|
|
2397
|
+
if (payload.checksumEtag) {
|
|
2398
|
+
conditionalHeaders["If-Match"] = payload.checksumEtag;
|
|
2399
|
+
}
|
|
2400
|
+
if (payload.checksumLastModified > 0) {
|
|
2401
|
+
conditionalHeaders["If-Unmodified-Since"] = new Date(
|
|
2402
|
+
payload.checksumLastModified * 1e3
|
|
2403
|
+
).toUTCString();
|
|
2404
|
+
}
|
|
2405
|
+
}
|
|
2406
|
+
const checksumKey = validate ? ["checked", payload.checksumEtag ?? "", payload.checksumLastModified] : ["unchecked"];
|
|
2407
|
+
const createStore = () => makeUrlStore({
|
|
2408
|
+
url: httpUrl,
|
|
2409
|
+
fetchClient: options?.fetchClient,
|
|
2410
|
+
conditionalHeaders
|
|
2411
|
+
});
|
|
2412
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
2413
|
+
if (!withRangeCoalescing) {
|
|
2414
|
+
return Promise.resolve(createStore());
|
|
2415
|
+
}
|
|
2416
|
+
return this.getRangeStore(
|
|
2417
|
+
[
|
|
2418
|
+
"virtual",
|
|
2419
|
+
httpUrl,
|
|
2420
|
+
["fetch", this.getFetchClientKey(options?.fetchClient)],
|
|
2421
|
+
checksumKey
|
|
2422
|
+
],
|
|
2423
|
+
createStore,
|
|
2424
|
+
withRangeCoalescing
|
|
2425
|
+
);
|
|
2115
2426
|
}
|
|
2116
2427
|
/**
|
|
2117
2428
|
* Open a read session for a specific snapshot.
|
|
@@ -2131,7 +2442,8 @@ var ReadSession = class _ReadSession {
|
|
|
2131
2442
|
storage,
|
|
2132
2443
|
snapshot,
|
|
2133
2444
|
specVersion,
|
|
2134
|
-
options?.maxManifestCacheSize
|
|
2445
|
+
options?.maxManifestCacheSize,
|
|
2446
|
+
options?.virtualChunkContainers
|
|
2135
2447
|
);
|
|
2136
2448
|
}
|
|
2137
2449
|
/** Load and parse a snapshot from storage */
|
|
@@ -2149,22 +2461,40 @@ var ReadSession = class _ReadSession {
|
|
|
2149
2461
|
specVersion: header.specVersion
|
|
2150
2462
|
};
|
|
2151
2463
|
}
|
|
2152
|
-
/**
|
|
2153
|
-
|
|
2464
|
+
/**
|
|
2465
|
+
* Load and parse a manifest from storage.
|
|
2466
|
+
*
|
|
2467
|
+
* Concurrent calls for the same manifest are coalesced through
|
|
2468
|
+
* `manifestLoader` so a fan-out of chunk reads — the typical case
|
|
2469
|
+
* when many tiles cross into a new pyramid level at once — issues one
|
|
2470
|
+
* HTTP GET instead of one per caller. A caller's `signal` rejects only
|
|
2471
|
+
* that caller's await; the shared fetch is aborted only once every
|
|
2472
|
+
* active waiter has aborted.
|
|
2473
|
+
*/
|
|
2474
|
+
loadManifest(manifestId, options) {
|
|
2154
2475
|
const idStr = encodeObjectId12(manifestId);
|
|
2155
|
-
|
|
2156
|
-
|
|
2476
|
+
return this.manifestLoader.load(
|
|
2477
|
+
idStr,
|
|
2478
|
+
(signal) => this.fetchManifest(idStr, signal),
|
|
2479
|
+
options?.signal
|
|
2480
|
+
);
|
|
2481
|
+
}
|
|
2482
|
+
/**
|
|
2483
|
+
* Fetch and parse a single manifest. The `signal` here is the
|
|
2484
|
+
* single-flight loader's own signal — it fires only when every waiter
|
|
2485
|
+
* has aborted, so a hung manifest fetch doesn't trap later callers
|
|
2486
|
+
* behind it.
|
|
2487
|
+
*/
|
|
2488
|
+
async fetchManifest(idStr, signal) {
|
|
2157
2489
|
const path = getManifestPath(idStr);
|
|
2158
|
-
const data = await this.storage.getObject(path, void 0,
|
|
2490
|
+
const data = await this.storage.getObject(path, void 0, { signal });
|
|
2159
2491
|
const header = parseHeader(data);
|
|
2160
2492
|
validateFileType(header, 2 /* Manifest */);
|
|
2161
2493
|
let flatbufferData = getDataAfterHeader(data);
|
|
2162
2494
|
if (header.compression === 1 /* Zstd */) {
|
|
2163
2495
|
flatbufferData = decompress2(flatbufferData);
|
|
2164
2496
|
}
|
|
2165
|
-
|
|
2166
|
-
this.manifestCache.set(idStr, manifest);
|
|
2167
|
-
return manifest;
|
|
2497
|
+
return parseManifest(flatbufferData);
|
|
2168
2498
|
}
|
|
2169
2499
|
/**
|
|
2170
2500
|
* Get the snapshot ID.
|
|
@@ -2295,26 +2625,25 @@ var ReadSession = class _ReadSession {
|
|
|
2295
2625
|
* @returns Chunk data bytes or null if not found
|
|
2296
2626
|
*/
|
|
2297
2627
|
async getChunk(path, coords, options) {
|
|
2298
|
-
|
|
2628
|
+
options?.signal?.throwIfAborted();
|
|
2629
|
+
const requestOptions = toRequestOptions(options);
|
|
2299
2630
|
const node = this.getNode(path);
|
|
2300
2631
|
if (!node || node.nodeData.type !== "array") {
|
|
2301
2632
|
return null;
|
|
2302
2633
|
}
|
|
2303
2634
|
const arrayData = node.nodeData;
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
continue;
|
|
2308
|
-
}
|
|
2309
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2310
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2311
|
-
if (!chunkRef) continue;
|
|
2312
|
-
const payload = getChunkPayload(chunkRef);
|
|
2313
|
-
return this.fetchChunkPayload(payload, options);
|
|
2635
|
+
for (const manifestRef of arrayData.manifests) {
|
|
2636
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
2637
|
+
continue;
|
|
2314
2638
|
}
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2639
|
+
const manifest = await this.loadManifest(
|
|
2640
|
+
manifestRef.objectId,
|
|
2641
|
+
requestOptions
|
|
2642
|
+
);
|
|
2643
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2644
|
+
if (!chunkRef) continue;
|
|
2645
|
+
const payload = getChunkPayload(chunkRef);
|
|
2646
|
+
return this.fetchChunkPayload(payload, options);
|
|
2318
2647
|
}
|
|
2319
2648
|
return null;
|
|
2320
2649
|
}
|
|
@@ -2332,26 +2661,25 @@ var ReadSession = class _ReadSession {
|
|
|
2332
2661
|
* @returns Chunk data bytes or null if not found
|
|
2333
2662
|
*/
|
|
2334
2663
|
async getChunkRange(path, coords, range, options) {
|
|
2335
|
-
|
|
2664
|
+
options?.signal?.throwIfAborted();
|
|
2665
|
+
const requestOptions = toRequestOptions(options);
|
|
2336
2666
|
const node = this.getNode(path);
|
|
2337
2667
|
if (!node || node.nodeData.type !== "array") {
|
|
2338
2668
|
return null;
|
|
2339
2669
|
}
|
|
2340
2670
|
const arrayData = node.nodeData;
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
continue;
|
|
2345
|
-
}
|
|
2346
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2347
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2348
|
-
if (!chunkRef) continue;
|
|
2349
|
-
const payload = getChunkPayload(chunkRef);
|
|
2350
|
-
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2671
|
+
for (const manifestRef of arrayData.manifests) {
|
|
2672
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
2673
|
+
continue;
|
|
2351
2674
|
}
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2675
|
+
const manifest = await this.loadManifest(
|
|
2676
|
+
manifestRef.objectId,
|
|
2677
|
+
requestOptions
|
|
2678
|
+
);
|
|
2679
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2680
|
+
if (!chunkRef) continue;
|
|
2681
|
+
const payload = getChunkPayload(chunkRef);
|
|
2682
|
+
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2355
2683
|
}
|
|
2356
2684
|
return null;
|
|
2357
2685
|
}
|
|
@@ -2373,77 +2701,54 @@ var ReadSession = class _ReadSession {
|
|
|
2373
2701
|
return payload.data;
|
|
2374
2702
|
case "native": {
|
|
2375
2703
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2376
|
-
const
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2704
|
+
const requestedStart = payload.offset;
|
|
2705
|
+
const requestedEnd = payload.offset + payload.length;
|
|
2706
|
+
const store = await this.getNativeStore(options);
|
|
2707
|
+
const data = await store.getRange(
|
|
2708
|
+
path,
|
|
2709
|
+
{ offset: requestedStart, length: payload.length },
|
|
2710
|
+
{ signal: options?.signal }
|
|
2711
|
+
);
|
|
2712
|
+
if (!data) {
|
|
2713
|
+
throw new Error(
|
|
2714
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
2715
|
+
);
|
|
2384
2716
|
}
|
|
2717
|
+
if (data.length === payload.length) return data;
|
|
2385
2718
|
throw new Error(
|
|
2386
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
2719
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${payload.length} bytes`
|
|
2387
2720
|
);
|
|
2388
2721
|
}
|
|
2389
2722
|
case "virtual": {
|
|
2390
|
-
const
|
|
2723
|
+
const absoluteLocation = expandVccUrl(
|
|
2391
2724
|
payload.location,
|
|
2725
|
+
this.virtualChunkContainers
|
|
2726
|
+
);
|
|
2727
|
+
const httpUrl = translateToHttpUrl(
|
|
2728
|
+
absoluteLocation,
|
|
2392
2729
|
options?.azureAccount
|
|
2393
2730
|
);
|
|
2394
|
-
const
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
}
|
|
2406
|
-
}
|
|
2407
|
-
const fetchInit = {
|
|
2408
|
-
headers,
|
|
2409
|
-
signal: options?.signal
|
|
2410
|
-
};
|
|
2411
|
-
let response;
|
|
2412
|
-
try {
|
|
2413
|
-
const client = options?.fetchClient;
|
|
2414
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2415
|
-
} catch (error) {
|
|
2416
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2417
|
-
throw new AbortError();
|
|
2418
|
-
}
|
|
2419
|
-
throw error;
|
|
2420
|
-
}
|
|
2421
|
-
if (response.status === 412) {
|
|
2731
|
+
const store = await this.getVirtualStoreForPayload(
|
|
2732
|
+
httpUrl,
|
|
2733
|
+
payload,
|
|
2734
|
+
options
|
|
2735
|
+
);
|
|
2736
|
+
const data = await store.getRange(
|
|
2737
|
+
"/",
|
|
2738
|
+
{ offset: payload.offset, length: payload.length },
|
|
2739
|
+
{ signal: options?.signal }
|
|
2740
|
+
);
|
|
2741
|
+
if (!data) {
|
|
2422
2742
|
throw new Error(
|
|
2423
|
-
`
|
|
2743
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2424
2744
|
);
|
|
2425
2745
|
}
|
|
2426
|
-
if (
|
|
2746
|
+
if (data.length !== payload.length) {
|
|
2427
2747
|
throw new Error(
|
|
2428
|
-
`
|
|
2748
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2429
2749
|
);
|
|
2430
2750
|
}
|
|
2431
|
-
|
|
2432
|
-
if (response.status === 206) {
|
|
2433
|
-
if (data.length !== payload.length) {
|
|
2434
|
-
throw new Error(
|
|
2435
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2436
|
-
);
|
|
2437
|
-
}
|
|
2438
|
-
return data;
|
|
2439
|
-
}
|
|
2440
|
-
const absoluteEnd = payload.offset + payload.length;
|
|
2441
|
-
if (data.length >= absoluteEnd) {
|
|
2442
|
-
return data.slice(payload.offset, absoluteEnd);
|
|
2443
|
-
}
|
|
2444
|
-
throw new Error(
|
|
2445
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2446
|
-
);
|
|
2751
|
+
return data;
|
|
2447
2752
|
}
|
|
2448
2753
|
}
|
|
2449
2754
|
}
|
|
@@ -2463,80 +2768,57 @@ var ReadSession = class _ReadSession {
|
|
|
2463
2768
|
return payload.data.slice(rangeStart, rangeEnd);
|
|
2464
2769
|
case "native": {
|
|
2465
2770
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2466
|
-
const
|
|
2467
|
-
start: payload.offset + rangeStart,
|
|
2468
|
-
end: payload.offset + rangeEnd
|
|
2469
|
-
};
|
|
2771
|
+
const requestedStart = payload.offset + rangeStart;
|
|
2470
2772
|
const expectedSize = rangeEnd - rangeStart;
|
|
2471
|
-
const
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2773
|
+
const requestedEnd = requestedStart + expectedSize;
|
|
2774
|
+
const store = await this.getNativeStore(options);
|
|
2775
|
+
const data = await store.getRange(
|
|
2776
|
+
path,
|
|
2777
|
+
{ offset: requestedStart, length: expectedSize },
|
|
2778
|
+
{ signal: options?.signal }
|
|
2779
|
+
);
|
|
2780
|
+
if (!data) {
|
|
2781
|
+
throw new Error(
|
|
2782
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
2783
|
+
);
|
|
2475
2784
|
}
|
|
2785
|
+
if (data.length === expectedSize) return data;
|
|
2476
2786
|
throw new Error(
|
|
2477
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
2787
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${expectedSize} bytes`
|
|
2478
2788
|
);
|
|
2479
2789
|
}
|
|
2480
2790
|
case "virtual": {
|
|
2481
2791
|
const absoluteStart = payload.offset + rangeStart;
|
|
2482
|
-
const
|
|
2483
|
-
const
|
|
2792
|
+
const expectedSize = rangeEnd - rangeStart;
|
|
2793
|
+
const absoluteLocation = expandVccUrl(
|
|
2484
2794
|
payload.location,
|
|
2795
|
+
this.virtualChunkContainers
|
|
2796
|
+
);
|
|
2797
|
+
const httpUrl = translateToHttpUrl(
|
|
2798
|
+
absoluteLocation,
|
|
2485
2799
|
options?.azureAccount
|
|
2486
2800
|
);
|
|
2487
|
-
const
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
}
|
|
2499
|
-
}
|
|
2500
|
-
const fetchInit = {
|
|
2501
|
-
headers,
|
|
2502
|
-
signal: options?.signal
|
|
2503
|
-
};
|
|
2504
|
-
let response;
|
|
2505
|
-
try {
|
|
2506
|
-
const client = options?.fetchClient;
|
|
2507
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2508
|
-
} catch (error) {
|
|
2509
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2510
|
-
throw new AbortError();
|
|
2511
|
-
}
|
|
2512
|
-
throw error;
|
|
2513
|
-
}
|
|
2514
|
-
if (response.status === 412) {
|
|
2801
|
+
const store = await this.getVirtualStoreForPayload(
|
|
2802
|
+
httpUrl,
|
|
2803
|
+
payload,
|
|
2804
|
+
options
|
|
2805
|
+
);
|
|
2806
|
+
const data = await store.getRange(
|
|
2807
|
+
"/",
|
|
2808
|
+
{ offset: absoluteStart, length: expectedSize },
|
|
2809
|
+
{ signal: options?.signal }
|
|
2810
|
+
);
|
|
2811
|
+
if (!data) {
|
|
2515
2812
|
throw new Error(
|
|
2516
|
-
`
|
|
2813
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2517
2814
|
);
|
|
2518
2815
|
}
|
|
2519
|
-
if (
|
|
2816
|
+
if (data.length !== expectedSize) {
|
|
2520
2817
|
throw new Error(
|
|
2521
|
-
`
|
|
2818
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2522
2819
|
);
|
|
2523
2820
|
}
|
|
2524
|
-
|
|
2525
|
-
const data = new Uint8Array(await response.arrayBuffer());
|
|
2526
|
-
if (response.status === 206) {
|
|
2527
|
-
if (data.length !== expectedSize) {
|
|
2528
|
-
throw new Error(
|
|
2529
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2530
|
-
);
|
|
2531
|
-
}
|
|
2532
|
-
return data;
|
|
2533
|
-
}
|
|
2534
|
-
if (data.length >= absoluteEnd) {
|
|
2535
|
-
return data.slice(absoluteStart, absoluteEnd);
|
|
2536
|
-
}
|
|
2537
|
-
throw new Error(
|
|
2538
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2539
|
-
);
|
|
2821
|
+
return data;
|
|
2540
2822
|
}
|
|
2541
2823
|
}
|
|
2542
2824
|
}
|
|
@@ -2560,6 +2842,19 @@ var ReadSession = class _ReadSession {
|
|
|
2560
2842
|
}
|
|
2561
2843
|
};
|
|
2562
2844
|
var utf8Encoder = new TextEncoder();
|
|
2845
|
+
function toRequestOptions(options) {
|
|
2846
|
+
if (!options) return void 0;
|
|
2847
|
+
return {
|
|
2848
|
+
...options.signal && { signal: options.signal },
|
|
2849
|
+
...options.fetchClient && { fetchClient: options.fetchClient },
|
|
2850
|
+
...options.validateChecksums !== void 0 && {
|
|
2851
|
+
validateChecksums: options.validateChecksums
|
|
2852
|
+
},
|
|
2853
|
+
...options.azureAccount !== void 0 && {
|
|
2854
|
+
azureAccount: options.azureAccount
|
|
2855
|
+
}
|
|
2856
|
+
};
|
|
2857
|
+
}
|
|
2563
2858
|
function compareUtf8Bytes(a, b) {
|
|
2564
2859
|
const bytesA = utf8Encoder.encode(a);
|
|
2565
2860
|
const bytesB = utf8Encoder.encode(b);
|
|
@@ -2571,6 +2866,28 @@ function compareUtf8Bytes(a, b) {
|
|
|
2571
2866
|
}
|
|
2572
2867
|
return bytesA.length - bytesB.length;
|
|
2573
2868
|
}
|
|
2869
|
+
var VCC_SCHEME = "vcc://";
|
|
2870
|
+
function expandVccUrl(location, containers) {
|
|
2871
|
+
if (!location.startsWith(VCC_SCHEME)) return location;
|
|
2872
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
2873
|
+
const slash = rest.indexOf("/");
|
|
2874
|
+
if (slash === -1) {
|
|
2875
|
+
throw new Error(
|
|
2876
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
2877
|
+
);
|
|
2878
|
+
}
|
|
2879
|
+
const name = rest.slice(0, slash);
|
|
2880
|
+
const relativePath = rest.slice(slash + 1);
|
|
2881
|
+
const urlPrefix = containers.get(name);
|
|
2882
|
+
if (urlPrefix === void 0) {
|
|
2883
|
+
if (containers.size === 0) return location;
|
|
2884
|
+
throw new Error(
|
|
2885
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
2886
|
+
);
|
|
2887
|
+
}
|
|
2888
|
+
const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
|
|
2889
|
+
return normalizedPrefix + relativePath;
|
|
2890
|
+
}
|
|
2574
2891
|
function translateToHttpUrl(url, azureAccount) {
|
|
2575
2892
|
if (url.startsWith("s3://")) {
|
|
2576
2893
|
const rest = url.slice(5);
|
|
@@ -2646,7 +2963,7 @@ var Repository = class _Repository {
|
|
|
2646
2963
|
* @param options - Optional request options (signal for cancellation)
|
|
2647
2964
|
* @returns ParsedRepo if v2 format, null if v1 format
|
|
2648
2965
|
* @throws Error if repo file exists but fails to parse
|
|
2649
|
-
* @throws AbortError if the operation was aborted
|
|
2966
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
2650
2967
|
*/
|
|
2651
2968
|
async loadRepoInfo(options) {
|
|
2652
2969
|
if (this.repoInfoAttempted) {
|
|
@@ -2662,7 +2979,7 @@ var Repository = class _Repository {
|
|
|
2662
2979
|
this.repoInfo = parseRepo(data);
|
|
2663
2980
|
return this.repoInfo;
|
|
2664
2981
|
} catch (error) {
|
|
2665
|
-
if (error
|
|
2982
|
+
if (isAbortError(error)) {
|
|
2666
2983
|
this.repoInfoAttempted = false;
|
|
2667
2984
|
throw error;
|
|
2668
2985
|
}
|
|
@@ -2727,7 +3044,7 @@ var Repository = class _Repository {
|
|
|
2727
3044
|
}
|
|
2728
3045
|
}
|
|
2729
3046
|
} catch (error) {
|
|
2730
|
-
if (error
|
|
3047
|
+
if (isAbortError(error)) {
|
|
2731
3048
|
throw error;
|
|
2732
3049
|
}
|
|
2733
3050
|
if (legacyPath) {
|
|
@@ -2743,7 +3060,7 @@ var Repository = class _Repository {
|
|
|
2743
3060
|
* version has the highest filename. Deletion tombstones are checked.
|
|
2744
3061
|
*
|
|
2745
3062
|
* When listing is not supported (e.g., HTTP storage), only the legacy
|
|
2746
|
-
* ref.json path is
|
|
3063
|
+
* ref.json path is returned (caller must handle NotFoundError on read).
|
|
2747
3064
|
*
|
|
2748
3065
|
* @param dirPrefix - Directory prefix to search
|
|
2749
3066
|
* @param legacyPath - Optional legacy ref.json path to try if listing fails
|
|
@@ -2760,7 +3077,7 @@ var Repository = class _Repository {
|
|
|
2760
3077
|
}
|
|
2761
3078
|
}
|
|
2762
3079
|
} catch (error) {
|
|
2763
|
-
if (error
|
|
3080
|
+
if (isAbortError(error)) {
|
|
2764
3081
|
throw error;
|
|
2765
3082
|
}
|
|
2766
3083
|
return legacyPath || null;
|
|
@@ -2779,7 +3096,9 @@ var Repository = class _Repository {
|
|
|
2779
3096
|
/**
|
|
2780
3097
|
* List all branches in the repository.
|
|
2781
3098
|
*
|
|
2782
|
-
*
|
|
3099
|
+
* Unlike tags, branches in icechunk v1 have no tombstone mechanism —
|
|
3100
|
+
* deletion removes the ref file outright. A branch is present iff it has
|
|
3101
|
+
* at least one ref file in its directory.
|
|
2783
3102
|
*
|
|
2784
3103
|
* @returns Array of branch names
|
|
2785
3104
|
*/
|
|
@@ -2788,49 +3107,23 @@ var Repository = class _Repository {
|
|
|
2788
3107
|
if (repoInfo) {
|
|
2789
3108
|
return listBranchesFromRepo(repoInfo);
|
|
2790
3109
|
}
|
|
2791
|
-
const
|
|
3110
|
+
const branches = /* @__PURE__ */ new Set();
|
|
2792
3111
|
try {
|
|
2793
3112
|
for await (const path of this.storage.listPrefix(
|
|
2794
3113
|
`${PATHS.REFS}/branch.`
|
|
2795
3114
|
)) {
|
|
2796
|
-
const
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
);
|
|
2800
|
-
if (deletedMatch) {
|
|
2801
|
-
const branchName = deletedMatch[1];
|
|
2802
|
-
const refFile = `refs/branch.${branchName}/${deletedMatch[2]}`;
|
|
2803
|
-
if (!branchFiles.has(branchName)) {
|
|
2804
|
-
branchFiles.set(branchName, {
|
|
2805
|
-
jsonFiles: [],
|
|
2806
|
-
deletedFiles: /* @__PURE__ */ new Set()
|
|
2807
|
-
});
|
|
2808
|
-
}
|
|
2809
|
-
branchFiles.get(branchName).deletedFiles.add(refFile);
|
|
2810
|
-
} else if (jsonMatch) {
|
|
2811
|
-
const branchName = jsonMatch[1];
|
|
2812
|
-
if (!branchFiles.has(branchName)) {
|
|
2813
|
-
branchFiles.set(branchName, {
|
|
2814
|
-
jsonFiles: [],
|
|
2815
|
-
deletedFiles: /* @__PURE__ */ new Set()
|
|
2816
|
-
});
|
|
2817
|
-
}
|
|
2818
|
-
branchFiles.get(branchName).jsonFiles.push(path);
|
|
3115
|
+
const match = path.match(/^refs\/branch\.(.+)\/([^/]+\.json)$/);
|
|
3116
|
+
if (match) {
|
|
3117
|
+
branches.add(match[1]);
|
|
2819
3118
|
}
|
|
2820
3119
|
}
|
|
2821
|
-
} catch {
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
const result = [];
|
|
2825
|
-
for (const [branchName, { jsonFiles, deletedFiles }] of branchFiles) {
|
|
2826
|
-
if (jsonFiles.length === 0) continue;
|
|
2827
|
-
jsonFiles.sort();
|
|
2828
|
-
const latestRef = jsonFiles[jsonFiles.length - 1];
|
|
2829
|
-
if (!deletedFiles.has(latestRef)) {
|
|
2830
|
-
result.push(branchName);
|
|
3120
|
+
} catch (error) {
|
|
3121
|
+
if (isAbortError(error)) {
|
|
3122
|
+
throw error;
|
|
2831
3123
|
}
|
|
3124
|
+
throw new Error("Cannot list branches: storage does not support listing");
|
|
2832
3125
|
}
|
|
2833
|
-
return
|
|
3126
|
+
return [...branches];
|
|
2834
3127
|
}
|
|
2835
3128
|
/**
|
|
2836
3129
|
* List all tags in the repository.
|
|
@@ -2866,7 +3159,10 @@ var Repository = class _Repository {
|
|
|
2866
3159
|
tagFiles.get(tagName).jsonFiles.push(path);
|
|
2867
3160
|
}
|
|
2868
3161
|
}
|
|
2869
|
-
} catch {
|
|
3162
|
+
} catch (error) {
|
|
3163
|
+
if (isAbortError(error)) {
|
|
3164
|
+
throw error;
|
|
3165
|
+
}
|
|
2870
3166
|
throw new Error("Cannot list tags: storage does not support listing");
|
|
2871
3167
|
}
|
|
2872
3168
|
const result = [];
|
|
@@ -2894,7 +3190,10 @@ var Repository = class _Repository {
|
|
|
2894
3190
|
if (!snapshotId2) {
|
|
2895
3191
|
throw new Error(`Branch not found: ${name}`);
|
|
2896
3192
|
}
|
|
2897
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3193
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3194
|
+
...options,
|
|
3195
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3196
|
+
});
|
|
2898
3197
|
}
|
|
2899
3198
|
const refDirPath = getBranchRefDirPath(name);
|
|
2900
3199
|
const refPath = await this.findLatestRefFile(
|
|
@@ -2921,7 +3220,10 @@ var Repository = class _Repository {
|
|
|
2921
3220
|
if (!snapshotId2) {
|
|
2922
3221
|
throw new Error(`Tag not found: ${name}`);
|
|
2923
3222
|
}
|
|
2924
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3223
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3224
|
+
...options,
|
|
3225
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3226
|
+
});
|
|
2925
3227
|
}
|
|
2926
3228
|
const refDirPath = getTagRefDirPath(name);
|
|
2927
3229
|
const legacyPath = getTagRefPath(name);
|
|
@@ -2944,7 +3246,12 @@ var Repository = class _Repository {
|
|
|
2944
3246
|
*/
|
|
2945
3247
|
async checkoutSnapshot(snapshotId, options) {
|
|
2946
3248
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
2947
|
-
|
|
3249
|
+
const repoInfo = await this.loadRepoInfo(options);
|
|
3250
|
+
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
3251
|
+
return ReadSession.open(this.storage, id, {
|
|
3252
|
+
...options,
|
|
3253
|
+
...virtualChunkContainers && { virtualChunkContainers }
|
|
3254
|
+
});
|
|
2948
3255
|
}
|
|
2949
3256
|
/**
|
|
2950
3257
|
* Walk the snapshot history chain starting from the given session.
|
|
@@ -3023,9 +3330,7 @@ var HttpStorage = class {
|
|
|
3023
3330
|
return headers;
|
|
3024
3331
|
}
|
|
3025
3332
|
async getObject(path, range, options) {
|
|
3026
|
-
|
|
3027
|
-
throw new AbortError();
|
|
3028
|
-
}
|
|
3333
|
+
options?.signal?.throwIfAborted();
|
|
3029
3334
|
const url = this.getUrl(path);
|
|
3030
3335
|
const headers = this.getHeaders(range);
|
|
3031
3336
|
let response;
|
|
@@ -3038,9 +3343,7 @@ var HttpStorage = class {
|
|
|
3038
3343
|
signal: options?.signal
|
|
3039
3344
|
});
|
|
3040
3345
|
} catch (error) {
|
|
3041
|
-
if (error
|
|
3042
|
-
throw new AbortError();
|
|
3043
|
-
}
|
|
3346
|
+
if (isAbortError(error)) throw error;
|
|
3044
3347
|
throw new StorageError(
|
|
3045
3348
|
`Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3046
3349
|
error instanceof Error ? error : void 0
|
|
@@ -3058,9 +3361,7 @@ var HttpStorage = class {
|
|
|
3058
3361
|
return new Uint8Array(arrayBuffer);
|
|
3059
3362
|
}
|
|
3060
3363
|
async exists(path, options) {
|
|
3061
|
-
|
|
3062
|
-
throw new AbortError();
|
|
3063
|
-
}
|
|
3364
|
+
options?.signal?.throwIfAborted();
|
|
3064
3365
|
const url = this.getUrl(path);
|
|
3065
3366
|
try {
|
|
3066
3367
|
const response = await fetch(url, {
|
|
@@ -3071,9 +3372,7 @@ var HttpStorage = class {
|
|
|
3071
3372
|
});
|
|
3072
3373
|
return response.ok;
|
|
3073
3374
|
} catch (error) {
|
|
3074
|
-
if (error
|
|
3075
|
-
throw new AbortError();
|
|
3076
|
-
}
|
|
3375
|
+
if (isAbortError(error)) throw error;
|
|
3077
3376
|
return false;
|
|
3078
3377
|
}
|
|
3079
3378
|
}
|
|
@@ -3091,8 +3390,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3091
3390
|
fetchClient;
|
|
3092
3391
|
validateChecksums;
|
|
3093
3392
|
azureAccount;
|
|
3393
|
+
withRangeCoalescing;
|
|
3094
3394
|
basePath = "";
|
|
3095
|
-
constructor(session, fetchClient, validateChecksums, azureAccount) {
|
|
3395
|
+
constructor(session, fetchClient, validateChecksums, azureAccount, withRangeCoalescing) {
|
|
3096
3396
|
if (!(session instanceof ReadSession)) {
|
|
3097
3397
|
throw new Error(
|
|
3098
3398
|
"IcechunkStore constructor is private. Use IcechunkStore.open() instead."
|
|
@@ -3102,6 +3402,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3102
3402
|
this.fetchClient = fetchClient;
|
|
3103
3403
|
this.validateChecksums = validateChecksums ?? false;
|
|
3104
3404
|
this.azureAccount = azureAccount;
|
|
3405
|
+
this.withRangeCoalescing = withRangeCoalescing;
|
|
3105
3406
|
}
|
|
3106
3407
|
static async open(arg, options = {}) {
|
|
3107
3408
|
if (arg instanceof ReadSession) {
|
|
@@ -3109,7 +3410,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3109
3410
|
arg,
|
|
3110
3411
|
options.fetchClient,
|
|
3111
3412
|
options.validateChecksums,
|
|
3112
|
-
options.azureAccount
|
|
3413
|
+
options.azureAccount,
|
|
3414
|
+
options.withRangeCoalescing
|
|
3113
3415
|
);
|
|
3114
3416
|
}
|
|
3115
3417
|
const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
|
|
@@ -3137,7 +3439,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3137
3439
|
session,
|
|
3138
3440
|
options.fetchClient,
|
|
3139
3441
|
options.validateChecksums,
|
|
3140
|
-
options.azureAccount
|
|
3442
|
+
options.azureAccount,
|
|
3443
|
+
options.withRangeCoalescing
|
|
3141
3444
|
);
|
|
3142
3445
|
}
|
|
3143
3446
|
/**
|
|
@@ -3153,7 +3456,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3153
3456
|
* @returns Data bytes or undefined if not found
|
|
3154
3457
|
*/
|
|
3155
3458
|
async get(key, opts) {
|
|
3156
|
-
|
|
3459
|
+
opts?.signal?.throwIfAborted();
|
|
3157
3460
|
const parsed = parseZarrKey(key);
|
|
3158
3461
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3159
3462
|
try {
|
|
@@ -3165,11 +3468,15 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3165
3468
|
signal: opts?.signal,
|
|
3166
3469
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3167
3470
|
validateChecksums: this.validateChecksums,
|
|
3168
|
-
azureAccount: this.azureAccount
|
|
3471
|
+
azureAccount: this.azureAccount,
|
|
3472
|
+
...this.withRangeCoalescing && {
|
|
3473
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
3474
|
+
}
|
|
3169
3475
|
});
|
|
3170
3476
|
return chunk ?? void 0;
|
|
3171
|
-
} catch {
|
|
3172
|
-
return void 0;
|
|
3477
|
+
} catch (err) {
|
|
3478
|
+
if (err instanceof NotFoundError) return void 0;
|
|
3479
|
+
throw err;
|
|
3173
3480
|
}
|
|
3174
3481
|
}
|
|
3175
3482
|
/**
|
|
@@ -3185,7 +3492,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3185
3492
|
* @returns Data bytes or undefined if not found
|
|
3186
3493
|
*/
|
|
3187
3494
|
async getRange(key, range, opts) {
|
|
3188
|
-
|
|
3495
|
+
opts?.signal?.throwIfAborted();
|
|
3189
3496
|
const parsed = parseZarrKey(key);
|
|
3190
3497
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3191
3498
|
try {
|
|
@@ -3198,7 +3505,10 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3198
3505
|
signal: opts?.signal,
|
|
3199
3506
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3200
3507
|
validateChecksums: this.validateChecksums,
|
|
3201
|
-
azureAccount: this.azureAccount
|
|
3508
|
+
azureAccount: this.azureAccount,
|
|
3509
|
+
...this.withRangeCoalescing && {
|
|
3510
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
3511
|
+
}
|
|
3202
3512
|
}
|
|
3203
3513
|
);
|
|
3204
3514
|
return data2 ?? void 0;
|
|
@@ -3209,8 +3519,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3209
3519
|
return data.slice(-range.suffixLength);
|
|
3210
3520
|
}
|
|
3211
3521
|
return data.slice(range.offset, range.offset + range.length);
|
|
3212
|
-
} catch {
|
|
3213
|
-
return void 0;
|
|
3522
|
+
} catch (err) {
|
|
3523
|
+
if (err instanceof NotFoundError) return void 0;
|
|
3524
|
+
throw err;
|
|
3214
3525
|
}
|
|
3215
3526
|
}
|
|
3216
3527
|
/** Prepend basePath to a parsed path. */
|
|
@@ -3234,7 +3545,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3234
3545
|
this.session,
|
|
3235
3546
|
this.fetchClient,
|
|
3236
3547
|
this.validateChecksums,
|
|
3237
|
-
this.azureAccount
|
|
3548
|
+
this.azureAccount,
|
|
3549
|
+
this.withRangeCoalescing
|
|
3238
3550
|
);
|
|
3239
3551
|
const cleanPath = path.replace(/^\/+|\/+$/g, "");
|
|
3240
3552
|
scoped.basePath = this.basePath ? `${this.basePath}/${cleanPath}` : cleanPath;
|