icechunk-js 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/index.cjs +141 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +51 -6
- package/dist/index.d.ts +51 -6
- package/dist/index.js +141 -74
- package/dist/index.js.map +1 -1
- package/package.json +4 -1
package/dist/index.d.cts
CHANGED
|
@@ -159,12 +159,20 @@ interface ChunkRef {
|
|
|
159
159
|
/** Last modified timestamp (seconds since epoch) for virtual chunks */
|
|
160
160
|
checksumLastModified: number;
|
|
161
161
|
}
|
|
162
|
-
/**
|
|
162
|
+
/**
|
|
163
|
+
* Chunk references for a single array, resolved lazily from the manifest's
|
|
164
|
+
* FlatBuffer. Keeping refs in the buffer avoids building a JS object per chunk,
|
|
165
|
+
* which would OOM on large virtual shards.
|
|
166
|
+
*/
|
|
163
167
|
interface ArrayManifest {
|
|
164
168
|
/** Node ID of the array */
|
|
165
169
|
nodeId: ObjectId8;
|
|
166
|
-
/**
|
|
167
|
-
|
|
170
|
+
/** Number of chunk refs (sorted by index). */
|
|
171
|
+
readonly numRefs: number;
|
|
172
|
+
/** Coordinate index of ref `i`, for the binary search in `findChunkRef`. */
|
|
173
|
+
refIndex(i: number): number[];
|
|
174
|
+
/** Materialize the full `ChunkRef` at position `i`. */
|
|
175
|
+
refAt(i: number): ChunkRef;
|
|
168
176
|
}
|
|
169
177
|
/** Manifest containing chunk references for multiple arrays */
|
|
170
178
|
interface Manifest {
|
|
@@ -329,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
|
|
|
329
337
|
coalesceSize?: number;
|
|
330
338
|
}) => AsyncReadable$1;
|
|
331
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Parser for icechunk Repo FlatBuffer format (v2).
|
|
342
|
+
*
|
|
343
|
+
* Uses flatc-generated TypeScript classes for type-safe field access.
|
|
344
|
+
*/
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* S3 store config carried by a virtual chunk container, mirroring Rust's
|
|
348
|
+
* `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
|
|
349
|
+
* locations to HTTPS — notably `region`, which lets dotted-name buckets
|
|
350
|
+
* address their regional endpoint directly (required for browser CORS, since
|
|
351
|
+
* the global endpoint's region redirect carries no CORS headers).
|
|
352
|
+
*/
|
|
353
|
+
interface S3ContainerConfig {
|
|
354
|
+
region?: string;
|
|
355
|
+
endpointUrl?: string;
|
|
356
|
+
forcePathStyle?: boolean;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* A virtual chunk container from the repo config, mirroring Rust's
|
|
360
|
+
* `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
|
|
361
|
+
* `urlPrefix` matches absolute chunk locations (longest prefix wins).
|
|
362
|
+
*/
|
|
363
|
+
interface VirtualChunkContainer {
|
|
364
|
+
name: string | null;
|
|
365
|
+
urlPrefix: string;
|
|
366
|
+
s3?: S3ContainerConfig;
|
|
367
|
+
}
|
|
368
|
+
|
|
332
369
|
/**
|
|
333
370
|
* ReadSession - Read-only session for accessing icechunk data.
|
|
334
371
|
*/
|
|
@@ -380,7 +417,7 @@ declare class ReadSession {
|
|
|
380
417
|
private nextFetchClientId;
|
|
381
418
|
private rangeCoalescerIds?;
|
|
382
419
|
private nextRangeCoalescerId;
|
|
383
|
-
/**
|
|
420
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
384
421
|
private virtualChunkContainers;
|
|
385
422
|
private constructor();
|
|
386
423
|
private getFetchClientKey;
|
|
@@ -398,7 +435,7 @@ declare class ReadSession {
|
|
|
398
435
|
*/
|
|
399
436
|
static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
|
|
400
437
|
maxManifestCacheSize?: number;
|
|
401
|
-
virtualChunkContainers?:
|
|
438
|
+
virtualChunkContainers?: VirtualChunkContainer[];
|
|
402
439
|
}): Promise<ReadSession>;
|
|
403
440
|
/** Load and parse a snapshot from storage */
|
|
404
441
|
private static loadSnapshot;
|
|
@@ -576,7 +613,15 @@ interface IcechunkStoreOptions {
|
|
|
576
613
|
* - Route through a proxy
|
|
577
614
|
*/
|
|
578
615
|
fetchClient?: FetchClient;
|
|
579
|
-
/**
|
|
616
|
+
/**
|
|
617
|
+
* Maximum number of manifests to keep in the LRU cache (default: 100).
|
|
618
|
+
*
|
|
619
|
+
* This bounds the *number* of cached manifests, not their total bytes. Each
|
|
620
|
+
* entry retains a whole decompressed manifest shard — hundreds of MB for
|
|
621
|
+
* fine-level virtual stores — so a single oversized shard, or a few held at
|
|
622
|
+
* once, can still dominate memory. Lower this when reading stores with very
|
|
623
|
+
* large manifests.
|
|
624
|
+
*/
|
|
580
625
|
maxManifestCacheSize?: number;
|
|
581
626
|
/**
|
|
582
627
|
* Zarrita-backed range coalescing function for chunk payload reads.
|
package/dist/index.d.ts
CHANGED
|
@@ -159,12 +159,20 @@ interface ChunkRef {
|
|
|
159
159
|
/** Last modified timestamp (seconds since epoch) for virtual chunks */
|
|
160
160
|
checksumLastModified: number;
|
|
161
161
|
}
|
|
162
|
-
/**
|
|
162
|
+
/**
|
|
163
|
+
* Chunk references for a single array, resolved lazily from the manifest's
|
|
164
|
+
* FlatBuffer. Keeping refs in the buffer avoids building a JS object per chunk,
|
|
165
|
+
* which would OOM on large virtual shards.
|
|
166
|
+
*/
|
|
163
167
|
interface ArrayManifest {
|
|
164
168
|
/** Node ID of the array */
|
|
165
169
|
nodeId: ObjectId8;
|
|
166
|
-
/**
|
|
167
|
-
|
|
170
|
+
/** Number of chunk refs (sorted by index). */
|
|
171
|
+
readonly numRefs: number;
|
|
172
|
+
/** Coordinate index of ref `i`, for the binary search in `findChunkRef`. */
|
|
173
|
+
refIndex(i: number): number[];
|
|
174
|
+
/** Materialize the full `ChunkRef` at position `i`. */
|
|
175
|
+
refAt(i: number): ChunkRef;
|
|
168
176
|
}
|
|
169
177
|
/** Manifest containing chunk references for multiple arrays */
|
|
170
178
|
interface Manifest {
|
|
@@ -329,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
|
|
|
329
337
|
coalesceSize?: number;
|
|
330
338
|
}) => AsyncReadable$1;
|
|
331
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Parser for icechunk Repo FlatBuffer format (v2).
|
|
342
|
+
*
|
|
343
|
+
* Uses flatc-generated TypeScript classes for type-safe field access.
|
|
344
|
+
*/
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* S3 store config carried by a virtual chunk container, mirroring Rust's
|
|
348
|
+
* `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
|
|
349
|
+
* locations to HTTPS — notably `region`, which lets dotted-name buckets
|
|
350
|
+
* address their regional endpoint directly (required for browser CORS, since
|
|
351
|
+
* the global endpoint's region redirect carries no CORS headers).
|
|
352
|
+
*/
|
|
353
|
+
interface S3ContainerConfig {
|
|
354
|
+
region?: string;
|
|
355
|
+
endpointUrl?: string;
|
|
356
|
+
forcePathStyle?: boolean;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* A virtual chunk container from the repo config, mirroring Rust's
|
|
360
|
+
* `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
|
|
361
|
+
* `urlPrefix` matches absolute chunk locations (longest prefix wins).
|
|
362
|
+
*/
|
|
363
|
+
interface VirtualChunkContainer {
|
|
364
|
+
name: string | null;
|
|
365
|
+
urlPrefix: string;
|
|
366
|
+
s3?: S3ContainerConfig;
|
|
367
|
+
}
|
|
368
|
+
|
|
332
369
|
/**
|
|
333
370
|
* ReadSession - Read-only session for accessing icechunk data.
|
|
334
371
|
*/
|
|
@@ -380,7 +417,7 @@ declare class ReadSession {
|
|
|
380
417
|
private nextFetchClientId;
|
|
381
418
|
private rangeCoalescerIds?;
|
|
382
419
|
private nextRangeCoalescerId;
|
|
383
|
-
/**
|
|
420
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
384
421
|
private virtualChunkContainers;
|
|
385
422
|
private constructor();
|
|
386
423
|
private getFetchClientKey;
|
|
@@ -398,7 +435,7 @@ declare class ReadSession {
|
|
|
398
435
|
*/
|
|
399
436
|
static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
|
|
400
437
|
maxManifestCacheSize?: number;
|
|
401
|
-
virtualChunkContainers?:
|
|
438
|
+
virtualChunkContainers?: VirtualChunkContainer[];
|
|
402
439
|
}): Promise<ReadSession>;
|
|
403
440
|
/** Load and parse a snapshot from storage */
|
|
404
441
|
private static loadSnapshot;
|
|
@@ -576,7 +613,15 @@ interface IcechunkStoreOptions {
|
|
|
576
613
|
* - Route through a proxy
|
|
577
614
|
*/
|
|
578
615
|
fetchClient?: FetchClient;
|
|
579
|
-
/**
|
|
616
|
+
/**
|
|
617
|
+
* Maximum number of manifests to keep in the LRU cache (default: 100).
|
|
618
|
+
*
|
|
619
|
+
* This bounds the *number* of cached manifests, not their total bytes. Each
|
|
620
|
+
* entry retains a whole decompressed manifest shard — hundreds of MB for
|
|
621
|
+
* fine-level virtual stores — so a single oversized shard, or a few held at
|
|
622
|
+
* once, can still dominate memory. Lower this when reading stores with very
|
|
623
|
+
* large manifests.
|
|
624
|
+
*/
|
|
580
625
|
maxManifestCacheSize?: number;
|
|
581
626
|
/**
|
|
582
627
|
* Zarrita-backed range coalescing function for chunk payload reads.
|
package/dist/index.js
CHANGED
|
@@ -1226,7 +1226,7 @@ function parseRepo(data) {
|
|
|
1226
1226
|
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
1227
1227
|
}
|
|
1228
1228
|
function parseVirtualChunkContainers(repo) {
|
|
1229
|
-
const result =
|
|
1229
|
+
const result = [];
|
|
1230
1230
|
const configBytes = repo.configArray();
|
|
1231
1231
|
if (!configBytes || configBytes.length === 0) return result;
|
|
1232
1232
|
let config;
|
|
@@ -1244,12 +1244,27 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1244
1244
|
if (!isRecord(containers)) return result;
|
|
1245
1245
|
for (const container of Object.values(containers)) {
|
|
1246
1246
|
if (!isRecord(container)) continue;
|
|
1247
|
-
const
|
|
1248
|
-
if (typeof
|
|
1249
|
-
|
|
1247
|
+
const urlPrefix = container.url_prefix;
|
|
1248
|
+
if (typeof urlPrefix !== "string") continue;
|
|
1249
|
+
const name = typeof container.name === "string" ? container.name : null;
|
|
1250
|
+
const s3 = parseS3Store(container.store);
|
|
1251
|
+
result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
|
|
1250
1252
|
}
|
|
1253
|
+
result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
|
|
1251
1254
|
return result;
|
|
1252
1255
|
}
|
|
1256
|
+
function parseS3Store(store) {
|
|
1257
|
+
if (!isRecord(store)) return void 0;
|
|
1258
|
+
const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
|
|
1259
|
+
if (!isRecord(s3)) return void 0;
|
|
1260
|
+
const cfg = {};
|
|
1261
|
+
if (typeof s3.region === "string") cfg.region = s3.region;
|
|
1262
|
+
if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
|
|
1263
|
+
if (typeof s3.force_path_style === "boolean") {
|
|
1264
|
+
cfg.forcePathStyle = s3.force_path_style;
|
|
1265
|
+
}
|
|
1266
|
+
return cfg;
|
|
1267
|
+
}
|
|
1253
1268
|
function isRecord(value) {
|
|
1254
1269
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1255
1270
|
}
|
|
@@ -2388,7 +2403,7 @@ function parseManifest(data) {
|
|
|
2388
2403
|
for (let i = 0; i < arraysLength; i++) {
|
|
2389
2404
|
const fbsArray = fbsManifest.arrays(i);
|
|
2390
2405
|
if (fbsArray) {
|
|
2391
|
-
arrays.push(
|
|
2406
|
+
arrays.push(makeArrayManifest(fbsArray, decodeLocation));
|
|
2392
2407
|
}
|
|
2393
2408
|
}
|
|
2394
2409
|
return { id, arrays };
|
|
@@ -2416,22 +2431,32 @@ function makeLocationDecoder(fbsManifest) {
|
|
|
2416
2431
|
return textDecoder.decode(decompressed);
|
|
2417
2432
|
};
|
|
2418
2433
|
}
|
|
2419
|
-
function
|
|
2434
|
+
function makeArrayManifest(fbsArray, decodeLocation) {
|
|
2420
2435
|
const nodeIdObj = fbsArray.nodeId();
|
|
2421
2436
|
if (!nodeIdObj)
|
|
2422
2437
|
throw new Error("ArrayManifest missing required node_id field");
|
|
2423
2438
|
const nodeId = asObjectId8(
|
|
2424
2439
|
nodeIdObj.bb.bytes().slice(nodeIdObj.bb_pos, nodeIdObj.bb_pos + 8)
|
|
2425
2440
|
);
|
|
2426
|
-
const
|
|
2427
|
-
const
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2441
|
+
const numRefs = fbsArray.refsLength();
|
|
2442
|
+
const probe = new ChunkRef();
|
|
2443
|
+
return {
|
|
2444
|
+
nodeId,
|
|
2445
|
+
numRefs,
|
|
2446
|
+
refIndex(i) {
|
|
2447
|
+
const fbsRef = fbsArray.refs(i, probe);
|
|
2448
|
+
if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
|
|
2449
|
+
const indexLength = fbsRef.indexLength();
|
|
2450
|
+
const index = new Array(indexLength);
|
|
2451
|
+
for (let d = 0; d < indexLength; d++) index[d] = fbsRef.index(d);
|
|
2452
|
+
return index;
|
|
2453
|
+
},
|
|
2454
|
+
refAt(i) {
|
|
2455
|
+
const fbsRef = fbsArray.refs(i);
|
|
2456
|
+
if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
|
|
2457
|
+
return parseChunkRef(fbsRef, decodeLocation);
|
|
2432
2458
|
}
|
|
2433
|
-
}
|
|
2434
|
-
return { nodeId, refs };
|
|
2459
|
+
};
|
|
2435
2460
|
}
|
|
2436
2461
|
function parseChunkRef(fbsRef, decodeLocation) {
|
|
2437
2462
|
const indexLength = fbsRef.indexLength();
|
|
@@ -2482,7 +2507,7 @@ function parseChunkRef(fbsRef, decodeLocation) {
|
|
|
2482
2507
|
function findChunkRef(manifest, nodeId, coords) {
|
|
2483
2508
|
const arrayManifest = binarySearchArrayManifest(manifest.arrays, nodeId);
|
|
2484
2509
|
if (!arrayManifest) return null;
|
|
2485
|
-
return binarySearchChunkRef(arrayManifest
|
|
2510
|
+
return binarySearchChunkRef(arrayManifest, coords);
|
|
2486
2511
|
}
|
|
2487
2512
|
function binarySearchArrayManifest(arrays, nodeId) {
|
|
2488
2513
|
let low = 0;
|
|
@@ -2514,14 +2539,14 @@ function compareCoords(a, b) {
|
|
|
2514
2539
|
}
|
|
2515
2540
|
return a.length - b.length;
|
|
2516
2541
|
}
|
|
2517
|
-
function binarySearchChunkRef(
|
|
2542
|
+
function binarySearchChunkRef(array, coords) {
|
|
2518
2543
|
let low = 0;
|
|
2519
|
-
let high =
|
|
2544
|
+
let high = array.numRefs - 1;
|
|
2520
2545
|
while (low <= high) {
|
|
2521
2546
|
const mid = Math.floor((low + high) / 2);
|
|
2522
|
-
const cmp = compareCoords(
|
|
2547
|
+
const cmp = compareCoords(array.refIndex(mid), coords);
|
|
2523
2548
|
if (cmp === 0) {
|
|
2524
|
-
return
|
|
2549
|
+
return array.refAt(mid);
|
|
2525
2550
|
} else if (cmp < 0) {
|
|
2526
2551
|
low = mid + 1;
|
|
2527
2552
|
} else {
|
|
@@ -2897,29 +2922,53 @@ function parseTransactionLog(data) {
|
|
|
2897
2922
|
function expectedRangeLength(range) {
|
|
2898
2923
|
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2899
2924
|
}
|
|
2925
|
+
function regionalS3RedirectUrl(url, response) {
|
|
2926
|
+
if (response.status !== 301 && response.status !== 307) return null;
|
|
2927
|
+
const region = response.headers.get("x-amz-bucket-region");
|
|
2928
|
+
if (!region) return null;
|
|
2929
|
+
const globalPrefix = "https://s3.amazonaws.com/";
|
|
2930
|
+
if (!url.startsWith(globalPrefix)) return null;
|
|
2931
|
+
return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
|
|
2932
|
+
}
|
|
2900
2933
|
function makeUrlStore(opts) {
|
|
2901
|
-
const {
|
|
2902
|
-
|
|
2903
|
-
|
|
2934
|
+
const { fetchClient, conditionalHeaders } = opts;
|
|
2935
|
+
let pinnedUrl = opts.url;
|
|
2936
|
+
async function doFetch(target, init) {
|
|
2937
|
+
return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
|
|
2904
2938
|
}
|
|
2905
2939
|
return {
|
|
2906
2940
|
async get() {
|
|
2907
2941
|
throw new Error(
|
|
2908
|
-
`Virtual chunk URL store for ${
|
|
2942
|
+
`Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
|
|
2909
2943
|
);
|
|
2910
2944
|
},
|
|
2911
2945
|
async getRange(_key, range, options) {
|
|
2912
2946
|
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2913
2947
|
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2914
|
-
|
|
2948
|
+
let requestUrl = pinnedUrl;
|
|
2949
|
+
let response = await doFetch(requestUrl, {
|
|
2950
|
+
headers,
|
|
2951
|
+
signal: options?.signal
|
|
2952
|
+
});
|
|
2953
|
+
const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
|
|
2954
|
+
if (regionalUrl) {
|
|
2955
|
+
response.body?.cancel().catch(() => {
|
|
2956
|
+
});
|
|
2957
|
+
requestUrl = regionalUrl;
|
|
2958
|
+
pinnedUrl = regionalUrl;
|
|
2959
|
+
response = await doFetch(requestUrl, {
|
|
2960
|
+
headers,
|
|
2961
|
+
signal: options?.signal
|
|
2962
|
+
});
|
|
2963
|
+
}
|
|
2915
2964
|
if (response.status === 412) {
|
|
2916
2965
|
throw new Error(
|
|
2917
|
-
`Virtual chunk at ${
|
|
2966
|
+
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2918
2967
|
);
|
|
2919
2968
|
}
|
|
2920
2969
|
if (response.status !== 200 && response.status !== 206) {
|
|
2921
2970
|
throw new Error(
|
|
2922
|
-
`Failed to fetch virtual chunk from ${
|
|
2971
|
+
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
|
|
2923
2972
|
);
|
|
2924
2973
|
}
|
|
2925
2974
|
const data = new Uint8Array(await response.arrayBuffer());
|
|
@@ -2927,21 +2976,21 @@ function makeUrlStore(opts) {
|
|
|
2927
2976
|
const expected = expectedRangeLength(range);
|
|
2928
2977
|
if (data.length === expected) return data;
|
|
2929
2978
|
throw new Error(
|
|
2930
|
-
`Virtual range response size mismatch for ${
|
|
2979
|
+
`Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
|
|
2931
2980
|
);
|
|
2932
2981
|
}
|
|
2933
2982
|
if ("offset" in range) {
|
|
2934
2983
|
const end = range.offset + range.length;
|
|
2935
2984
|
if (data.length >= end) return data.slice(range.offset, end);
|
|
2936
2985
|
throw new Error(
|
|
2937
|
-
`Virtual range request not honored for ${
|
|
2986
|
+
`Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2938
2987
|
);
|
|
2939
2988
|
}
|
|
2940
2989
|
if (data.length >= range.suffixLength) {
|
|
2941
2990
|
return data.slice(data.length - range.suffixLength);
|
|
2942
2991
|
}
|
|
2943
2992
|
throw new Error(
|
|
2944
|
-
`Virtual suffix range request not honored for ${
|
|
2993
|
+
`Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2945
2994
|
);
|
|
2946
2995
|
}
|
|
2947
2996
|
};
|
|
@@ -3009,7 +3058,7 @@ var ReadSession = class _ReadSession {
|
|
|
3009
3058
|
nextFetchClientId = 1;
|
|
3010
3059
|
rangeCoalescerIds;
|
|
3011
3060
|
nextRangeCoalescerId = 1;
|
|
3012
|
-
/**
|
|
3061
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
3013
3062
|
virtualChunkContainers;
|
|
3014
3063
|
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
3015
3064
|
this.storage = storage;
|
|
@@ -3017,7 +3066,7 @@ var ReadSession = class _ReadSession {
|
|
|
3017
3066
|
this.specVersion = specVersion;
|
|
3018
3067
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
3019
3068
|
this.manifestLoader = singleFlight(this.manifestCache);
|
|
3020
|
-
this.virtualChunkContainers = virtualChunkContainers ??
|
|
3069
|
+
this.virtualChunkContainers = virtualChunkContainers ?? [];
|
|
3021
3070
|
}
|
|
3022
3071
|
getFetchClientKey(fetchClient) {
|
|
3023
3072
|
if (!fetchClient) return "default";
|
|
@@ -3403,14 +3452,14 @@ var ReadSession = class _ReadSession {
|
|
|
3403
3452
|
);
|
|
3404
3453
|
}
|
|
3405
3454
|
case "virtual": {
|
|
3406
|
-
const absoluteLocation =
|
|
3455
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3407
3456
|
payload.location,
|
|
3408
3457
|
this.virtualChunkContainers
|
|
3409
3458
|
);
|
|
3410
|
-
const httpUrl = translateToHttpUrl(
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
);
|
|
3459
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3460
|
+
azureAccount: options?.azureAccount,
|
|
3461
|
+
s3: container?.s3
|
|
3462
|
+
});
|
|
3414
3463
|
const store = await this.getVirtualStoreForPayload(
|
|
3415
3464
|
httpUrl,
|
|
3416
3465
|
payload,
|
|
@@ -3473,14 +3522,14 @@ var ReadSession = class _ReadSession {
|
|
|
3473
3522
|
case "virtual": {
|
|
3474
3523
|
const absoluteStart = payload.offset + rangeStart;
|
|
3475
3524
|
const expectedSize = rangeEnd - rangeStart;
|
|
3476
|
-
const absoluteLocation =
|
|
3525
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3477
3526
|
payload.location,
|
|
3478
3527
|
this.virtualChunkContainers
|
|
3479
3528
|
);
|
|
3480
|
-
const httpUrl = translateToHttpUrl(
|
|
3481
|
-
|
|
3482
|
-
|
|
3483
|
-
);
|
|
3529
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3530
|
+
azureAccount: options?.azureAccount,
|
|
3531
|
+
s3: container?.s3
|
|
3532
|
+
});
|
|
3484
3533
|
const store = await this.getVirtualStoreForPayload(
|
|
3485
3534
|
httpUrl,
|
|
3486
3535
|
payload,
|
|
@@ -3550,44 +3599,62 @@ function compareUtf8Bytes(a, b) {
|
|
|
3550
3599
|
return bytesA.length - bytesB.length;
|
|
3551
3600
|
}
|
|
3552
3601
|
var VCC_SCHEME = "vcc://";
|
|
3553
|
-
function
|
|
3554
|
-
if (
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3602
|
+
function resolveVirtualChunkLocation(location, containers) {
|
|
3603
|
+
if (location.startsWith(VCC_SCHEME)) {
|
|
3604
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
3605
|
+
const slash = rest.indexOf("/");
|
|
3606
|
+
if (slash === -1) {
|
|
3607
|
+
throw new Error(
|
|
3608
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
3609
|
+
);
|
|
3610
|
+
}
|
|
3611
|
+
const name = rest.slice(0, slash);
|
|
3612
|
+
const relativePath = rest.slice(slash + 1);
|
|
3613
|
+
const container2 = containers.find((c) => c.name === name);
|
|
3614
|
+
if (!container2) {
|
|
3615
|
+
if (containers.length === 0) return { url: location };
|
|
3616
|
+
throw new Error(
|
|
3617
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
3618
|
+
);
|
|
3619
|
+
}
|
|
3620
|
+
return {
|
|
3621
|
+
url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
|
|
3622
|
+
container: container2
|
|
3623
|
+
};
|
|
3570
3624
|
}
|
|
3571
|
-
const
|
|
3572
|
-
|
|
3625
|
+
const container = containers.find(
|
|
3626
|
+
(c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
|
|
3627
|
+
);
|
|
3628
|
+
return { url: location, container };
|
|
3629
|
+
}
|
|
3630
|
+
function ensureTrailingSlash(prefix) {
|
|
3631
|
+
return prefix.endsWith("/") ? prefix : `${prefix}/`;
|
|
3632
|
+
}
|
|
3633
|
+
function buildS3HttpUrl(bucket, key, s3) {
|
|
3634
|
+
if (s3?.endpointUrl) {
|
|
3635
|
+
const base = s3.endpointUrl.replace(/\/+$/, "");
|
|
3636
|
+
return `${base}/${bucket}/${key}`;
|
|
3637
|
+
}
|
|
3638
|
+
const region = s3?.region;
|
|
3639
|
+
const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
|
|
3640
|
+
if (pathStyle) {
|
|
3641
|
+
const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
|
|
3642
|
+
return `https://${host2}/${bucket}/${key}`;
|
|
3643
|
+
}
|
|
3644
|
+
const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
|
|
3645
|
+
return `https://${host}/${key}`;
|
|
3573
3646
|
}
|
|
3574
|
-
function
|
|
3647
|
+
function awsEndpointSuffix(region) {
|
|
3648
|
+
return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
|
|
3649
|
+
}
|
|
3650
|
+
function translateToHttpUrl(url, options) {
|
|
3651
|
+
const azureAccount = options?.azureAccount;
|
|
3575
3652
|
if (url.startsWith("s3://")) {
|
|
3576
3653
|
const rest = url.slice(5);
|
|
3577
3654
|
const slashIndex = rest.indexOf("/");
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
return `https://s3.amazonaws.com/${bucket2}/`;
|
|
3582
|
-
}
|
|
3583
|
-
return `https://${bucket2}.s3.amazonaws.com/`;
|
|
3584
|
-
}
|
|
3585
|
-
const bucket = rest.slice(0, slashIndex);
|
|
3586
|
-
const key = rest.slice(slashIndex + 1);
|
|
3587
|
-
if (bucket.includes(".")) {
|
|
3588
|
-
return `https://s3.amazonaws.com/${bucket}/${key}`;
|
|
3589
|
-
}
|
|
3590
|
-
return `https://${bucket}.s3.amazonaws.com/${key}`;
|
|
3655
|
+
const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
|
|
3656
|
+
const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
|
|
3657
|
+
return buildS3HttpUrl(bucket, key, options?.s3);
|
|
3591
3658
|
}
|
|
3592
3659
|
if (url.startsWith("gs://") || url.startsWith("gcs://")) {
|
|
3593
3660
|
const prefixLen = url.startsWith("gs://") ? 5 : 6;
|