icechunk-js 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/index.cjs +116 -59
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +31 -2
- package/dist/index.d.ts +31 -2
- package/dist/index.js +116 -59
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -337,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
|
|
|
337
337
|
coalesceSize?: number;
|
|
338
338
|
}) => AsyncReadable$1;
|
|
339
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Parser for icechunk Repo FlatBuffer format (v2).
|
|
342
|
+
*
|
|
343
|
+
* Uses flatc-generated TypeScript classes for type-safe field access.
|
|
344
|
+
*/
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* S3 store config carried by a virtual chunk container, mirroring Rust's
|
|
348
|
+
* `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
|
|
349
|
+
* locations to HTTPS — notably `region`, which lets dotted-name buckets
|
|
350
|
+
* address their regional endpoint directly (required for browser CORS, since
|
|
351
|
+
* the global endpoint's region redirect carries no CORS headers).
|
|
352
|
+
*/
|
|
353
|
+
interface S3ContainerConfig {
|
|
354
|
+
region?: string;
|
|
355
|
+
endpointUrl?: string;
|
|
356
|
+
forcePathStyle?: boolean;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* A virtual chunk container from the repo config, mirroring Rust's
|
|
360
|
+
* `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
|
|
361
|
+
* `urlPrefix` matches absolute chunk locations (longest prefix wins).
|
|
362
|
+
*/
|
|
363
|
+
interface VirtualChunkContainer {
|
|
364
|
+
name: string | null;
|
|
365
|
+
urlPrefix: string;
|
|
366
|
+
s3?: S3ContainerConfig;
|
|
367
|
+
}
|
|
368
|
+
|
|
340
369
|
/**
|
|
341
370
|
* ReadSession - Read-only session for accessing icechunk data.
|
|
342
371
|
*/
|
|
@@ -388,7 +417,7 @@ declare class ReadSession {
|
|
|
388
417
|
private nextFetchClientId;
|
|
389
418
|
private rangeCoalescerIds?;
|
|
390
419
|
private nextRangeCoalescerId;
|
|
391
|
-
/**
|
|
420
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
392
421
|
private virtualChunkContainers;
|
|
393
422
|
private constructor();
|
|
394
423
|
private getFetchClientKey;
|
|
@@ -406,7 +435,7 @@ declare class ReadSession {
|
|
|
406
435
|
*/
|
|
407
436
|
static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
|
|
408
437
|
maxManifestCacheSize?: number;
|
|
409
|
-
virtualChunkContainers?:
|
|
438
|
+
virtualChunkContainers?: VirtualChunkContainer[];
|
|
410
439
|
}): Promise<ReadSession>;
|
|
411
440
|
/** Load and parse a snapshot from storage */
|
|
412
441
|
private static loadSnapshot;
|
package/dist/index.d.ts
CHANGED
|
@@ -337,6 +337,35 @@ type RangeCoalescingFn = (store: AsyncReadable$1, opts?: {
|
|
|
337
337
|
coalesceSize?: number;
|
|
338
338
|
}) => AsyncReadable$1;
|
|
339
339
|
|
|
340
|
+
/**
|
|
341
|
+
* Parser for icechunk Repo FlatBuffer format (v2).
|
|
342
|
+
*
|
|
343
|
+
* Uses flatc-generated TypeScript classes for type-safe field access.
|
|
344
|
+
*/
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* S3 store config carried by a virtual chunk container, mirroring Rust's
|
|
348
|
+
* `S3Options`. Drives endpoint/addressing when translating `s3://` chunk
|
|
349
|
+
* locations to HTTPS — notably `region`, which lets dotted-name buckets
|
|
350
|
+
* address their regional endpoint directly (required for browser CORS, since
|
|
351
|
+
* the global endpoint's region redirect carries no CORS headers).
|
|
352
|
+
*/
|
|
353
|
+
interface S3ContainerConfig {
|
|
354
|
+
region?: string;
|
|
355
|
+
endpointUrl?: string;
|
|
356
|
+
forcePathStyle?: boolean;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* A virtual chunk container from the repo config, mirroring Rust's
|
|
360
|
+
* `VirtualChunkContainer`. `name` resolves `vcc://name/...` references;
|
|
361
|
+
* `urlPrefix` matches absolute chunk locations (longest prefix wins).
|
|
362
|
+
*/
|
|
363
|
+
interface VirtualChunkContainer {
|
|
364
|
+
name: string | null;
|
|
365
|
+
urlPrefix: string;
|
|
366
|
+
s3?: S3ContainerConfig;
|
|
367
|
+
}
|
|
368
|
+
|
|
340
369
|
/**
|
|
341
370
|
* ReadSession - Read-only session for accessing icechunk data.
|
|
342
371
|
*/
|
|
@@ -388,7 +417,7 @@ declare class ReadSession {
|
|
|
388
417
|
private nextFetchClientId;
|
|
389
418
|
private rangeCoalescerIds?;
|
|
390
419
|
private nextRangeCoalescerId;
|
|
391
|
-
/**
|
|
420
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
392
421
|
private virtualChunkContainers;
|
|
393
422
|
private constructor();
|
|
394
423
|
private getFetchClientKey;
|
|
@@ -406,7 +435,7 @@ declare class ReadSession {
|
|
|
406
435
|
*/
|
|
407
436
|
static open(storage: Storage, snapshotId: Uint8Array, options?: RequestOptions & {
|
|
408
437
|
maxManifestCacheSize?: number;
|
|
409
|
-
virtualChunkContainers?:
|
|
438
|
+
virtualChunkContainers?: VirtualChunkContainer[];
|
|
410
439
|
}): Promise<ReadSession>;
|
|
411
440
|
/** Load and parse a snapshot from storage */
|
|
412
441
|
private static loadSnapshot;
|
package/dist/index.js
CHANGED
|
@@ -1226,7 +1226,7 @@ function parseRepo(data) {
|
|
|
1226
1226
|
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
1227
1227
|
}
|
|
1228
1228
|
function parseVirtualChunkContainers(repo) {
|
|
1229
|
-
const result =
|
|
1229
|
+
const result = [];
|
|
1230
1230
|
const configBytes = repo.configArray();
|
|
1231
1231
|
if (!configBytes || configBytes.length === 0) return result;
|
|
1232
1232
|
let config;
|
|
@@ -1244,12 +1244,27 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1244
1244
|
if (!isRecord(containers)) return result;
|
|
1245
1245
|
for (const container of Object.values(containers)) {
|
|
1246
1246
|
if (!isRecord(container)) continue;
|
|
1247
|
-
const
|
|
1248
|
-
if (typeof
|
|
1249
|
-
|
|
1247
|
+
const urlPrefix = container.url_prefix;
|
|
1248
|
+
if (typeof urlPrefix !== "string") continue;
|
|
1249
|
+
const name = typeof container.name === "string" ? container.name : null;
|
|
1250
|
+
const s3 = parseS3Store(container.store);
|
|
1251
|
+
result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
|
|
1250
1252
|
}
|
|
1253
|
+
result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
|
|
1251
1254
|
return result;
|
|
1252
1255
|
}
|
|
1256
|
+
function parseS3Store(store) {
|
|
1257
|
+
if (!isRecord(store)) return void 0;
|
|
1258
|
+
const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
|
|
1259
|
+
if (!isRecord(s3)) return void 0;
|
|
1260
|
+
const cfg = {};
|
|
1261
|
+
if (typeof s3.region === "string") cfg.region = s3.region;
|
|
1262
|
+
if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
|
|
1263
|
+
if (typeof s3.force_path_style === "boolean") {
|
|
1264
|
+
cfg.forcePathStyle = s3.force_path_style;
|
|
1265
|
+
}
|
|
1266
|
+
return cfg;
|
|
1267
|
+
}
|
|
1253
1268
|
function isRecord(value) {
|
|
1254
1269
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1255
1270
|
}
|
|
@@ -2907,29 +2922,53 @@ function parseTransactionLog(data) {
|
|
|
2907
2922
|
function expectedRangeLength(range) {
|
|
2908
2923
|
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2909
2924
|
}
|
|
2925
|
+
function regionalS3RedirectUrl(url, response) {
|
|
2926
|
+
if (response.status !== 301 && response.status !== 307) return null;
|
|
2927
|
+
const region = response.headers.get("x-amz-bucket-region");
|
|
2928
|
+
if (!region) return null;
|
|
2929
|
+
const globalPrefix = "https://s3.amazonaws.com/";
|
|
2930
|
+
if (!url.startsWith(globalPrefix)) return null;
|
|
2931
|
+
return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
|
|
2932
|
+
}
|
|
2910
2933
|
function makeUrlStore(opts) {
|
|
2911
|
-
const {
|
|
2912
|
-
|
|
2913
|
-
|
|
2934
|
+
const { fetchClient, conditionalHeaders } = opts;
|
|
2935
|
+
let pinnedUrl = opts.url;
|
|
2936
|
+
async function doFetch(target, init) {
|
|
2937
|
+
return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
|
|
2914
2938
|
}
|
|
2915
2939
|
return {
|
|
2916
2940
|
async get() {
|
|
2917
2941
|
throw new Error(
|
|
2918
|
-
`Virtual chunk URL store for ${
|
|
2942
|
+
`Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
|
|
2919
2943
|
);
|
|
2920
2944
|
},
|
|
2921
2945
|
async getRange(_key, range, options) {
|
|
2922
2946
|
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2923
2947
|
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2924
|
-
|
|
2948
|
+
let requestUrl = pinnedUrl;
|
|
2949
|
+
let response = await doFetch(requestUrl, {
|
|
2950
|
+
headers,
|
|
2951
|
+
signal: options?.signal
|
|
2952
|
+
});
|
|
2953
|
+
const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
|
|
2954
|
+
if (regionalUrl) {
|
|
2955
|
+
response.body?.cancel().catch(() => {
|
|
2956
|
+
});
|
|
2957
|
+
requestUrl = regionalUrl;
|
|
2958
|
+
pinnedUrl = regionalUrl;
|
|
2959
|
+
response = await doFetch(requestUrl, {
|
|
2960
|
+
headers,
|
|
2961
|
+
signal: options?.signal
|
|
2962
|
+
});
|
|
2963
|
+
}
|
|
2925
2964
|
if (response.status === 412) {
|
|
2926
2965
|
throw new Error(
|
|
2927
|
-
`Virtual chunk at ${
|
|
2966
|
+
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2928
2967
|
);
|
|
2929
2968
|
}
|
|
2930
2969
|
if (response.status !== 200 && response.status !== 206) {
|
|
2931
2970
|
throw new Error(
|
|
2932
|
-
`Failed to fetch virtual chunk from ${
|
|
2971
|
+
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
|
|
2933
2972
|
);
|
|
2934
2973
|
}
|
|
2935
2974
|
const data = new Uint8Array(await response.arrayBuffer());
|
|
@@ -2937,21 +2976,21 @@ function makeUrlStore(opts) {
|
|
|
2937
2976
|
const expected = expectedRangeLength(range);
|
|
2938
2977
|
if (data.length === expected) return data;
|
|
2939
2978
|
throw new Error(
|
|
2940
|
-
`Virtual range response size mismatch for ${
|
|
2979
|
+
`Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
|
|
2941
2980
|
);
|
|
2942
2981
|
}
|
|
2943
2982
|
if ("offset" in range) {
|
|
2944
2983
|
const end = range.offset + range.length;
|
|
2945
2984
|
if (data.length >= end) return data.slice(range.offset, end);
|
|
2946
2985
|
throw new Error(
|
|
2947
|
-
`Virtual range request not honored for ${
|
|
2986
|
+
`Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2948
2987
|
);
|
|
2949
2988
|
}
|
|
2950
2989
|
if (data.length >= range.suffixLength) {
|
|
2951
2990
|
return data.slice(data.length - range.suffixLength);
|
|
2952
2991
|
}
|
|
2953
2992
|
throw new Error(
|
|
2954
|
-
`Virtual suffix range request not honored for ${
|
|
2993
|
+
`Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2955
2994
|
);
|
|
2956
2995
|
}
|
|
2957
2996
|
};
|
|
@@ -3019,7 +3058,7 @@ var ReadSession = class _ReadSession {
|
|
|
3019
3058
|
nextFetchClientId = 1;
|
|
3020
3059
|
rangeCoalescerIds;
|
|
3021
3060
|
nextRangeCoalescerId = 1;
|
|
3022
|
-
/**
|
|
3061
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
3023
3062
|
virtualChunkContainers;
|
|
3024
3063
|
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
3025
3064
|
this.storage = storage;
|
|
@@ -3027,7 +3066,7 @@ var ReadSession = class _ReadSession {
|
|
|
3027
3066
|
this.specVersion = specVersion;
|
|
3028
3067
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
3029
3068
|
this.manifestLoader = singleFlight(this.manifestCache);
|
|
3030
|
-
this.virtualChunkContainers = virtualChunkContainers ??
|
|
3069
|
+
this.virtualChunkContainers = virtualChunkContainers ?? [];
|
|
3031
3070
|
}
|
|
3032
3071
|
getFetchClientKey(fetchClient) {
|
|
3033
3072
|
if (!fetchClient) return "default";
|
|
@@ -3413,14 +3452,14 @@ var ReadSession = class _ReadSession {
|
|
|
3413
3452
|
);
|
|
3414
3453
|
}
|
|
3415
3454
|
case "virtual": {
|
|
3416
|
-
const absoluteLocation =
|
|
3455
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3417
3456
|
payload.location,
|
|
3418
3457
|
this.virtualChunkContainers
|
|
3419
3458
|
);
|
|
3420
|
-
const httpUrl = translateToHttpUrl(
|
|
3421
|
-
|
|
3422
|
-
|
|
3423
|
-
);
|
|
3459
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3460
|
+
azureAccount: options?.azureAccount,
|
|
3461
|
+
s3: container?.s3
|
|
3462
|
+
});
|
|
3424
3463
|
const store = await this.getVirtualStoreForPayload(
|
|
3425
3464
|
httpUrl,
|
|
3426
3465
|
payload,
|
|
@@ -3483,14 +3522,14 @@ var ReadSession = class _ReadSession {
|
|
|
3483
3522
|
case "virtual": {
|
|
3484
3523
|
const absoluteStart = payload.offset + rangeStart;
|
|
3485
3524
|
const expectedSize = rangeEnd - rangeStart;
|
|
3486
|
-
const absoluteLocation =
|
|
3525
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3487
3526
|
payload.location,
|
|
3488
3527
|
this.virtualChunkContainers
|
|
3489
3528
|
);
|
|
3490
|
-
const httpUrl = translateToHttpUrl(
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
);
|
|
3529
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3530
|
+
azureAccount: options?.azureAccount,
|
|
3531
|
+
s3: container?.s3
|
|
3532
|
+
});
|
|
3494
3533
|
const store = await this.getVirtualStoreForPayload(
|
|
3495
3534
|
httpUrl,
|
|
3496
3535
|
payload,
|
|
@@ -3560,44 +3599,62 @@ function compareUtf8Bytes(a, b) {
|
|
|
3560
3599
|
return bytesA.length - bytesB.length;
|
|
3561
3600
|
}
|
|
3562
3601
|
var VCC_SCHEME = "vcc://";
|
|
3563
|
-
function
|
|
3564
|
-
if (
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3602
|
+
function resolveVirtualChunkLocation(location, containers) {
|
|
3603
|
+
if (location.startsWith(VCC_SCHEME)) {
|
|
3604
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
3605
|
+
const slash = rest.indexOf("/");
|
|
3606
|
+
if (slash === -1) {
|
|
3607
|
+
throw new Error(
|
|
3608
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
3609
|
+
);
|
|
3610
|
+
}
|
|
3611
|
+
const name = rest.slice(0, slash);
|
|
3612
|
+
const relativePath = rest.slice(slash + 1);
|
|
3613
|
+
const container2 = containers.find((c) => c.name === name);
|
|
3614
|
+
if (!container2) {
|
|
3615
|
+
if (containers.length === 0) return { url: location };
|
|
3616
|
+
throw new Error(
|
|
3617
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
3618
|
+
);
|
|
3619
|
+
}
|
|
3620
|
+
return {
|
|
3621
|
+
url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
|
|
3622
|
+
container: container2
|
|
3623
|
+
};
|
|
3580
3624
|
}
|
|
3581
|
-
const
|
|
3582
|
-
|
|
3625
|
+
const container = containers.find(
|
|
3626
|
+
(c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
|
|
3627
|
+
);
|
|
3628
|
+
return { url: location, container };
|
|
3629
|
+
}
|
|
3630
|
+
function ensureTrailingSlash(prefix) {
|
|
3631
|
+
return prefix.endsWith("/") ? prefix : `${prefix}/`;
|
|
3632
|
+
}
|
|
3633
|
+
function buildS3HttpUrl(bucket, key, s3) {
|
|
3634
|
+
if (s3?.endpointUrl) {
|
|
3635
|
+
const base = s3.endpointUrl.replace(/\/+$/, "");
|
|
3636
|
+
return `${base}/${bucket}/${key}`;
|
|
3637
|
+
}
|
|
3638
|
+
const region = s3?.region;
|
|
3639
|
+
const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
|
|
3640
|
+
if (pathStyle) {
|
|
3641
|
+
const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
|
|
3642
|
+
return `https://${host2}/${bucket}/${key}`;
|
|
3643
|
+
}
|
|
3644
|
+
const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
|
|
3645
|
+
return `https://${host}/${key}`;
|
|
3583
3646
|
}
|
|
3584
|
-
function
|
|
3647
|
+
function awsEndpointSuffix(region) {
|
|
3648
|
+
return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
|
|
3649
|
+
}
|
|
3650
|
+
function translateToHttpUrl(url, options) {
|
|
3651
|
+
const azureAccount = options?.azureAccount;
|
|
3585
3652
|
if (url.startsWith("s3://")) {
|
|
3586
3653
|
const rest = url.slice(5);
|
|
3587
3654
|
const slashIndex = rest.indexOf("/");
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
return `https://s3.amazonaws.com/${bucket2}/`;
|
|
3592
|
-
}
|
|
3593
|
-
return `https://${bucket2}.s3.amazonaws.com/`;
|
|
3594
|
-
}
|
|
3595
|
-
const bucket = rest.slice(0, slashIndex);
|
|
3596
|
-
const key = rest.slice(slashIndex + 1);
|
|
3597
|
-
if (bucket.includes(".")) {
|
|
3598
|
-
return `https://s3.amazonaws.com/${bucket}/${key}`;
|
|
3599
|
-
}
|
|
3600
|
-
return `https://${bucket}.s3.amazonaws.com/${key}`;
|
|
3655
|
+
const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
|
|
3656
|
+
const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
|
|
3657
|
+
return buildS3HttpUrl(bucket, key, options?.s3);
|
|
3601
3658
|
}
|
|
3602
3659
|
if (url.startsWith("gs://") || url.startsWith("gcs://")) {
|
|
3603
3660
|
const prefixLen = url.startsWith("gs://") ? 5 : 6;
|