icechunk-js 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/index.cjs +116 -59
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +31 -2
- package/dist/index.d.ts +31 -2
- package/dist/index.js +116 -59
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -137,6 +137,19 @@ Cloud storage URLs in virtual chunk references are automatically translated:
|
|
|
137
137
|
- `abfs://container@account.dfs.core.windows.net/path` →
|
|
138
138
|
`https://account.blob.core.windows.net/container/path`
|
|
139
139
|
|
|
140
|
+
For S3, addressing follows the repo's virtual-chunk-container config (region,
|
|
141
|
+
endpoint, path-style), mirroring the Rust implementation. Buckets whose name
|
|
142
|
+
contains a dot must use path-style addressing; when the container config records
|
|
143
|
+
a region (repos written by Icechunk do), reads go straight to that regional
|
|
144
|
+
endpoint — which serves CORS and works in both Node and the browser, with no
|
|
145
|
+
extra configuration.
|
|
146
|
+
|
|
147
|
+
If no region is known, dotted-name buckets fall back to the global endpoint,
|
|
148
|
+
whose region redirect (for buckets outside `us-east-1`) is resolved
|
|
149
|
+
automatically in Node but not in browsers — the cross-origin redirect carries no
|
|
150
|
+
CORS headers. Supply a [`fetchClient`](#virtual-chunk-authentication) that routes
|
|
151
|
+
to the regional endpoint or a CORS-enabled proxy in that case.
|
|
152
|
+
|
|
140
153
|
### Repository
|
|
141
154
|
|
|
142
155
|
For direct access to branches, tags, and checkouts.
|
package/dist/index.cjs
CHANGED
|
@@ -1279,7 +1279,7 @@ function parseRepo(data) {
|
|
|
1279
1279
|
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
1280
1280
|
}
|
|
1281
1281
|
function parseVirtualChunkContainers(repo) {
|
|
1282
|
-
const result =
|
|
1282
|
+
const result = [];
|
|
1283
1283
|
const configBytes = repo.configArray();
|
|
1284
1284
|
if (!configBytes || configBytes.length === 0) return result;
|
|
1285
1285
|
let config;
|
|
@@ -1297,12 +1297,27 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1297
1297
|
if (!isRecord(containers)) return result;
|
|
1298
1298
|
for (const container of Object.values(containers)) {
|
|
1299
1299
|
if (!isRecord(container)) continue;
|
|
1300
|
-
const
|
|
1301
|
-
if (typeof
|
|
1302
|
-
|
|
1300
|
+
const urlPrefix = container.url_prefix;
|
|
1301
|
+
if (typeof urlPrefix !== "string") continue;
|
|
1302
|
+
const name = typeof container.name === "string" ? container.name : null;
|
|
1303
|
+
const s3 = parseS3Store(container.store);
|
|
1304
|
+
result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
|
|
1303
1305
|
}
|
|
1306
|
+
result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
|
|
1304
1307
|
return result;
|
|
1305
1308
|
}
|
|
1309
|
+
function parseS3Store(store) {
|
|
1310
|
+
if (!isRecord(store)) return void 0;
|
|
1311
|
+
const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
|
|
1312
|
+
if (!isRecord(s3)) return void 0;
|
|
1313
|
+
const cfg = {};
|
|
1314
|
+
if (typeof s3.region === "string") cfg.region = s3.region;
|
|
1315
|
+
if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
|
|
1316
|
+
if (typeof s3.force_path_style === "boolean") {
|
|
1317
|
+
cfg.forcePathStyle = s3.force_path_style;
|
|
1318
|
+
}
|
|
1319
|
+
return cfg;
|
|
1320
|
+
}
|
|
1306
1321
|
function isRecord(value) {
|
|
1307
1322
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1308
1323
|
}
|
|
@@ -2960,29 +2975,53 @@ function parseTransactionLog(data) {
|
|
|
2960
2975
|
function expectedRangeLength(range) {
|
|
2961
2976
|
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2962
2977
|
}
|
|
2978
|
+
function regionalS3RedirectUrl(url, response) {
|
|
2979
|
+
if (response.status !== 301 && response.status !== 307) return null;
|
|
2980
|
+
const region = response.headers.get("x-amz-bucket-region");
|
|
2981
|
+
if (!region) return null;
|
|
2982
|
+
const globalPrefix = "https://s3.amazonaws.com/";
|
|
2983
|
+
if (!url.startsWith(globalPrefix)) return null;
|
|
2984
|
+
return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
|
|
2985
|
+
}
|
|
2963
2986
|
function makeUrlStore(opts) {
|
|
2964
|
-
const {
|
|
2965
|
-
|
|
2966
|
-
|
|
2987
|
+
const { fetchClient, conditionalHeaders } = opts;
|
|
2988
|
+
let pinnedUrl = opts.url;
|
|
2989
|
+
async function doFetch(target, init) {
|
|
2990
|
+
return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
|
|
2967
2991
|
}
|
|
2968
2992
|
return {
|
|
2969
2993
|
async get() {
|
|
2970
2994
|
throw new Error(
|
|
2971
|
-
`Virtual chunk URL store for ${
|
|
2995
|
+
`Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
|
|
2972
2996
|
);
|
|
2973
2997
|
},
|
|
2974
2998
|
async getRange(_key, range, options) {
|
|
2975
2999
|
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2976
3000
|
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2977
|
-
|
|
3001
|
+
let requestUrl = pinnedUrl;
|
|
3002
|
+
let response = await doFetch(requestUrl, {
|
|
3003
|
+
headers,
|
|
3004
|
+
signal: options?.signal
|
|
3005
|
+
});
|
|
3006
|
+
const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
|
|
3007
|
+
if (regionalUrl) {
|
|
3008
|
+
response.body?.cancel().catch(() => {
|
|
3009
|
+
});
|
|
3010
|
+
requestUrl = regionalUrl;
|
|
3011
|
+
pinnedUrl = regionalUrl;
|
|
3012
|
+
response = await doFetch(requestUrl, {
|
|
3013
|
+
headers,
|
|
3014
|
+
signal: options?.signal
|
|
3015
|
+
});
|
|
3016
|
+
}
|
|
2978
3017
|
if (response.status === 412) {
|
|
2979
3018
|
throw new Error(
|
|
2980
|
-
`Virtual chunk at ${
|
|
3019
|
+
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2981
3020
|
);
|
|
2982
3021
|
}
|
|
2983
3022
|
if (response.status !== 200 && response.status !== 206) {
|
|
2984
3023
|
throw new Error(
|
|
2985
|
-
`Failed to fetch virtual chunk from ${
|
|
3024
|
+
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
|
|
2986
3025
|
);
|
|
2987
3026
|
}
|
|
2988
3027
|
const data = new Uint8Array(await response.arrayBuffer());
|
|
@@ -2990,21 +3029,21 @@ function makeUrlStore(opts) {
|
|
|
2990
3029
|
const expected = expectedRangeLength(range);
|
|
2991
3030
|
if (data.length === expected) return data;
|
|
2992
3031
|
throw new Error(
|
|
2993
|
-
`Virtual range response size mismatch for ${
|
|
3032
|
+
`Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
|
|
2994
3033
|
);
|
|
2995
3034
|
}
|
|
2996
3035
|
if ("offset" in range) {
|
|
2997
3036
|
const end = range.offset + range.length;
|
|
2998
3037
|
if (data.length >= end) return data.slice(range.offset, end);
|
|
2999
3038
|
throw new Error(
|
|
3000
|
-
`Virtual range request not honored for ${
|
|
3039
|
+
`Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
3001
3040
|
);
|
|
3002
3041
|
}
|
|
3003
3042
|
if (data.length >= range.suffixLength) {
|
|
3004
3043
|
return data.slice(data.length - range.suffixLength);
|
|
3005
3044
|
}
|
|
3006
3045
|
throw new Error(
|
|
3007
|
-
`Virtual suffix range request not honored for ${
|
|
3046
|
+
`Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
3008
3047
|
);
|
|
3009
3048
|
}
|
|
3010
3049
|
};
|
|
@@ -3072,7 +3111,7 @@ var ReadSession = class _ReadSession {
|
|
|
3072
3111
|
nextFetchClientId = 1;
|
|
3073
3112
|
rangeCoalescerIds;
|
|
3074
3113
|
nextRangeCoalescerId = 1;
|
|
3075
|
-
/**
|
|
3114
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
3076
3115
|
virtualChunkContainers;
|
|
3077
3116
|
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
3078
3117
|
this.storage = storage;
|
|
@@ -3080,7 +3119,7 @@ var ReadSession = class _ReadSession {
|
|
|
3080
3119
|
this.specVersion = specVersion;
|
|
3081
3120
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
3082
3121
|
this.manifestLoader = singleFlight(this.manifestCache);
|
|
3083
|
-
this.virtualChunkContainers = virtualChunkContainers ??
|
|
3122
|
+
this.virtualChunkContainers = virtualChunkContainers ?? [];
|
|
3084
3123
|
}
|
|
3085
3124
|
getFetchClientKey(fetchClient) {
|
|
3086
3125
|
if (!fetchClient) return "default";
|
|
@@ -3466,14 +3505,14 @@ var ReadSession = class _ReadSession {
|
|
|
3466
3505
|
);
|
|
3467
3506
|
}
|
|
3468
3507
|
case "virtual": {
|
|
3469
|
-
const absoluteLocation =
|
|
3508
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3470
3509
|
payload.location,
|
|
3471
3510
|
this.virtualChunkContainers
|
|
3472
3511
|
);
|
|
3473
|
-
const httpUrl = translateToHttpUrl(
|
|
3474
|
-
|
|
3475
|
-
|
|
3476
|
-
);
|
|
3512
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3513
|
+
azureAccount: options?.azureAccount,
|
|
3514
|
+
s3: container?.s3
|
|
3515
|
+
});
|
|
3477
3516
|
const store = await this.getVirtualStoreForPayload(
|
|
3478
3517
|
httpUrl,
|
|
3479
3518
|
payload,
|
|
@@ -3536,14 +3575,14 @@ var ReadSession = class _ReadSession {
|
|
|
3536
3575
|
case "virtual": {
|
|
3537
3576
|
const absoluteStart = payload.offset + rangeStart;
|
|
3538
3577
|
const expectedSize = rangeEnd - rangeStart;
|
|
3539
|
-
const absoluteLocation =
|
|
3578
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3540
3579
|
payload.location,
|
|
3541
3580
|
this.virtualChunkContainers
|
|
3542
3581
|
);
|
|
3543
|
-
const httpUrl = translateToHttpUrl(
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
);
|
|
3582
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3583
|
+
azureAccount: options?.azureAccount,
|
|
3584
|
+
s3: container?.s3
|
|
3585
|
+
});
|
|
3547
3586
|
const store = await this.getVirtualStoreForPayload(
|
|
3548
3587
|
httpUrl,
|
|
3549
3588
|
payload,
|
|
@@ -3613,44 +3652,62 @@ function compareUtf8Bytes(a, b) {
|
|
|
3613
3652
|
return bytesA.length - bytesB.length;
|
|
3614
3653
|
}
|
|
3615
3654
|
var VCC_SCHEME = "vcc://";
|
|
3616
|
-
function
|
|
3617
|
-
if (
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3655
|
+
function resolveVirtualChunkLocation(location, containers) {
|
|
3656
|
+
if (location.startsWith(VCC_SCHEME)) {
|
|
3657
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
3658
|
+
const slash = rest.indexOf("/");
|
|
3659
|
+
if (slash === -1) {
|
|
3660
|
+
throw new Error(
|
|
3661
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
3662
|
+
);
|
|
3663
|
+
}
|
|
3664
|
+
const name = rest.slice(0, slash);
|
|
3665
|
+
const relativePath = rest.slice(slash + 1);
|
|
3666
|
+
const container2 = containers.find((c) => c.name === name);
|
|
3667
|
+
if (!container2) {
|
|
3668
|
+
if (containers.length === 0) return { url: location };
|
|
3669
|
+
throw new Error(
|
|
3670
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
3671
|
+
);
|
|
3672
|
+
}
|
|
3673
|
+
return {
|
|
3674
|
+
url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
|
|
3675
|
+
container: container2
|
|
3676
|
+
};
|
|
3633
3677
|
}
|
|
3634
|
-
const
|
|
3635
|
-
|
|
3678
|
+
const container = containers.find(
|
|
3679
|
+
(c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
|
|
3680
|
+
);
|
|
3681
|
+
return { url: location, container };
|
|
3682
|
+
}
|
|
3683
|
+
function ensureTrailingSlash(prefix) {
|
|
3684
|
+
return prefix.endsWith("/") ? prefix : `${prefix}/`;
|
|
3685
|
+
}
|
|
3686
|
+
function buildS3HttpUrl(bucket, key, s3) {
|
|
3687
|
+
if (s3?.endpointUrl) {
|
|
3688
|
+
const base = s3.endpointUrl.replace(/\/+$/, "");
|
|
3689
|
+
return `${base}/${bucket}/${key}`;
|
|
3690
|
+
}
|
|
3691
|
+
const region = s3?.region;
|
|
3692
|
+
const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
|
|
3693
|
+
if (pathStyle) {
|
|
3694
|
+
const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
|
|
3695
|
+
return `https://${host2}/${bucket}/${key}`;
|
|
3696
|
+
}
|
|
3697
|
+
const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
|
|
3698
|
+
return `https://${host}/${key}`;
|
|
3636
3699
|
}
|
|
3637
|
-
function
|
|
3700
|
+
function awsEndpointSuffix(region) {
|
|
3701
|
+
return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
|
|
3702
|
+
}
|
|
3703
|
+
function translateToHttpUrl(url, options) {
|
|
3704
|
+
const azureAccount = options?.azureAccount;
|
|
3638
3705
|
if (url.startsWith("s3://")) {
|
|
3639
3706
|
const rest = url.slice(5);
|
|
3640
3707
|
const slashIndex = rest.indexOf("/");
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
return `https://s3.amazonaws.com/${bucket2}/`;
|
|
3645
|
-
}
|
|
3646
|
-
return `https://${bucket2}.s3.amazonaws.com/`;
|
|
3647
|
-
}
|
|
3648
|
-
const bucket = rest.slice(0, slashIndex);
|
|
3649
|
-
const key = rest.slice(slashIndex + 1);
|
|
3650
|
-
if (bucket.includes(".")) {
|
|
3651
|
-
return `https://s3.amazonaws.com/${bucket}/${key}`;
|
|
3652
|
-
}
|
|
3653
|
-
return `https://${bucket}.s3.amazonaws.com/${key}`;
|
|
3708
|
+
const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
|
|
3709
|
+
const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
|
|
3710
|
+
return buildS3HttpUrl(bucket, key, options?.s3);
|
|
3654
3711
|
}
|
|
3655
3712
|
if (url.startsWith("gs://") || url.startsWith("gcs://")) {
|
|
3656
3713
|
const prefixLen = url.startsWith("gs://") ? 5 : 6;
|