icechunk-js 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/index.cjs +141 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +51 -6
- package/dist/index.d.ts +51 -6
- package/dist/index.js +141 -74
- package/dist/index.js.map +1 -1
- package/package.json +4 -1
package/README.md
CHANGED
|
@@ -137,6 +137,19 @@ Cloud storage URLs in virtual chunk references are automatically translated:
|
|
|
137
137
|
- `abfs://container@account.dfs.core.windows.net/path` →
|
|
138
138
|
`https://account.blob.core.windows.net/container/path`
|
|
139
139
|
|
|
140
|
+
For S3, addressing follows the repo's virtual-chunk-container config (region,
|
|
141
|
+
endpoint, path-style), mirroring the Rust implementation. Buckets whose name
|
|
142
|
+
contains a dot must use path-style addressing; when the container config records
|
|
143
|
+
a region (repos written by Icechunk do), reads go straight to that regional
|
|
144
|
+
endpoint — which serves CORS and works in both Node and the browser, with no
|
|
145
|
+
extra configuration.
|
|
146
|
+
|
|
147
|
+
If no region is known, dotted-name buckets fall back to the global endpoint,
|
|
148
|
+
whose region redirect (for buckets outside `us-east-1`) is resolved
|
|
149
|
+
automatically in Node but not in browsers — the cross-origin redirect carries no
|
|
150
|
+
CORS headers. Supply a [`fetchClient`](#virtual-chunk-authentication) that routes
|
|
151
|
+
to the regional endpoint or a CORS-enabled proxy in that case.
|
|
152
|
+
|
|
140
153
|
### Repository
|
|
141
154
|
|
|
142
155
|
For direct access to branches, tags, and checkouts.
|
package/dist/index.cjs
CHANGED
|
@@ -1279,7 +1279,7 @@ function parseRepo(data) {
|
|
|
1279
1279
|
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
1280
1280
|
}
|
|
1281
1281
|
function parseVirtualChunkContainers(repo) {
|
|
1282
|
-
const result =
|
|
1282
|
+
const result = [];
|
|
1283
1283
|
const configBytes = repo.configArray();
|
|
1284
1284
|
if (!configBytes || configBytes.length === 0) return result;
|
|
1285
1285
|
let config;
|
|
@@ -1297,12 +1297,27 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1297
1297
|
if (!isRecord(containers)) return result;
|
|
1298
1298
|
for (const container of Object.values(containers)) {
|
|
1299
1299
|
if (!isRecord(container)) continue;
|
|
1300
|
-
const
|
|
1301
|
-
if (typeof
|
|
1302
|
-
|
|
1300
|
+
const urlPrefix = container.url_prefix;
|
|
1301
|
+
if (typeof urlPrefix !== "string") continue;
|
|
1302
|
+
const name = typeof container.name === "string" ? container.name : null;
|
|
1303
|
+
const s3 = parseS3Store(container.store);
|
|
1304
|
+
result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
|
|
1303
1305
|
}
|
|
1306
|
+
result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
|
|
1304
1307
|
return result;
|
|
1305
1308
|
}
|
|
1309
|
+
function parseS3Store(store) {
|
|
1310
|
+
if (!isRecord(store)) return void 0;
|
|
1311
|
+
const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
|
|
1312
|
+
if (!isRecord(s3)) return void 0;
|
|
1313
|
+
const cfg = {};
|
|
1314
|
+
if (typeof s3.region === "string") cfg.region = s3.region;
|
|
1315
|
+
if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
|
|
1316
|
+
if (typeof s3.force_path_style === "boolean") {
|
|
1317
|
+
cfg.forcePathStyle = s3.force_path_style;
|
|
1318
|
+
}
|
|
1319
|
+
return cfg;
|
|
1320
|
+
}
|
|
1306
1321
|
function isRecord(value) {
|
|
1307
1322
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1308
1323
|
}
|
|
@@ -2441,7 +2456,7 @@ function parseManifest(data) {
|
|
|
2441
2456
|
for (let i = 0; i < arraysLength; i++) {
|
|
2442
2457
|
const fbsArray = fbsManifest.arrays(i);
|
|
2443
2458
|
if (fbsArray) {
|
|
2444
|
-
arrays.push(
|
|
2459
|
+
arrays.push(makeArrayManifest(fbsArray, decodeLocation));
|
|
2445
2460
|
}
|
|
2446
2461
|
}
|
|
2447
2462
|
return { id, arrays };
|
|
@@ -2469,22 +2484,32 @@ function makeLocationDecoder(fbsManifest) {
|
|
|
2469
2484
|
return textDecoder.decode(decompressed);
|
|
2470
2485
|
};
|
|
2471
2486
|
}
|
|
2472
|
-
function
|
|
2487
|
+
function makeArrayManifest(fbsArray, decodeLocation) {
|
|
2473
2488
|
const nodeIdObj = fbsArray.nodeId();
|
|
2474
2489
|
if (!nodeIdObj)
|
|
2475
2490
|
throw new Error("ArrayManifest missing required node_id field");
|
|
2476
2491
|
const nodeId = asObjectId8(
|
|
2477
2492
|
nodeIdObj.bb.bytes().slice(nodeIdObj.bb_pos, nodeIdObj.bb_pos + 8)
|
|
2478
2493
|
);
|
|
2479
|
-
const
|
|
2480
|
-
const
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2494
|
+
const numRefs = fbsArray.refsLength();
|
|
2495
|
+
const probe = new ChunkRef();
|
|
2496
|
+
return {
|
|
2497
|
+
nodeId,
|
|
2498
|
+
numRefs,
|
|
2499
|
+
refIndex(i) {
|
|
2500
|
+
const fbsRef = fbsArray.refs(i, probe);
|
|
2501
|
+
if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
|
|
2502
|
+
const indexLength = fbsRef.indexLength();
|
|
2503
|
+
const index = new Array(indexLength);
|
|
2504
|
+
for (let d = 0; d < indexLength; d++) index[d] = fbsRef.index(d);
|
|
2505
|
+
return index;
|
|
2506
|
+
},
|
|
2507
|
+
refAt(i) {
|
|
2508
|
+
const fbsRef = fbsArray.refs(i);
|
|
2509
|
+
if (!fbsRef) throw new Error(`Manifest ref ${i} out of range`);
|
|
2510
|
+
return parseChunkRef(fbsRef, decodeLocation);
|
|
2485
2511
|
}
|
|
2486
|
-
}
|
|
2487
|
-
return { nodeId, refs };
|
|
2512
|
+
};
|
|
2488
2513
|
}
|
|
2489
2514
|
function parseChunkRef(fbsRef, decodeLocation) {
|
|
2490
2515
|
const indexLength = fbsRef.indexLength();
|
|
@@ -2535,7 +2560,7 @@ function parseChunkRef(fbsRef, decodeLocation) {
|
|
|
2535
2560
|
function findChunkRef(manifest, nodeId, coords) {
|
|
2536
2561
|
const arrayManifest = binarySearchArrayManifest(manifest.arrays, nodeId);
|
|
2537
2562
|
if (!arrayManifest) return null;
|
|
2538
|
-
return binarySearchChunkRef(arrayManifest
|
|
2563
|
+
return binarySearchChunkRef(arrayManifest, coords);
|
|
2539
2564
|
}
|
|
2540
2565
|
function binarySearchArrayManifest(arrays, nodeId) {
|
|
2541
2566
|
let low = 0;
|
|
@@ -2567,14 +2592,14 @@ function compareCoords(a, b) {
|
|
|
2567
2592
|
}
|
|
2568
2593
|
return a.length - b.length;
|
|
2569
2594
|
}
|
|
2570
|
-
function binarySearchChunkRef(
|
|
2595
|
+
function binarySearchChunkRef(array, coords) {
|
|
2571
2596
|
let low = 0;
|
|
2572
|
-
let high =
|
|
2597
|
+
let high = array.numRefs - 1;
|
|
2573
2598
|
while (low <= high) {
|
|
2574
2599
|
const mid = Math.floor((low + high) / 2);
|
|
2575
|
-
const cmp = compareCoords(
|
|
2600
|
+
const cmp = compareCoords(array.refIndex(mid), coords);
|
|
2576
2601
|
if (cmp === 0) {
|
|
2577
|
-
return
|
|
2602
|
+
return array.refAt(mid);
|
|
2578
2603
|
} else if (cmp < 0) {
|
|
2579
2604
|
low = mid + 1;
|
|
2580
2605
|
} else {
|
|
@@ -2950,29 +2975,53 @@ function parseTransactionLog(data) {
|
|
|
2950
2975
|
function expectedRangeLength(range) {
|
|
2951
2976
|
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2952
2977
|
}
|
|
2978
|
+
function regionalS3RedirectUrl(url, response) {
|
|
2979
|
+
if (response.status !== 301 && response.status !== 307) return null;
|
|
2980
|
+
const region = response.headers.get("x-amz-bucket-region");
|
|
2981
|
+
if (!region) return null;
|
|
2982
|
+
const globalPrefix = "https://s3.amazonaws.com/";
|
|
2983
|
+
if (!url.startsWith(globalPrefix)) return null;
|
|
2984
|
+
return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
|
|
2985
|
+
}
|
|
2953
2986
|
function makeUrlStore(opts) {
|
|
2954
|
-
const {
|
|
2955
|
-
|
|
2956
|
-
|
|
2987
|
+
const { fetchClient, conditionalHeaders } = opts;
|
|
2988
|
+
let pinnedUrl = opts.url;
|
|
2989
|
+
async function doFetch(target, init) {
|
|
2990
|
+
return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
|
|
2957
2991
|
}
|
|
2958
2992
|
return {
|
|
2959
2993
|
async get() {
|
|
2960
2994
|
throw new Error(
|
|
2961
|
-
`Virtual chunk URL store for ${
|
|
2995
|
+
`Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
|
|
2962
2996
|
);
|
|
2963
2997
|
},
|
|
2964
2998
|
async getRange(_key, range, options) {
|
|
2965
2999
|
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2966
3000
|
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2967
|
-
|
|
3001
|
+
let requestUrl = pinnedUrl;
|
|
3002
|
+
let response = await doFetch(requestUrl, {
|
|
3003
|
+
headers,
|
|
3004
|
+
signal: options?.signal
|
|
3005
|
+
});
|
|
3006
|
+
const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
|
|
3007
|
+
if (regionalUrl) {
|
|
3008
|
+
response.body?.cancel().catch(() => {
|
|
3009
|
+
});
|
|
3010
|
+
requestUrl = regionalUrl;
|
|
3011
|
+
pinnedUrl = regionalUrl;
|
|
3012
|
+
response = await doFetch(requestUrl, {
|
|
3013
|
+
headers,
|
|
3014
|
+
signal: options?.signal
|
|
3015
|
+
});
|
|
3016
|
+
}
|
|
2968
3017
|
if (response.status === 412) {
|
|
2969
3018
|
throw new Error(
|
|
2970
|
-
`Virtual chunk at ${
|
|
3019
|
+
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2971
3020
|
);
|
|
2972
3021
|
}
|
|
2973
3022
|
if (response.status !== 200 && response.status !== 206) {
|
|
2974
3023
|
throw new Error(
|
|
2975
|
-
`Failed to fetch virtual chunk from ${
|
|
3024
|
+
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
|
|
2976
3025
|
);
|
|
2977
3026
|
}
|
|
2978
3027
|
const data = new Uint8Array(await response.arrayBuffer());
|
|
@@ -2980,21 +3029,21 @@ function makeUrlStore(opts) {
|
|
|
2980
3029
|
const expected = expectedRangeLength(range);
|
|
2981
3030
|
if (data.length === expected) return data;
|
|
2982
3031
|
throw new Error(
|
|
2983
|
-
`Virtual range response size mismatch for ${
|
|
3032
|
+
`Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
|
|
2984
3033
|
);
|
|
2985
3034
|
}
|
|
2986
3035
|
if ("offset" in range) {
|
|
2987
3036
|
const end = range.offset + range.length;
|
|
2988
3037
|
if (data.length >= end) return data.slice(range.offset, end);
|
|
2989
3038
|
throw new Error(
|
|
2990
|
-
`Virtual range request not honored for ${
|
|
3039
|
+
`Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2991
3040
|
);
|
|
2992
3041
|
}
|
|
2993
3042
|
if (data.length >= range.suffixLength) {
|
|
2994
3043
|
return data.slice(data.length - range.suffixLength);
|
|
2995
3044
|
}
|
|
2996
3045
|
throw new Error(
|
|
2997
|
-
`Virtual suffix range request not honored for ${
|
|
3046
|
+
`Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2998
3047
|
);
|
|
2999
3048
|
}
|
|
3000
3049
|
};
|
|
@@ -3062,7 +3111,7 @@ var ReadSession = class _ReadSession {
|
|
|
3062
3111
|
nextFetchClientId = 1;
|
|
3063
3112
|
rangeCoalescerIds;
|
|
3064
3113
|
nextRangeCoalescerId = 1;
|
|
3065
|
-
/**
|
|
3114
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
3066
3115
|
virtualChunkContainers;
|
|
3067
3116
|
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
3068
3117
|
this.storage = storage;
|
|
@@ -3070,7 +3119,7 @@ var ReadSession = class _ReadSession {
|
|
|
3070
3119
|
this.specVersion = specVersion;
|
|
3071
3120
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
3072
3121
|
this.manifestLoader = singleFlight(this.manifestCache);
|
|
3073
|
-
this.virtualChunkContainers = virtualChunkContainers ??
|
|
3122
|
+
this.virtualChunkContainers = virtualChunkContainers ?? [];
|
|
3074
3123
|
}
|
|
3075
3124
|
getFetchClientKey(fetchClient) {
|
|
3076
3125
|
if (!fetchClient) return "default";
|
|
@@ -3456,14 +3505,14 @@ var ReadSession = class _ReadSession {
|
|
|
3456
3505
|
);
|
|
3457
3506
|
}
|
|
3458
3507
|
case "virtual": {
|
|
3459
|
-
const absoluteLocation =
|
|
3508
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3460
3509
|
payload.location,
|
|
3461
3510
|
this.virtualChunkContainers
|
|
3462
3511
|
);
|
|
3463
|
-
const httpUrl = translateToHttpUrl(
|
|
3464
|
-
|
|
3465
|
-
|
|
3466
|
-
);
|
|
3512
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3513
|
+
azureAccount: options?.azureAccount,
|
|
3514
|
+
s3: container?.s3
|
|
3515
|
+
});
|
|
3467
3516
|
const store = await this.getVirtualStoreForPayload(
|
|
3468
3517
|
httpUrl,
|
|
3469
3518
|
payload,
|
|
@@ -3526,14 +3575,14 @@ var ReadSession = class _ReadSession {
|
|
|
3526
3575
|
case "virtual": {
|
|
3527
3576
|
const absoluteStart = payload.offset + rangeStart;
|
|
3528
3577
|
const expectedSize = rangeEnd - rangeStart;
|
|
3529
|
-
const absoluteLocation =
|
|
3578
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3530
3579
|
payload.location,
|
|
3531
3580
|
this.virtualChunkContainers
|
|
3532
3581
|
);
|
|
3533
|
-
const httpUrl = translateToHttpUrl(
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
);
|
|
3582
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3583
|
+
azureAccount: options?.azureAccount,
|
|
3584
|
+
s3: container?.s3
|
|
3585
|
+
});
|
|
3537
3586
|
const store = await this.getVirtualStoreForPayload(
|
|
3538
3587
|
httpUrl,
|
|
3539
3588
|
payload,
|
|
@@ -3603,44 +3652,62 @@ function compareUtf8Bytes(a, b) {
|
|
|
3603
3652
|
return bytesA.length - bytesB.length;
|
|
3604
3653
|
}
|
|
3605
3654
|
var VCC_SCHEME = "vcc://";
|
|
3606
|
-
function
|
|
3607
|
-
if (
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3655
|
+
function resolveVirtualChunkLocation(location, containers) {
|
|
3656
|
+
if (location.startsWith(VCC_SCHEME)) {
|
|
3657
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
3658
|
+
const slash = rest.indexOf("/");
|
|
3659
|
+
if (slash === -1) {
|
|
3660
|
+
throw new Error(
|
|
3661
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
3662
|
+
);
|
|
3663
|
+
}
|
|
3664
|
+
const name = rest.slice(0, slash);
|
|
3665
|
+
const relativePath = rest.slice(slash + 1);
|
|
3666
|
+
const container2 = containers.find((c) => c.name === name);
|
|
3667
|
+
if (!container2) {
|
|
3668
|
+
if (containers.length === 0) return { url: location };
|
|
3669
|
+
throw new Error(
|
|
3670
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
3671
|
+
);
|
|
3672
|
+
}
|
|
3673
|
+
return {
|
|
3674
|
+
url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
|
|
3675
|
+
container: container2
|
|
3676
|
+
};
|
|
3623
3677
|
}
|
|
3624
|
-
const
|
|
3625
|
-
|
|
3678
|
+
const container = containers.find(
|
|
3679
|
+
(c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
|
|
3680
|
+
);
|
|
3681
|
+
return { url: location, container };
|
|
3682
|
+
}
|
|
3683
|
+
function ensureTrailingSlash(prefix) {
|
|
3684
|
+
return prefix.endsWith("/") ? prefix : `${prefix}/`;
|
|
3685
|
+
}
|
|
3686
|
+
function buildS3HttpUrl(bucket, key, s3) {
|
|
3687
|
+
if (s3?.endpointUrl) {
|
|
3688
|
+
const base = s3.endpointUrl.replace(/\/+$/, "");
|
|
3689
|
+
return `${base}/${bucket}/${key}`;
|
|
3690
|
+
}
|
|
3691
|
+
const region = s3?.region;
|
|
3692
|
+
const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
|
|
3693
|
+
if (pathStyle) {
|
|
3694
|
+
const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
|
|
3695
|
+
return `https://${host2}/${bucket}/${key}`;
|
|
3696
|
+
}
|
|
3697
|
+
const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
|
|
3698
|
+
return `https://${host}/${key}`;
|
|
3626
3699
|
}
|
|
3627
|
-
function
|
|
3700
|
+
function awsEndpointSuffix(region) {
|
|
3701
|
+
return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
|
|
3702
|
+
}
|
|
3703
|
+
function translateToHttpUrl(url, options) {
|
|
3704
|
+
const azureAccount = options?.azureAccount;
|
|
3628
3705
|
if (url.startsWith("s3://")) {
|
|
3629
3706
|
const rest = url.slice(5);
|
|
3630
3707
|
const slashIndex = rest.indexOf("/");
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
return `https://s3.amazonaws.com/${bucket2}/`;
|
|
3635
|
-
}
|
|
3636
|
-
return `https://${bucket2}.s3.amazonaws.com/`;
|
|
3637
|
-
}
|
|
3638
|
-
const bucket = rest.slice(0, slashIndex);
|
|
3639
|
-
const key = rest.slice(slashIndex + 1);
|
|
3640
|
-
if (bucket.includes(".")) {
|
|
3641
|
-
return `https://s3.amazonaws.com/${bucket}/${key}`;
|
|
3642
|
-
}
|
|
3643
|
-
return `https://${bucket}.s3.amazonaws.com/${key}`;
|
|
3708
|
+
const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
|
|
3709
|
+
const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
|
|
3710
|
+
return buildS3HttpUrl(bucket, key, options?.s3);
|
|
3644
3711
|
}
|
|
3645
3712
|
if (url.startsWith("gs://") || url.startsWith("gcs://")) {
|
|
3646
3713
|
const prefixLen = url.startsWith("gs://") ? 5 : 6;
|