icechunk-js 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/index.cjs +164 -61
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +33 -2
- package/dist/index.d.ts +33 -2
- package/dist/index.js +164 -61
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -137,6 +137,19 @@ Cloud storage URLs in virtual chunk references are automatically translated:
|
|
|
137
137
|
- `abfs://container@account.dfs.core.windows.net/path` →
|
|
138
138
|
`https://account.blob.core.windows.net/container/path`
|
|
139
139
|
|
|
140
|
+
For S3, addressing follows the repo's virtual-chunk-container config (region,
|
|
141
|
+
endpoint, path-style), mirroring the Rust implementation. Buckets whose name
|
|
142
|
+
contains a dot must use path-style addressing; when the container config records
|
|
143
|
+
a region (repos written by Icechunk do), reads go straight to that regional
|
|
144
|
+
endpoint — which serves CORS and works in both Node and the browser, with no
|
|
145
|
+
extra configuration.
|
|
146
|
+
|
|
147
|
+
If no region is known, dotted-name buckets fall back to the global endpoint,
|
|
148
|
+
whose region redirect (for buckets outside `us-east-1`) is resolved
|
|
149
|
+
automatically in Node but not in browsers — the cross-origin redirect carries no
|
|
150
|
+
CORS headers. Supply a [`fetchClient`](#virtual-chunk-authentication) that routes
|
|
151
|
+
to the regional endpoint or a CORS-enabled proxy in that case.
|
|
152
|
+
|
|
140
153
|
### Repository
|
|
141
154
|
|
|
142
155
|
For direct access to branches, tags, and checkouts.
|
package/dist/index.cjs
CHANGED
|
@@ -1279,7 +1279,7 @@ function parseRepo(data) {
|
|
|
1279
1279
|
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
1280
1280
|
}
|
|
1281
1281
|
function parseVirtualChunkContainers(repo) {
|
|
1282
|
-
const result =
|
|
1282
|
+
const result = [];
|
|
1283
1283
|
const configBytes = repo.configArray();
|
|
1284
1284
|
if (!configBytes || configBytes.length === 0) return result;
|
|
1285
1285
|
let config;
|
|
@@ -1297,12 +1297,27 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1297
1297
|
if (!isRecord(containers)) return result;
|
|
1298
1298
|
for (const container of Object.values(containers)) {
|
|
1299
1299
|
if (!isRecord(container)) continue;
|
|
1300
|
-
const
|
|
1301
|
-
if (typeof
|
|
1302
|
-
|
|
1300
|
+
const urlPrefix = container.url_prefix;
|
|
1301
|
+
if (typeof urlPrefix !== "string") continue;
|
|
1302
|
+
const name = typeof container.name === "string" ? container.name : null;
|
|
1303
|
+
const s3 = parseS3Store(container.store);
|
|
1304
|
+
result.push(s3 ? { name, urlPrefix, s3 } : { name, urlPrefix });
|
|
1303
1305
|
}
|
|
1306
|
+
result.sort((a, b) => b.urlPrefix.length - a.urlPrefix.length);
|
|
1304
1307
|
return result;
|
|
1305
1308
|
}
|
|
1309
|
+
function parseS3Store(store) {
|
|
1310
|
+
if (!isRecord(store)) return void 0;
|
|
1311
|
+
const s3 = store.s3 ?? store.s3_compatible ?? store.tigris;
|
|
1312
|
+
if (!isRecord(s3)) return void 0;
|
|
1313
|
+
const cfg = {};
|
|
1314
|
+
if (typeof s3.region === "string") cfg.region = s3.region;
|
|
1315
|
+
if (typeof s3.endpoint_url === "string") cfg.endpointUrl = s3.endpoint_url;
|
|
1316
|
+
if (typeof s3.force_path_style === "boolean") {
|
|
1317
|
+
cfg.forcePathStyle = s3.force_path_style;
|
|
1318
|
+
}
|
|
1319
|
+
return cfg;
|
|
1320
|
+
}
|
|
1306
1321
|
function isRecord(value) {
|
|
1307
1322
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1308
1323
|
}
|
|
@@ -1400,6 +1415,33 @@ function listTagsFromRepo(parsedRepo) {
|
|
|
1400
1415
|
const { repo } = parsedRepo;
|
|
1401
1416
|
return listRefs((i) => repo.tags(i), repo.tagsLength());
|
|
1402
1417
|
}
|
|
1418
|
+
var snapshotIndexCache = /* @__PURE__ */ new WeakMap();
|
|
1419
|
+
function snapshotIndexById(parsedRepo) {
|
|
1420
|
+
let index = snapshotIndexCache.get(parsedRepo);
|
|
1421
|
+
if (!index) {
|
|
1422
|
+
index = /* @__PURE__ */ new Map();
|
|
1423
|
+
for (let i = 0; i < parsedRepo.snapshotsLength; i++) {
|
|
1424
|
+
index.set(bytesKey(getSnapshotIdByIndex(parsedRepo, i)), i);
|
|
1425
|
+
}
|
|
1426
|
+
snapshotIndexCache.set(parsedRepo, index);
|
|
1427
|
+
}
|
|
1428
|
+
return index;
|
|
1429
|
+
}
|
|
1430
|
+
function bytesKey(bytes) {
|
|
1431
|
+
return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
1432
|
+
}
|
|
1433
|
+
function resolveSnapshotParent(parsedRepo, snapshotId) {
|
|
1434
|
+
const index = snapshotIndexById(parsedRepo).get(bytesKey(snapshotId));
|
|
1435
|
+
if (index === void 0) return void 0;
|
|
1436
|
+
const parentOffset = parsedRepo.repo.snapshots(index).parentOffset();
|
|
1437
|
+
if (parentOffset === -1) return null;
|
|
1438
|
+
if (parentOffset < 0 || parentOffset >= parsedRepo.snapshotsLength) {
|
|
1439
|
+
throw new Error(
|
|
1440
|
+
`Invalid parent offset ${parentOffset} for snapshot at index ${index}, snapshots array has ${parsedRepo.snapshotsLength} entries`
|
|
1441
|
+
);
|
|
1442
|
+
}
|
|
1443
|
+
return getSnapshotIdByIndex(parsedRepo, parentOffset);
|
|
1444
|
+
}
|
|
1403
1445
|
|
|
1404
1446
|
// src/cache/lru.ts
|
|
1405
1447
|
var LRUCache = class {
|
|
@@ -2960,29 +3002,53 @@ function parseTransactionLog(data) {
|
|
|
2960
3002
|
function expectedRangeLength(range) {
|
|
2961
3003
|
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2962
3004
|
}
|
|
3005
|
+
function regionalS3RedirectUrl(url, response) {
|
|
3006
|
+
if (response.status !== 301 && response.status !== 307) return null;
|
|
3007
|
+
const region = response.headers.get("x-amz-bucket-region");
|
|
3008
|
+
if (!region) return null;
|
|
3009
|
+
const globalPrefix = "https://s3.amazonaws.com/";
|
|
3010
|
+
if (!url.startsWith(globalPrefix)) return null;
|
|
3011
|
+
return `https://s3.${region}.amazonaws.com/${url.slice(globalPrefix.length)}`;
|
|
3012
|
+
}
|
|
2963
3013
|
function makeUrlStore(opts) {
|
|
2964
|
-
const {
|
|
2965
|
-
|
|
2966
|
-
|
|
3014
|
+
const { fetchClient, conditionalHeaders } = opts;
|
|
3015
|
+
let pinnedUrl = opts.url;
|
|
3016
|
+
async function doFetch(target, init) {
|
|
3017
|
+
return fetchClient ? await fetchClient.fetch(target, init) : await fetch(target, init);
|
|
2967
3018
|
}
|
|
2968
3019
|
return {
|
|
2969
3020
|
async get() {
|
|
2970
3021
|
throw new Error(
|
|
2971
|
-
`Virtual chunk URL store for ${
|
|
3022
|
+
`Virtual chunk URL store for ${pinnedUrl} only supports ranged reads`
|
|
2972
3023
|
);
|
|
2973
3024
|
},
|
|
2974
3025
|
async getRange(_key, range, options) {
|
|
2975
3026
|
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2976
3027
|
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2977
|
-
|
|
3028
|
+
let requestUrl = pinnedUrl;
|
|
3029
|
+
let response = await doFetch(requestUrl, {
|
|
3030
|
+
headers,
|
|
3031
|
+
signal: options?.signal
|
|
3032
|
+
});
|
|
3033
|
+
const regionalUrl = regionalS3RedirectUrl(requestUrl, response);
|
|
3034
|
+
if (regionalUrl) {
|
|
3035
|
+
response.body?.cancel().catch(() => {
|
|
3036
|
+
});
|
|
3037
|
+
requestUrl = regionalUrl;
|
|
3038
|
+
pinnedUrl = regionalUrl;
|
|
3039
|
+
response = await doFetch(requestUrl, {
|
|
3040
|
+
headers,
|
|
3041
|
+
signal: options?.signal
|
|
3042
|
+
});
|
|
3043
|
+
}
|
|
2978
3044
|
if (response.status === 412) {
|
|
2979
3045
|
throw new Error(
|
|
2980
|
-
`Virtual chunk at ${
|
|
3046
|
+
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2981
3047
|
);
|
|
2982
3048
|
}
|
|
2983
3049
|
if (response.status !== 200 && response.status !== 206) {
|
|
2984
3050
|
throw new Error(
|
|
2985
|
-
`Failed to fetch virtual chunk from ${
|
|
3051
|
+
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`
|
|
2986
3052
|
);
|
|
2987
3053
|
}
|
|
2988
3054
|
const data = new Uint8Array(await response.arrayBuffer());
|
|
@@ -2990,21 +3056,21 @@ function makeUrlStore(opts) {
|
|
|
2990
3056
|
const expected = expectedRangeLength(range);
|
|
2991
3057
|
if (data.length === expected) return data;
|
|
2992
3058
|
throw new Error(
|
|
2993
|
-
`Virtual range response size mismatch for ${
|
|
3059
|
+
`Virtual range response size mismatch for ${requestUrl}: expected ${expected} bytes, got ${data.length}`
|
|
2994
3060
|
);
|
|
2995
3061
|
}
|
|
2996
3062
|
if ("offset" in range) {
|
|
2997
3063
|
const end = range.offset + range.length;
|
|
2998
3064
|
if (data.length >= end) return data.slice(range.offset, end);
|
|
2999
3065
|
throw new Error(
|
|
3000
|
-
`Virtual range request not honored for ${
|
|
3066
|
+
`Virtual range request not honored for ${requestUrl}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
3001
3067
|
);
|
|
3002
3068
|
}
|
|
3003
3069
|
if (data.length >= range.suffixLength) {
|
|
3004
3070
|
return data.slice(data.length - range.suffixLength);
|
|
3005
3071
|
}
|
|
3006
3072
|
throw new Error(
|
|
3007
|
-
`Virtual suffix range request not honored for ${
|
|
3073
|
+
`Virtual suffix range request not honored for ${requestUrl}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
3008
3074
|
);
|
|
3009
3075
|
}
|
|
3010
3076
|
};
|
|
@@ -3072,7 +3138,7 @@ var ReadSession = class _ReadSession {
|
|
|
3072
3138
|
nextFetchClientId = 1;
|
|
3073
3139
|
rangeCoalescerIds;
|
|
3074
3140
|
nextRangeCoalescerId = 1;
|
|
3075
|
-
/**
|
|
3141
|
+
/** Virtual chunk containers (from repo config) for resolving chunk locations. */
|
|
3076
3142
|
virtualChunkContainers;
|
|
3077
3143
|
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
3078
3144
|
this.storage = storage;
|
|
@@ -3080,7 +3146,7 @@ var ReadSession = class _ReadSession {
|
|
|
3080
3146
|
this.specVersion = specVersion;
|
|
3081
3147
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
3082
3148
|
this.manifestLoader = singleFlight(this.manifestCache);
|
|
3083
|
-
this.virtualChunkContainers = virtualChunkContainers ??
|
|
3149
|
+
this.virtualChunkContainers = virtualChunkContainers ?? [];
|
|
3084
3150
|
}
|
|
3085
3151
|
getFetchClientKey(fetchClient) {
|
|
3086
3152
|
if (!fetchClient) return "default";
|
|
@@ -3466,14 +3532,14 @@ var ReadSession = class _ReadSession {
|
|
|
3466
3532
|
);
|
|
3467
3533
|
}
|
|
3468
3534
|
case "virtual": {
|
|
3469
|
-
const absoluteLocation =
|
|
3535
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3470
3536
|
payload.location,
|
|
3471
3537
|
this.virtualChunkContainers
|
|
3472
3538
|
);
|
|
3473
|
-
const httpUrl = translateToHttpUrl(
|
|
3474
|
-
|
|
3475
|
-
|
|
3476
|
-
);
|
|
3539
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3540
|
+
azureAccount: options?.azureAccount,
|
|
3541
|
+
s3: container?.s3
|
|
3542
|
+
});
|
|
3477
3543
|
const store = await this.getVirtualStoreForPayload(
|
|
3478
3544
|
httpUrl,
|
|
3479
3545
|
payload,
|
|
@@ -3536,14 +3602,14 @@ var ReadSession = class _ReadSession {
|
|
|
3536
3602
|
case "virtual": {
|
|
3537
3603
|
const absoluteStart = payload.offset + rangeStart;
|
|
3538
3604
|
const expectedSize = rangeEnd - rangeStart;
|
|
3539
|
-
const absoluteLocation =
|
|
3605
|
+
const { url: absoluteLocation, container } = resolveVirtualChunkLocation(
|
|
3540
3606
|
payload.location,
|
|
3541
3607
|
this.virtualChunkContainers
|
|
3542
3608
|
);
|
|
3543
|
-
const httpUrl = translateToHttpUrl(
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
);
|
|
3609
|
+
const httpUrl = translateToHttpUrl(absoluteLocation, {
|
|
3610
|
+
azureAccount: options?.azureAccount,
|
|
3611
|
+
s3: container?.s3
|
|
3612
|
+
});
|
|
3547
3613
|
const store = await this.getVirtualStoreForPayload(
|
|
3548
3614
|
httpUrl,
|
|
3549
3615
|
payload,
|
|
@@ -3613,44 +3679,62 @@ function compareUtf8Bytes(a, b) {
|
|
|
3613
3679
|
return bytesA.length - bytesB.length;
|
|
3614
3680
|
}
|
|
3615
3681
|
var VCC_SCHEME = "vcc://";
|
|
3616
|
-
function
|
|
3617
|
-
if (
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3682
|
+
function resolveVirtualChunkLocation(location, containers) {
|
|
3683
|
+
if (location.startsWith(VCC_SCHEME)) {
|
|
3684
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
3685
|
+
const slash = rest.indexOf("/");
|
|
3686
|
+
if (slash === -1) {
|
|
3687
|
+
throw new Error(
|
|
3688
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
3689
|
+
);
|
|
3690
|
+
}
|
|
3691
|
+
const name = rest.slice(0, slash);
|
|
3692
|
+
const relativePath = rest.slice(slash + 1);
|
|
3693
|
+
const container2 = containers.find((c) => c.name === name);
|
|
3694
|
+
if (!container2) {
|
|
3695
|
+
if (containers.length === 0) return { url: location };
|
|
3696
|
+
throw new Error(
|
|
3697
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
3698
|
+
);
|
|
3699
|
+
}
|
|
3700
|
+
return {
|
|
3701
|
+
url: ensureTrailingSlash(container2.urlPrefix) + relativePath,
|
|
3702
|
+
container: container2
|
|
3703
|
+
};
|
|
3633
3704
|
}
|
|
3634
|
-
const
|
|
3635
|
-
|
|
3705
|
+
const container = containers.find(
|
|
3706
|
+
(c) => location.startsWith(ensureTrailingSlash(c.urlPrefix))
|
|
3707
|
+
);
|
|
3708
|
+
return { url: location, container };
|
|
3709
|
+
}
|
|
3710
|
+
function ensureTrailingSlash(prefix) {
|
|
3711
|
+
return prefix.endsWith("/") ? prefix : `${prefix}/`;
|
|
3636
3712
|
}
|
|
3637
|
-
function
|
|
3713
|
+
function buildS3HttpUrl(bucket, key, s3) {
|
|
3714
|
+
if (s3?.endpointUrl) {
|
|
3715
|
+
const base = s3.endpointUrl.replace(/\/+$/, "");
|
|
3716
|
+
return `${base}/${bucket}/${key}`;
|
|
3717
|
+
}
|
|
3718
|
+
const region = s3?.region;
|
|
3719
|
+
const pathStyle = s3?.forcePathStyle === true || bucket.includes(".");
|
|
3720
|
+
if (pathStyle) {
|
|
3721
|
+
const host2 = region ? `s3.${region}.${awsEndpointSuffix(region)}` : "s3.amazonaws.com";
|
|
3722
|
+
return `https://${host2}/${bucket}/${key}`;
|
|
3723
|
+
}
|
|
3724
|
+
const host = region ? `${bucket}.s3.${region}.${awsEndpointSuffix(region)}` : `${bucket}.s3.amazonaws.com`;
|
|
3725
|
+
return `https://${host}/${key}`;
|
|
3726
|
+
}
|
|
3727
|
+
function awsEndpointSuffix(region) {
|
|
3728
|
+
return region.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
|
|
3729
|
+
}
|
|
3730
|
+
function translateToHttpUrl(url, options) {
|
|
3731
|
+
const azureAccount = options?.azureAccount;
|
|
3638
3732
|
if (url.startsWith("s3://")) {
|
|
3639
3733
|
const rest = url.slice(5);
|
|
3640
3734
|
const slashIndex = rest.indexOf("/");
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
return `https://s3.amazonaws.com/${bucket2}/`;
|
|
3645
|
-
}
|
|
3646
|
-
return `https://${bucket2}.s3.amazonaws.com/`;
|
|
3647
|
-
}
|
|
3648
|
-
const bucket = rest.slice(0, slashIndex);
|
|
3649
|
-
const key = rest.slice(slashIndex + 1);
|
|
3650
|
-
if (bucket.includes(".")) {
|
|
3651
|
-
return `https://s3.amazonaws.com/${bucket}/${key}`;
|
|
3652
|
-
}
|
|
3653
|
-
return `https://${bucket}.s3.amazonaws.com/${key}`;
|
|
3735
|
+
const bucket = slashIndex === -1 ? rest : rest.slice(0, slashIndex);
|
|
3736
|
+
const key = slashIndex === -1 ? "" : rest.slice(slashIndex + 1);
|
|
3737
|
+
return buildS3HttpUrl(bucket, key, options?.s3);
|
|
3654
3738
|
}
|
|
3655
3739
|
if (url.startsWith("gs://") || url.startsWith("gcs://")) {
|
|
3656
3740
|
const prefixLen = url.startsWith("gs://") ? 5 : 6;
|
|
@@ -4010,18 +4094,37 @@ var Repository = class _Repository {
|
|
|
4010
4094
|
*/
|
|
4011
4095
|
async *walkHistory(session, options) {
|
|
4012
4096
|
let current = session;
|
|
4097
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4013
4098
|
while (current) {
|
|
4099
|
+
const id = encodeObjectId12(current.getSnapshotId());
|
|
4100
|
+
if (seen.has(id)) {
|
|
4101
|
+
throw new Error(`Cycle in snapshot history at ${id}`);
|
|
4102
|
+
}
|
|
4103
|
+
seen.add(id);
|
|
4014
4104
|
yield {
|
|
4015
|
-
id
|
|
4105
|
+
id,
|
|
4016
4106
|
message: current.getMessage(),
|
|
4017
4107
|
flushedAt: current.getFlushedAt(),
|
|
4018
4108
|
metadata: current.getSnapshotMetadata()
|
|
4019
4109
|
};
|
|
4020
|
-
const parentId = current.getParentSnapshotId();
|
|
4110
|
+
const parentId = current.getParentSnapshotId() ?? await this.resolveParentFromRepoInfo(current, options);
|
|
4021
4111
|
if (!parentId) break;
|
|
4022
4112
|
current = await this.checkoutSnapshot(parentId, options);
|
|
4023
4113
|
}
|
|
4024
4114
|
}
|
|
4115
|
+
/** v2 snapshots record ancestry only in the repo file. */
|
|
4116
|
+
async resolveParentFromRepoInfo(session, options) {
|
|
4117
|
+
const repoInfo = await this.loadRepoInfo(options);
|
|
4118
|
+
if (!repoInfo) return null;
|
|
4119
|
+
const snapshotId = session.getSnapshotId();
|
|
4120
|
+
const parent = resolveSnapshotParent(repoInfo, snapshotId);
|
|
4121
|
+
if (parent === void 0) {
|
|
4122
|
+
throw new Error(
|
|
4123
|
+
`Snapshot ${encodeObjectId12(snapshotId)} not found in repo info`
|
|
4124
|
+
);
|
|
4125
|
+
}
|
|
4126
|
+
return parent;
|
|
4127
|
+
}
|
|
4025
4128
|
/**
|
|
4026
4129
|
* Get the storage backend.
|
|
4027
4130
|
*/
|