icechunk-js 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -10
- package/dist/index.cjs +476 -41
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +140 -13
- package/dist/index.d.ts +140 -13
- package/dist/index.js +475 -41
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -42,6 +42,7 @@ __export(index_exports, {
|
|
|
42
42
|
Repository: () => Repository,
|
|
43
43
|
SpecVersion: () => SpecVersion,
|
|
44
44
|
StorageError: () => StorageError,
|
|
45
|
+
createRangeCoalescer: () => createRangeCoalescer,
|
|
45
46
|
decodeBase32: () => decodeBase32,
|
|
46
47
|
decodeObjectId12: () => decodeObjectId12,
|
|
47
48
|
decodeObjectId8: () => decodeObjectId8,
|
|
@@ -65,12 +66,24 @@ var NotFoundError = class extends Error {
|
|
|
65
66
|
}
|
|
66
67
|
};
|
|
67
68
|
var StorageError = class extends Error {
|
|
68
|
-
constructor(message, cause) {
|
|
69
|
+
constructor(message, cause, status) {
|
|
69
70
|
super(message);
|
|
70
71
|
this.cause = cause;
|
|
72
|
+
this.status = status;
|
|
71
73
|
this.name = "StorageError";
|
|
72
74
|
}
|
|
73
75
|
};
|
|
76
|
+
function parseContentRange(response) {
|
|
77
|
+
const value = response.headers?.get?.("content-range");
|
|
78
|
+
if (!value) return null;
|
|
79
|
+
const match = /^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i.exec(value.trim());
|
|
80
|
+
if (!match) return null;
|
|
81
|
+
return {
|
|
82
|
+
start: Number(match[1]),
|
|
83
|
+
end: Number(match[2]),
|
|
84
|
+
size: match[3] === "*" ? null : Number(match[3])
|
|
85
|
+
};
|
|
86
|
+
}
|
|
74
87
|
function hasErrorName(error) {
|
|
75
88
|
return typeof error === "object" && error !== null && "name" in error;
|
|
76
89
|
}
|
|
@@ -92,6 +105,7 @@ var PATHS = {
|
|
|
92
105
|
TRANSACTIONS: "transactions"
|
|
93
106
|
};
|
|
94
107
|
var REPO_INFO_PATH = "repo";
|
|
108
|
+
var CONFIG_PATH = "config.yaml";
|
|
95
109
|
function getBranchRefDirPath(name) {
|
|
96
110
|
return `${PATHS.REFS}/branch.${name}/`;
|
|
97
111
|
}
|
|
@@ -1292,6 +1306,10 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1292
1306
|
} catch {
|
|
1293
1307
|
return result;
|
|
1294
1308
|
}
|
|
1309
|
+
return virtualChunkContainersFromConfig(config);
|
|
1310
|
+
}
|
|
1311
|
+
function virtualChunkContainersFromConfig(config) {
|
|
1312
|
+
const result = [];
|
|
1295
1313
|
if (!isRecord(config)) return result;
|
|
1296
1314
|
const containers = config.virtual_chunk_containers;
|
|
1297
1315
|
if (!isRecord(containers)) return result;
|
|
@@ -1443,6 +1461,81 @@ function resolveSnapshotParent(parsedRepo, snapshotId) {
|
|
|
1443
1461
|
return getSnapshotIdByIndex(parsedRepo, parentOffset);
|
|
1444
1462
|
}
|
|
1445
1463
|
|
|
1464
|
+
// src/format/config-yaml.ts
|
|
1465
|
+
var SECTION = "virtual_chunk_containers";
|
|
1466
|
+
var ENTRY = /^('(?:[^']|'')*'|"(?:[^"\\]|\\.)*"|.+?):(?: +(.*))?$/;
|
|
1467
|
+
function parseConfigYamlContainers(yaml) {
|
|
1468
|
+
const lines = yaml.split(/\r?\n/);
|
|
1469
|
+
const start = lines.findIndex((line) => line.startsWith(`${SECTION}:`));
|
|
1470
|
+
if (start < 0) return [];
|
|
1471
|
+
const body = [];
|
|
1472
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
1473
|
+
if (/^\S/.test(lines[i])) break;
|
|
1474
|
+
if (lines[i].trim() !== "") body.push(lines[i]);
|
|
1475
|
+
}
|
|
1476
|
+
for (let i = 0; i + 1 < body.length; i++) {
|
|
1477
|
+
const indent = indentOf(body[i]);
|
|
1478
|
+
const value = body[i + 1].slice(indent);
|
|
1479
|
+
if (body[i].startsWith("? ", indent) && indentOf(body[i + 1]) === indent && value.startsWith(": ")) {
|
|
1480
|
+
body[i] = `${" ".repeat(indent)}${JSON.stringify(body[i].slice(indent + 2))}:`;
|
|
1481
|
+
body[i + 1] = `${" ".repeat(indent)} ${value.slice(2)}`;
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
try {
|
|
1485
|
+
const [containers] = parseMapping(body, 0);
|
|
1486
|
+
return virtualChunkContainersFromConfig({ [SECTION]: containers });
|
|
1487
|
+
} catch {
|
|
1488
|
+
return [];
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
function parseMapping(lines, from) {
|
|
1492
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
1493
|
+
if (from >= lines.length) return [result, from];
|
|
1494
|
+
const indent = indentOf(lines[from]);
|
|
1495
|
+
let i = from;
|
|
1496
|
+
while (i < lines.length && indentOf(lines[i]) >= indent) {
|
|
1497
|
+
const entry = indentOf(lines[i]) === indent ? ENTRY.exec(lines[i].slice(indent)) : null;
|
|
1498
|
+
if (!entry || /^[-?:] /.test(entry[1])) {
|
|
1499
|
+
throw new Error(`Unsupported config.yaml line: ${lines[i]}`);
|
|
1500
|
+
}
|
|
1501
|
+
i++;
|
|
1502
|
+
const key = unquote(entry[1]);
|
|
1503
|
+
const tagged = /^!(\S+)(?: +(.*))?$/.exec(entry[2] ?? "");
|
|
1504
|
+
const rest = (tagged ? tagged[2] ?? "" : entry[2] ?? "").trim();
|
|
1505
|
+
let value;
|
|
1506
|
+
if (rest === "" && i < lines.length && indentOf(lines[i]) > indent) {
|
|
1507
|
+
[value, i] = parseMapping(lines, i);
|
|
1508
|
+
} else {
|
|
1509
|
+
value = parseScalar(rest);
|
|
1510
|
+
}
|
|
1511
|
+
result[key] = tagged ? { [tagged[1]]: value } : value;
|
|
1512
|
+
}
|
|
1513
|
+
return [result, i];
|
|
1514
|
+
}
|
|
1515
|
+
function indentOf(line) {
|
|
1516
|
+
return line.length - line.trimStart().length;
|
|
1517
|
+
}
|
|
1518
|
+
function parseScalar(text) {
|
|
1519
|
+
if (text === "" || text === "null" || text === "~") return null;
|
|
1520
|
+
if (text === "true") return true;
|
|
1521
|
+
if (text === "false") return false;
|
|
1522
|
+
if (text === "{}") return {};
|
|
1523
|
+
const quoted = /^['"]/.test(text);
|
|
1524
|
+
if (/^[[{|>&*]/.test(text) || !quoted && /(^| )#/.test(text)) {
|
|
1525
|
+
throw new Error(`Unsupported config.yaml value: ${text}`);
|
|
1526
|
+
}
|
|
1527
|
+
return unquote(text);
|
|
1528
|
+
}
|
|
1529
|
+
function unquote(text) {
|
|
1530
|
+
if (text.length >= 2 && text.startsWith("'") && text.endsWith("'")) {
|
|
1531
|
+
return text.slice(1, -1).replace(/''/g, "'");
|
|
1532
|
+
}
|
|
1533
|
+
if (text.length >= 2 && text.startsWith('"') && text.endsWith('"')) {
|
|
1534
|
+
return JSON.parse(text);
|
|
1535
|
+
}
|
|
1536
|
+
return text;
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1446
1539
|
// src/cache/lru.ts
|
|
1447
1540
|
var LRUCache = class {
|
|
1448
1541
|
cache;
|
|
@@ -3002,6 +3095,25 @@ function parseTransactionLog(data) {
|
|
|
3002
3095
|
function expectedRangeLength(range) {
|
|
3003
3096
|
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
3004
3097
|
}
|
|
3098
|
+
function describeRange(range) {
|
|
3099
|
+
return "suffixLength" in range ? `last ${range.suffixLength} bytes` : `bytes ${range.offset}-${range.offset + range.length - 1}`;
|
|
3100
|
+
}
|
|
3101
|
+
function httpError(message, status) {
|
|
3102
|
+
return Object.assign(new Error(message), { status });
|
|
3103
|
+
}
|
|
3104
|
+
function assertContentRangeMatches(url, range, response) {
|
|
3105
|
+
const got = parseContentRange(response);
|
|
3106
|
+
if (!got) return;
|
|
3107
|
+
const matches = "suffixLength" in range ? got.end - got.start + 1 === range.suffixLength && (got.size === null || got.end === got.size - 1) : got.start === range.offset && got.end === range.offset + range.length - 1;
|
|
3108
|
+
if (!matches) {
|
|
3109
|
+
throw new Error(
|
|
3110
|
+
`Range response for ${url} covers bytes ${got.start}-${got.end}, not the requested ${describeRange(range)}`
|
|
3111
|
+
);
|
|
3112
|
+
}
|
|
3113
|
+
}
|
|
3114
|
+
function copyBytes(data, start, end) {
|
|
3115
|
+
return new Uint8Array(data.subarray(start, end));
|
|
3116
|
+
}
|
|
3005
3117
|
function regionalS3RedirectUrl(url, response) {
|
|
3006
3118
|
if (response.status !== 301 && response.status !== 307) return null;
|
|
3007
3119
|
const region = response.headers.get("x-amz-bucket-region");
|
|
@@ -3042,17 +3154,20 @@ function makeUrlStore(opts) {
|
|
|
3042
3154
|
});
|
|
3043
3155
|
}
|
|
3044
3156
|
if (response.status === 412) {
|
|
3045
|
-
throw
|
|
3046
|
-
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created
|
|
3157
|
+
throw httpError(
|
|
3158
|
+
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`,
|
|
3159
|
+
412
|
|
3047
3160
|
);
|
|
3048
3161
|
}
|
|
3049
3162
|
if (response.status !== 200 && response.status !== 206) {
|
|
3050
|
-
throw
|
|
3051
|
-
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}
|
|
3163
|
+
throw httpError(
|
|
3164
|
+
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`,
|
|
3165
|
+
response.status
|
|
3052
3166
|
);
|
|
3053
3167
|
}
|
|
3054
3168
|
const data = new Uint8Array(await response.arrayBuffer());
|
|
3055
3169
|
if (response.status === 206) {
|
|
3170
|
+
assertContentRangeMatches(requestUrl, range, response);
|
|
3056
3171
|
const expected = expectedRangeLength(range);
|
|
3057
3172
|
if (data.length === expected) return data;
|
|
3058
3173
|
throw new Error(
|
|
@@ -3095,7 +3210,7 @@ function makeStorageStore(storage) {
|
|
|
3095
3210
|
const data = await storage.getObject(key, storageRange, storageOptions);
|
|
3096
3211
|
if (data.length === range.length) return data;
|
|
3097
3212
|
if (data.length >= storageRange.end) {
|
|
3098
|
-
return data
|
|
3213
|
+
return copyBytes(data, storageRange.start, storageRange.end);
|
|
3099
3214
|
}
|
|
3100
3215
|
throw new Error(
|
|
3101
3216
|
`Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
|
|
@@ -3103,6 +3218,238 @@ function makeStorageStore(storage) {
|
|
|
3103
3218
|
}
|
|
3104
3219
|
};
|
|
3105
3220
|
}
|
|
3221
|
+
var DEFAULT_MAX_GAP = 256 * 1024;
|
|
3222
|
+
var DEFAULT_MAX_BYTES = 16 << 20;
|
|
3223
|
+
var DEFAULT_MAX_WASTE_RATIO = 1;
|
|
3224
|
+
var COALESCER_BRAND = /* @__PURE__ */ Symbol("icechunk-js.rangeCoalescer");
|
|
3225
|
+
var WHOLE_OBJECT_STATUSES = /* @__PURE__ */ new Set([401, 403, 404, 412]);
|
|
3226
|
+
function isRangeCoalescer(fn) {
|
|
3227
|
+
return typeof fn === "function" && fn[COALESCER_BRAND] === true;
|
|
3228
|
+
}
|
|
3229
|
+
function freshStats() {
|
|
3230
|
+
return {
|
|
3231
|
+
ranges: 0,
|
|
3232
|
+
requests: 0,
|
|
3233
|
+
bytesRequested: 0,
|
|
3234
|
+
bytesFetched: 0,
|
|
3235
|
+
bytesDiscarded: 0,
|
|
3236
|
+
fallbacks: 0
|
|
3237
|
+
};
|
|
3238
|
+
}
|
|
3239
|
+
function nonNegative(name, value, allowInfinite = false) {
|
|
3240
|
+
const valid = allowInfinite ? !Number.isNaN(value) && value >= 0 : Number.isFinite(value) && value >= 0;
|
|
3241
|
+
if (!valid) {
|
|
3242
|
+
throw new RangeError(`coalesce.${name} must be a non-negative number`);
|
|
3243
|
+
}
|
|
3244
|
+
return value;
|
|
3245
|
+
}
|
|
3246
|
+
function abortReason(signal) {
|
|
3247
|
+
return signal.reason ?? new DOMException("The operation was aborted.", "AbortError");
|
|
3248
|
+
}
|
|
3249
|
+
function settle(waiter, outcome) {
|
|
3250
|
+
if (waiter.settled) return;
|
|
3251
|
+
waiter.settled = true;
|
|
3252
|
+
waiter.detach?.();
|
|
3253
|
+
waiter.detach = void 0;
|
|
3254
|
+
waiter.onAbort = void 0;
|
|
3255
|
+
outcome();
|
|
3256
|
+
}
|
|
3257
|
+
function failsWholeObject(error) {
|
|
3258
|
+
if (error instanceof NotFoundError) return true;
|
|
3259
|
+
const status = error?.status;
|
|
3260
|
+
return typeof status === "number" && WHOLE_OBJECT_STATUSES.has(status);
|
|
3261
|
+
}
|
|
3262
|
+
function createRangeCoalescer(options = {}) {
|
|
3263
|
+
const resolved = {
|
|
3264
|
+
maxGap: nonNegative("maxGap", options.maxGap ?? DEFAULT_MAX_GAP),
|
|
3265
|
+
maxBytes: nonNegative("maxBytes", options.maxBytes ?? DEFAULT_MAX_BYTES),
|
|
3266
|
+
maxWasteRatio: nonNegative(
|
|
3267
|
+
"maxWasteRatio",
|
|
3268
|
+
options.maxWasteRatio ?? DEFAULT_MAX_WASTE_RATIO,
|
|
3269
|
+
true
|
|
3270
|
+
),
|
|
3271
|
+
window: nonNegative("window", options.window ?? 0)
|
|
3272
|
+
};
|
|
3273
|
+
const stats = freshStats();
|
|
3274
|
+
const wrap = (store) => wrapStore(store, resolved, stats);
|
|
3275
|
+
return Object.assign(wrap, {
|
|
3276
|
+
[COALESCER_BRAND]: true,
|
|
3277
|
+
stats,
|
|
3278
|
+
resetStats() {
|
|
3279
|
+
Object.assign(stats, freshStats());
|
|
3280
|
+
}
|
|
3281
|
+
});
|
|
3282
|
+
}
|
|
3283
|
+
function wrapStore(store, options, stats) {
|
|
3284
|
+
if (typeof store.getRange !== "function") {
|
|
3285
|
+
throw new Error("Range coalescing requires a store with getRange");
|
|
3286
|
+
}
|
|
3287
|
+
const inner = store.getRange.bind(store);
|
|
3288
|
+
const pending = /* @__PURE__ */ new Map();
|
|
3289
|
+
let scheduled = false;
|
|
3290
|
+
function flush() {
|
|
3291
|
+
scheduled = false;
|
|
3292
|
+
const work = [...pending];
|
|
3293
|
+
pending.clear();
|
|
3294
|
+
for (const [key, reads] of work) {
|
|
3295
|
+
const live = reads.filter((read) => !read.waiter.settled);
|
|
3296
|
+
for (const run of planRuns(live, options)) {
|
|
3297
|
+
const length = run.end - run.offset;
|
|
3298
|
+
stats.bytesDiscarded += length - run.delivered;
|
|
3299
|
+
void issue(inner, stats, key, run.offset, length, run.members);
|
|
3300
|
+
}
|
|
3301
|
+
}
|
|
3302
|
+
}
|
|
3303
|
+
function schedule() {
|
|
3304
|
+
if (scheduled) return;
|
|
3305
|
+
scheduled = true;
|
|
3306
|
+
if (options.window > 0) setTimeout(flush, options.window);
|
|
3307
|
+
else queueMicrotask(flush);
|
|
3308
|
+
}
|
|
3309
|
+
return {
|
|
3310
|
+
get: (key, getOptions) => store.get(key, getOptions),
|
|
3311
|
+
getRange(key, range, getOptions) {
|
|
3312
|
+
if ("suffixLength" in range) return inner(key, range, getOptions);
|
|
3313
|
+
const signal = getOptions?.signal;
|
|
3314
|
+
if (signal?.aborted) return Promise.reject(abortReason(signal));
|
|
3315
|
+
stats.ranges++;
|
|
3316
|
+
stats.bytesRequested += range.length;
|
|
3317
|
+
return new Promise((resolve, reject) => {
|
|
3318
|
+
const waiter = { signal, settled: false, resolve, reject };
|
|
3319
|
+
if (signal) {
|
|
3320
|
+
const onAbort = () => {
|
|
3321
|
+
const hook = waiter.onAbort;
|
|
3322
|
+
settle(waiter, () => waiter.reject(abortReason(signal)));
|
|
3323
|
+
hook?.();
|
|
3324
|
+
};
|
|
3325
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
3326
|
+
waiter.detach = () => signal.removeEventListener("abort", onAbort);
|
|
3327
|
+
}
|
|
3328
|
+
let reads = pending.get(key);
|
|
3329
|
+
if (!reads) {
|
|
3330
|
+
reads = [];
|
|
3331
|
+
pending.set(key, reads);
|
|
3332
|
+
}
|
|
3333
|
+
reads.push({ offset: range.offset, length: range.length, waiter });
|
|
3334
|
+
schedule();
|
|
3335
|
+
});
|
|
3336
|
+
}
|
|
3337
|
+
};
|
|
3338
|
+
}
|
|
3339
|
+
function planRuns(reads, options) {
|
|
3340
|
+
reads.sort((a, b) => a.offset - b.offset || a.length - b.length);
|
|
3341
|
+
const members = [];
|
|
3342
|
+
for (const read of reads) {
|
|
3343
|
+
const last = members[members.length - 1];
|
|
3344
|
+
if (last && last.offset === read.offset && last.length === read.length) {
|
|
3345
|
+
last.waiters.push(read.waiter);
|
|
3346
|
+
} else {
|
|
3347
|
+
members.push({
|
|
3348
|
+
offset: read.offset,
|
|
3349
|
+
length: read.length,
|
|
3350
|
+
waiters: [read.waiter]
|
|
3351
|
+
});
|
|
3352
|
+
}
|
|
3353
|
+
}
|
|
3354
|
+
const runs = [];
|
|
3355
|
+
let run;
|
|
3356
|
+
for (const member of members) {
|
|
3357
|
+
const memberEnd = member.offset + member.length;
|
|
3358
|
+
if (run) {
|
|
3359
|
+
const added = Math.max(0, memberEnd - Math.max(member.offset, run.end));
|
|
3360
|
+
const end = Math.max(run.end, memberEnd);
|
|
3361
|
+
const length = end - run.offset;
|
|
3362
|
+
const delivered = run.delivered + added;
|
|
3363
|
+
if (member.offset - run.end <= options.maxGap && length <= options.maxBytes && length - delivered <= options.maxWasteRatio * delivered) {
|
|
3364
|
+
run.end = end;
|
|
3365
|
+
run.delivered = delivered;
|
|
3366
|
+
run.members.push(member);
|
|
3367
|
+
continue;
|
|
3368
|
+
}
|
|
3369
|
+
}
|
|
3370
|
+
run = {
|
|
3371
|
+
offset: member.offset,
|
|
3372
|
+
end: memberEnd,
|
|
3373
|
+
members: [member],
|
|
3374
|
+
delivered: member.length
|
|
3375
|
+
};
|
|
3376
|
+
runs.push(run);
|
|
3377
|
+
}
|
|
3378
|
+
return runs;
|
|
3379
|
+
}
|
|
3380
|
+
async function issue(inner, stats, key, offset, length, members) {
|
|
3381
|
+
const waiters = [];
|
|
3382
|
+
for (const member of members) {
|
|
3383
|
+
for (const waiter of member.waiters) {
|
|
3384
|
+
if (!waiter.settled) waiters.push(waiter);
|
|
3385
|
+
}
|
|
3386
|
+
}
|
|
3387
|
+
if (waiters.length === 0) return;
|
|
3388
|
+
let signal;
|
|
3389
|
+
let controller;
|
|
3390
|
+
if (waiters.length === 1) {
|
|
3391
|
+
signal = waiters[0].signal;
|
|
3392
|
+
} else if (waiters.some((waiter) => waiter.signal)) {
|
|
3393
|
+
const shared = new AbortController();
|
|
3394
|
+
controller = shared;
|
|
3395
|
+
signal = shared.signal;
|
|
3396
|
+
let live = waiters.length;
|
|
3397
|
+
for (const waiter of waiters) {
|
|
3398
|
+
const waiterSignal = waiter.signal;
|
|
3399
|
+
if (!waiterSignal) continue;
|
|
3400
|
+
waiter.onAbort = () => {
|
|
3401
|
+
if (--live === 0) shared.abort(waiterSignal.reason);
|
|
3402
|
+
};
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
3405
|
+
stats.requests++;
|
|
3406
|
+
stats.bytesFetched += length;
|
|
3407
|
+
let data;
|
|
3408
|
+
try {
|
|
3409
|
+
data = await inner(
|
|
3410
|
+
key,
|
|
3411
|
+
{ offset, length },
|
|
3412
|
+
signal ? { signal } : void 0
|
|
3413
|
+
);
|
|
3414
|
+
if (members.length > 1 && data && data.length < length) {
|
|
3415
|
+
throw new Error(
|
|
3416
|
+
`Short read for ${key} at ${offset}: expected ${length} bytes, got ${data.length}`
|
|
3417
|
+
);
|
|
3418
|
+
}
|
|
3419
|
+
} catch (error) {
|
|
3420
|
+
if (members.length > 1 && !controller?.signal.aborted && !failsWholeObject(error)) {
|
|
3421
|
+
stats.fallbacks++;
|
|
3422
|
+
for (const waiter of waiters) waiter.onAbort = void 0;
|
|
3423
|
+
await Promise.all(
|
|
3424
|
+
members.map(
|
|
3425
|
+
(member) => issue(inner, stats, key, member.offset, member.length, [member])
|
|
3426
|
+
)
|
|
3427
|
+
);
|
|
3428
|
+
return;
|
|
3429
|
+
}
|
|
3430
|
+
for (const waiter of waiters) settle(waiter, () => waiter.reject(error));
|
|
3431
|
+
return;
|
|
3432
|
+
}
|
|
3433
|
+
for (const member of members) {
|
|
3434
|
+
const start = member.offset - offset;
|
|
3435
|
+
let view;
|
|
3436
|
+
for (const waiter of member.waiters) {
|
|
3437
|
+
if (waiter.settled) continue;
|
|
3438
|
+
if (!data) {
|
|
3439
|
+
settle(waiter, () => waiter.resolve(void 0));
|
|
3440
|
+
continue;
|
|
3441
|
+
}
|
|
3442
|
+
if (!view) {
|
|
3443
|
+
view = members.length === 1 ? data : copyBytes(data, start, start + member.length);
|
|
3444
|
+
const first = view;
|
|
3445
|
+
settle(waiter, () => waiter.resolve(first));
|
|
3446
|
+
} else {
|
|
3447
|
+
const copy = new Uint8Array(view);
|
|
3448
|
+
settle(waiter, () => waiter.resolve(copy));
|
|
3449
|
+
}
|
|
3450
|
+
}
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3106
3453
|
|
|
3107
3454
|
// src/reader/session.ts
|
|
3108
3455
|
var RANGE_COALESCE_SIZE = 32 * 1024;
|
|
@@ -3780,6 +4127,7 @@ var Repository = class _Repository {
|
|
|
3780
4127
|
storage;
|
|
3781
4128
|
repoInfo = null;
|
|
3782
4129
|
repoInfoAttempted = false;
|
|
4130
|
+
v1Containers = null;
|
|
3783
4131
|
constructor(storage) {
|
|
3784
4132
|
this.storage = storage;
|
|
3785
4133
|
}
|
|
@@ -3801,11 +4149,10 @@ var Repository = class _Repository {
|
|
|
3801
4149
|
}
|
|
3802
4150
|
this.repoInfoAttempted = true;
|
|
3803
4151
|
try {
|
|
3804
|
-
const data = await this.storage.getObject(
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
);
|
|
4152
|
+
const data = await this.storage.getObject(REPO_INFO_PATH, void 0, {
|
|
4153
|
+
...options,
|
|
4154
|
+
cache: "no-cache"
|
|
4155
|
+
});
|
|
3809
4156
|
this.repoInfo = parseRepo(data);
|
|
3810
4157
|
return this.repoInfo;
|
|
3811
4158
|
} catch (error) {
|
|
@@ -3821,6 +4168,42 @@ var Repository = class _Repository {
|
|
|
3821
4168
|
);
|
|
3822
4169
|
}
|
|
3823
4170
|
}
|
|
4171
|
+
/**
|
|
4172
|
+
* Load and cache the virtual chunk containers of a v1 repository, which
|
|
4173
|
+
* keeps its config in `config.yaml` rather than in a repo file.
|
|
4174
|
+
*
|
|
4175
|
+
* Best-effort, like the v2 config. A repository that never saved a config
|
|
4176
|
+
* has no such file (S3 answers 403 rather than 404 for a missing key of a
|
|
4177
|
+
* bucket that cannot be listed), and a file that cannot be parsed declares
|
|
4178
|
+
* no containers; chunk locations then resolve as they would without a
|
|
4179
|
+
* config. Any other failure also yields no containers but is not cached, so
|
|
4180
|
+
* the next checkout asks again.
|
|
4181
|
+
*
|
|
4182
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
4183
|
+
*/
|
|
4184
|
+
async loadV1Containers(options) {
|
|
4185
|
+
if (this.v1Containers) {
|
|
4186
|
+
return this.v1Containers;
|
|
4187
|
+
}
|
|
4188
|
+
let containers = [];
|
|
4189
|
+
let settled = true;
|
|
4190
|
+
try {
|
|
4191
|
+
const data = await this.storage.getObject(CONFIG_PATH, void 0, {
|
|
4192
|
+
...options,
|
|
4193
|
+
cache: "no-cache"
|
|
4194
|
+
});
|
|
4195
|
+
containers = parseConfigYamlContainers(new TextDecoder().decode(data));
|
|
4196
|
+
} catch (error) {
|
|
4197
|
+
if (isAbortError(error)) {
|
|
4198
|
+
throw error;
|
|
4199
|
+
}
|
|
4200
|
+
settled = error instanceof NotFoundError || error instanceof StorageError && error.status === 403;
|
|
4201
|
+
}
|
|
4202
|
+
if (settled) {
|
|
4203
|
+
this.v1Containers = containers;
|
|
4204
|
+
}
|
|
4205
|
+
return containers;
|
|
4206
|
+
}
|
|
3824
4207
|
/**
|
|
3825
4208
|
* Open an icechunk repository.
|
|
3826
4209
|
*
|
|
@@ -4025,16 +4408,25 @@ var Repository = class _Repository {
|
|
|
4025
4408
|
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
4026
4409
|
});
|
|
4027
4410
|
}
|
|
4028
|
-
const
|
|
4029
|
-
|
|
4030
|
-
|
|
4031
|
-
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
|
|
4037
|
-
|
|
4411
|
+
const resolveSnapshotId = async () => {
|
|
4412
|
+
const refDirPath = getBranchRefDirPath(name);
|
|
4413
|
+
const refPath = await this.findLatestRefFile(
|
|
4414
|
+
refDirPath,
|
|
4415
|
+
getBranchRefPath(name)
|
|
4416
|
+
);
|
|
4417
|
+
if (!refPath) {
|
|
4418
|
+
throw new Error(`Reference not found: ${refDirPath}`);
|
|
4419
|
+
}
|
|
4420
|
+
return this.readSnapshotIdFromRef(refPath, options);
|
|
4421
|
+
};
|
|
4422
|
+
const [snapshotId, virtualChunkContainers] = await Promise.all([
|
|
4423
|
+
resolveSnapshotId(),
|
|
4424
|
+
this.loadV1Containers(options)
|
|
4425
|
+
]);
|
|
4426
|
+
return ReadSession.open(this.storage, snapshotId, {
|
|
4427
|
+
...options,
|
|
4428
|
+
virtualChunkContainers
|
|
4429
|
+
});
|
|
4038
4430
|
}
|
|
4039
4431
|
/**
|
|
4040
4432
|
* Checkout a tag to get a read session.
|
|
@@ -4055,17 +4447,26 @@ var Repository = class _Repository {
|
|
|
4055
4447
|
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
4056
4448
|
});
|
|
4057
4449
|
}
|
|
4058
|
-
const
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
|
|
4067
|
-
|
|
4068
|
-
|
|
4450
|
+
const resolveSnapshotId = async () => {
|
|
4451
|
+
const refDirPath = getTagRefDirPath(name);
|
|
4452
|
+
const legacyPath = getTagRefPath(name);
|
|
4453
|
+
const refPath = await this.findLatestRefFile(refDirPath, legacyPath);
|
|
4454
|
+
if (!refPath) {
|
|
4455
|
+
throw new Error(`Reference not found: ${refDirPath}`);
|
|
4456
|
+
}
|
|
4457
|
+
if (refPath === legacyPath && await this.storage.exists(`${refPath}.deleted`, options)) {
|
|
4458
|
+
throw new Error(`Tag not found: ${name}`);
|
|
4459
|
+
}
|
|
4460
|
+
return this.readSnapshotIdFromRef(refPath, options);
|
|
4461
|
+
};
|
|
4462
|
+
const [snapshotId, virtualChunkContainers] = await Promise.all([
|
|
4463
|
+
resolveSnapshotId(),
|
|
4464
|
+
this.loadV1Containers(options)
|
|
4465
|
+
]);
|
|
4466
|
+
return ReadSession.open(this.storage, snapshotId, {
|
|
4467
|
+
...options,
|
|
4468
|
+
virtualChunkContainers
|
|
4469
|
+
});
|
|
4069
4470
|
}
|
|
4070
4471
|
/**
|
|
4071
4472
|
* Checkout a specific snapshot by ID.
|
|
@@ -4077,10 +4478,9 @@ var Repository = class _Repository {
|
|
|
4077
4478
|
async checkoutSnapshot(snapshotId, options) {
|
|
4078
4479
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
4079
4480
|
const repoInfo = await this.loadRepoInfo(options);
|
|
4080
|
-
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
4081
4481
|
return ReadSession.open(this.storage, id, {
|
|
4082
4482
|
...options,
|
|
4083
|
-
|
|
4483
|
+
virtualChunkContainers: repoInfo ? repoInfo.virtualChunkContainers : await this.loadV1Containers(options)
|
|
4084
4484
|
});
|
|
4085
4485
|
}
|
|
4086
4486
|
/**
|
|
@@ -4133,7 +4533,10 @@ var Repository = class _Repository {
|
|
|
4133
4533
|
}
|
|
4134
4534
|
/** Read and parse a ref file */
|
|
4135
4535
|
async readRef(path, options) {
|
|
4136
|
-
const data = await this.storage.getObject(path, void 0,
|
|
4536
|
+
const data = await this.storage.getObject(path, void 0, {
|
|
4537
|
+
...options,
|
|
4538
|
+
cache: "no-cache"
|
|
4539
|
+
});
|
|
4137
4540
|
const json = new TextDecoder().decode(data);
|
|
4138
4541
|
return JSON.parse(json);
|
|
4139
4542
|
}
|
|
@@ -4188,7 +4591,7 @@ var HttpStorage = class {
|
|
|
4188
4591
|
method: "GET",
|
|
4189
4592
|
headers,
|
|
4190
4593
|
credentials: this.options.credentials,
|
|
4191
|
-
cache: this.options.cache,
|
|
4594
|
+
cache: options?.cache ?? this.options.cache,
|
|
4192
4595
|
signal: options?.signal
|
|
4193
4596
|
});
|
|
4194
4597
|
} catch (error) {
|
|
@@ -4203,11 +4606,28 @@ var HttpStorage = class {
|
|
|
4203
4606
|
}
|
|
4204
4607
|
if (response.status !== 200 && response.status !== 206) {
|
|
4205
4608
|
throw new StorageError(
|
|
4206
|
-
`HTTP ${response.status} ${response.statusText} for ${url}
|
|
4609
|
+
`HTTP ${response.status} ${response.statusText} for ${url}`,
|
|
4610
|
+
void 0,
|
|
4611
|
+
response.status
|
|
4207
4612
|
);
|
|
4208
4613
|
}
|
|
4209
|
-
const
|
|
4210
|
-
|
|
4614
|
+
const body = new Uint8Array(await response.arrayBuffer());
|
|
4615
|
+
if (!range) return body;
|
|
4616
|
+
if (response.status === 206) {
|
|
4617
|
+
const got = parseContentRange(response);
|
|
4618
|
+
if (got && (got.start !== range.start || got.end !== range.end - 1)) {
|
|
4619
|
+
throw new StorageError(
|
|
4620
|
+
`Range response for ${url} covers bytes ${got.start}-${got.end}, not the requested ${range.start}-${range.end - 1}`
|
|
4621
|
+
);
|
|
4622
|
+
}
|
|
4623
|
+
return body;
|
|
4624
|
+
}
|
|
4625
|
+
if (body.length >= range.end) {
|
|
4626
|
+
return new Uint8Array(body.subarray(range.start, range.end));
|
|
4627
|
+
}
|
|
4628
|
+
throw new StorageError(
|
|
4629
|
+
`Range request not honored for ${url}: need at least ${range.end} bytes to slice the requested range, got ${body.length}`
|
|
4630
|
+
);
|
|
4211
4631
|
}
|
|
4212
4632
|
async exists(path, options) {
|
|
4213
4633
|
options?.signal?.throwIfAborted();
|
|
@@ -4254,13 +4674,14 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
4254
4674
|
this.withRangeCoalescing = withRangeCoalescing;
|
|
4255
4675
|
}
|
|
4256
4676
|
static async open(arg, options = {}) {
|
|
4677
|
+
const withRangeCoalescing = resolveRangeCoalescing(options);
|
|
4257
4678
|
if (arg instanceof ReadSession) {
|
|
4258
4679
|
return new _IcechunkStore(
|
|
4259
4680
|
arg,
|
|
4260
4681
|
options.fetchClient,
|
|
4261
4682
|
options.validateChecksums,
|
|
4262
4683
|
options.azureAccount,
|
|
4263
|
-
|
|
4684
|
+
withRangeCoalescing
|
|
4264
4685
|
);
|
|
4265
4686
|
}
|
|
4266
4687
|
const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
|
|
@@ -4289,9 +4710,17 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
4289
4710
|
options.fetchClient,
|
|
4290
4711
|
options.validateChecksums,
|
|
4291
4712
|
options.azureAccount,
|
|
4292
|
-
|
|
4713
|
+
withRangeCoalescing
|
|
4293
4714
|
);
|
|
4294
4715
|
}
|
|
4716
|
+
/**
|
|
4717
|
+
* The built-in range coalescer this store reads chunks through, with its
|
|
4718
|
+
* counters (`stats`: requests issued, bytes fetched and discarded, ...).
|
|
4719
|
+
* `undefined` when coalescing is off or a custom wrapper was supplied.
|
|
4720
|
+
*/
|
|
4721
|
+
get rangeCoalescer() {
|
|
4722
|
+
return isRangeCoalescer(this.withRangeCoalescing) ? this.withRangeCoalescing : void 0;
|
|
4723
|
+
}
|
|
4295
4724
|
/**
|
|
4296
4725
|
* Get data for a zarr key.
|
|
4297
4726
|
*
|
|
@@ -4455,6 +4884,11 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
4455
4884
|
return this.session.getMetadata(fullPath);
|
|
4456
4885
|
}
|
|
4457
4886
|
};
|
|
4887
|
+
function resolveRangeCoalescing(options) {
|
|
4888
|
+
if (options.withRangeCoalescing) return options.withRangeCoalescing;
|
|
4889
|
+
if (options.coalesce === false) return void 0;
|
|
4890
|
+
return createRangeCoalescer(options.coalesce);
|
|
4891
|
+
}
|
|
4458
4892
|
function parseZarrKey(key) {
|
|
4459
4893
|
const path = key.slice(1);
|
|
4460
4894
|
if (path === "zarr.json") {
|
|
@@ -4487,6 +4921,7 @@ function parseZarrKey(key) {
|
|
|
4487
4921
|
Repository,
|
|
4488
4922
|
SpecVersion,
|
|
4489
4923
|
StorageError,
|
|
4924
|
+
createRangeCoalescer,
|
|
4490
4925
|
decodeBase32,
|
|
4491
4926
|
decodeObjectId12,
|
|
4492
4927
|
decodeObjectId8,
|