icechunk-js 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -10
- package/dist/index.cjs +476 -41
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +140 -13
- package/dist/index.d.ts +140 -13
- package/dist/index.js +475 -41
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -12,12 +12,24 @@ var NotFoundError = class extends Error {
|
|
|
12
12
|
}
|
|
13
13
|
};
|
|
14
14
|
var StorageError = class extends Error {
|
|
15
|
-
constructor(message, cause) {
|
|
15
|
+
constructor(message, cause, status) {
|
|
16
16
|
super(message);
|
|
17
17
|
this.cause = cause;
|
|
18
|
+
this.status = status;
|
|
18
19
|
this.name = "StorageError";
|
|
19
20
|
}
|
|
20
21
|
};
|
|
22
|
+
function parseContentRange(response) {
|
|
23
|
+
const value = response.headers?.get?.("content-range");
|
|
24
|
+
if (!value) return null;
|
|
25
|
+
const match = /^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i.exec(value.trim());
|
|
26
|
+
if (!match) return null;
|
|
27
|
+
return {
|
|
28
|
+
start: Number(match[1]),
|
|
29
|
+
end: Number(match[2]),
|
|
30
|
+
size: match[3] === "*" ? null : Number(match[3])
|
|
31
|
+
};
|
|
32
|
+
}
|
|
21
33
|
function hasErrorName(error) {
|
|
22
34
|
return typeof error === "object" && error !== null && "name" in error;
|
|
23
35
|
}
|
|
@@ -39,6 +51,7 @@ var PATHS = {
|
|
|
39
51
|
TRANSACTIONS: "transactions"
|
|
40
52
|
};
|
|
41
53
|
var REPO_INFO_PATH = "repo";
|
|
54
|
+
var CONFIG_PATH = "config.yaml";
|
|
42
55
|
function getBranchRefDirPath(name) {
|
|
43
56
|
return `${PATHS.REFS}/branch.${name}/`;
|
|
44
57
|
}
|
|
@@ -1239,6 +1252,10 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1239
1252
|
} catch {
|
|
1240
1253
|
return result;
|
|
1241
1254
|
}
|
|
1255
|
+
return virtualChunkContainersFromConfig(config);
|
|
1256
|
+
}
|
|
1257
|
+
function virtualChunkContainersFromConfig(config) {
|
|
1258
|
+
const result = [];
|
|
1242
1259
|
if (!isRecord(config)) return result;
|
|
1243
1260
|
const containers = config.virtual_chunk_containers;
|
|
1244
1261
|
if (!isRecord(containers)) return result;
|
|
@@ -1390,6 +1407,81 @@ function resolveSnapshotParent(parsedRepo, snapshotId) {
|
|
|
1390
1407
|
return getSnapshotIdByIndex(parsedRepo, parentOffset);
|
|
1391
1408
|
}
|
|
1392
1409
|
|
|
1410
|
+
// src/format/config-yaml.ts
|
|
1411
|
+
var SECTION = "virtual_chunk_containers";
|
|
1412
|
+
var ENTRY = /^('(?:[^']|'')*'|"(?:[^"\\]|\\.)*"|.+?):(?: +(.*))?$/;
|
|
1413
|
+
function parseConfigYamlContainers(yaml) {
|
|
1414
|
+
const lines = yaml.split(/\r?\n/);
|
|
1415
|
+
const start = lines.findIndex((line) => line.startsWith(`${SECTION}:`));
|
|
1416
|
+
if (start < 0) return [];
|
|
1417
|
+
const body = [];
|
|
1418
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
1419
|
+
if (/^\S/.test(lines[i])) break;
|
|
1420
|
+
if (lines[i].trim() !== "") body.push(lines[i]);
|
|
1421
|
+
}
|
|
1422
|
+
for (let i = 0; i + 1 < body.length; i++) {
|
|
1423
|
+
const indent = indentOf(body[i]);
|
|
1424
|
+
const value = body[i + 1].slice(indent);
|
|
1425
|
+
if (body[i].startsWith("? ", indent) && indentOf(body[i + 1]) === indent && value.startsWith(": ")) {
|
|
1426
|
+
body[i] = `${" ".repeat(indent)}${JSON.stringify(body[i].slice(indent + 2))}:`;
|
|
1427
|
+
body[i + 1] = `${" ".repeat(indent)} ${value.slice(2)}`;
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
try {
|
|
1431
|
+
const [containers] = parseMapping(body, 0);
|
|
1432
|
+
return virtualChunkContainersFromConfig({ [SECTION]: containers });
|
|
1433
|
+
} catch {
|
|
1434
|
+
return [];
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
function parseMapping(lines, from) {
|
|
1438
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
1439
|
+
if (from >= lines.length) return [result, from];
|
|
1440
|
+
const indent = indentOf(lines[from]);
|
|
1441
|
+
let i = from;
|
|
1442
|
+
while (i < lines.length && indentOf(lines[i]) >= indent) {
|
|
1443
|
+
const entry = indentOf(lines[i]) === indent ? ENTRY.exec(lines[i].slice(indent)) : null;
|
|
1444
|
+
if (!entry || /^[-?:] /.test(entry[1])) {
|
|
1445
|
+
throw new Error(`Unsupported config.yaml line: ${lines[i]}`);
|
|
1446
|
+
}
|
|
1447
|
+
i++;
|
|
1448
|
+
const key = unquote(entry[1]);
|
|
1449
|
+
const tagged = /^!(\S+)(?: +(.*))?$/.exec(entry[2] ?? "");
|
|
1450
|
+
const rest = (tagged ? tagged[2] ?? "" : entry[2] ?? "").trim();
|
|
1451
|
+
let value;
|
|
1452
|
+
if (rest === "" && i < lines.length && indentOf(lines[i]) > indent) {
|
|
1453
|
+
[value, i] = parseMapping(lines, i);
|
|
1454
|
+
} else {
|
|
1455
|
+
value = parseScalar(rest);
|
|
1456
|
+
}
|
|
1457
|
+
result[key] = tagged ? { [tagged[1]]: value } : value;
|
|
1458
|
+
}
|
|
1459
|
+
return [result, i];
|
|
1460
|
+
}
|
|
1461
|
+
function indentOf(line) {
|
|
1462
|
+
return line.length - line.trimStart().length;
|
|
1463
|
+
}
|
|
1464
|
+
function parseScalar(text) {
|
|
1465
|
+
if (text === "" || text === "null" || text === "~") return null;
|
|
1466
|
+
if (text === "true") return true;
|
|
1467
|
+
if (text === "false") return false;
|
|
1468
|
+
if (text === "{}") return {};
|
|
1469
|
+
const quoted = /^['"]/.test(text);
|
|
1470
|
+
if (/^[[{|>&*]/.test(text) || !quoted && /(^| )#/.test(text)) {
|
|
1471
|
+
throw new Error(`Unsupported config.yaml value: ${text}`);
|
|
1472
|
+
}
|
|
1473
|
+
return unquote(text);
|
|
1474
|
+
}
|
|
1475
|
+
function unquote(text) {
|
|
1476
|
+
if (text.length >= 2 && text.startsWith("'") && text.endsWith("'")) {
|
|
1477
|
+
return text.slice(1, -1).replace(/''/g, "'");
|
|
1478
|
+
}
|
|
1479
|
+
if (text.length >= 2 && text.startsWith('"') && text.endsWith('"')) {
|
|
1480
|
+
return JSON.parse(text);
|
|
1481
|
+
}
|
|
1482
|
+
return text;
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1393
1485
|
// src/cache/lru.ts
|
|
1394
1486
|
var LRUCache = class {
|
|
1395
1487
|
cache;
|
|
@@ -2949,6 +3041,25 @@ function parseTransactionLog(data) {
|
|
|
2949
3041
|
function expectedRangeLength(range) {
|
|
2950
3042
|
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2951
3043
|
}
|
|
3044
|
+
function describeRange(range) {
|
|
3045
|
+
return "suffixLength" in range ? `last ${range.suffixLength} bytes` : `bytes ${range.offset}-${range.offset + range.length - 1}`;
|
|
3046
|
+
}
|
|
3047
|
+
function httpError(message, status) {
|
|
3048
|
+
return Object.assign(new Error(message), { status });
|
|
3049
|
+
}
|
|
3050
|
+
function assertContentRangeMatches(url, range, response) {
|
|
3051
|
+
const got = parseContentRange(response);
|
|
3052
|
+
if (!got) return;
|
|
3053
|
+
const matches = "suffixLength" in range ? got.end - got.start + 1 === range.suffixLength && (got.size === null || got.end === got.size - 1) : got.start === range.offset && got.end === range.offset + range.length - 1;
|
|
3054
|
+
if (!matches) {
|
|
3055
|
+
throw new Error(
|
|
3056
|
+
`Range response for ${url} covers bytes ${got.start}-${got.end}, not the requested ${describeRange(range)}`
|
|
3057
|
+
);
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3060
|
+
function copyBytes(data, start, end) {
|
|
3061
|
+
return new Uint8Array(data.subarray(start, end));
|
|
3062
|
+
}
|
|
2952
3063
|
function regionalS3RedirectUrl(url, response) {
|
|
2953
3064
|
if (response.status !== 301 && response.status !== 307) return null;
|
|
2954
3065
|
const region = response.headers.get("x-amz-bucket-region");
|
|
@@ -2989,17 +3100,20 @@ function makeUrlStore(opts) {
|
|
|
2989
3100
|
});
|
|
2990
3101
|
}
|
|
2991
3102
|
if (response.status === 412) {
|
|
2992
|
-
throw
|
|
2993
|
-
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created
|
|
3103
|
+
throw httpError(
|
|
3104
|
+
`Virtual chunk at ${requestUrl} failed integrity check \u2014 data has been modified since snapshot was created`,
|
|
3105
|
+
412
|
|
2994
3106
|
);
|
|
2995
3107
|
}
|
|
2996
3108
|
if (response.status !== 200 && response.status !== 206) {
|
|
2997
|
-
throw
|
|
2998
|
-
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}
|
|
3109
|
+
throw httpError(
|
|
3110
|
+
`Failed to fetch virtual chunk from ${requestUrl}: ${response.status} ${response.statusText}`,
|
|
3111
|
+
response.status
|
|
2999
3112
|
);
|
|
3000
3113
|
}
|
|
3001
3114
|
const data = new Uint8Array(await response.arrayBuffer());
|
|
3002
3115
|
if (response.status === 206) {
|
|
3116
|
+
assertContentRangeMatches(requestUrl, range, response);
|
|
3003
3117
|
const expected = expectedRangeLength(range);
|
|
3004
3118
|
if (data.length === expected) return data;
|
|
3005
3119
|
throw new Error(
|
|
@@ -3042,7 +3156,7 @@ function makeStorageStore(storage) {
|
|
|
3042
3156
|
const data = await storage.getObject(key, storageRange, storageOptions);
|
|
3043
3157
|
if (data.length === range.length) return data;
|
|
3044
3158
|
if (data.length >= storageRange.end) {
|
|
3045
|
-
return data
|
|
3159
|
+
return copyBytes(data, storageRange.start, storageRange.end);
|
|
3046
3160
|
}
|
|
3047
3161
|
throw new Error(
|
|
3048
3162
|
`Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
|
|
@@ -3050,6 +3164,238 @@ function makeStorageStore(storage) {
|
|
|
3050
3164
|
}
|
|
3051
3165
|
};
|
|
3052
3166
|
}
|
|
3167
|
+
var DEFAULT_MAX_GAP = 256 * 1024;
|
|
3168
|
+
var DEFAULT_MAX_BYTES = 16 << 20;
|
|
3169
|
+
var DEFAULT_MAX_WASTE_RATIO = 1;
|
|
3170
|
+
var COALESCER_BRAND = /* @__PURE__ */ Symbol("icechunk-js.rangeCoalescer");
|
|
3171
|
+
var WHOLE_OBJECT_STATUSES = /* @__PURE__ */ new Set([401, 403, 404, 412]);
|
|
3172
|
+
function isRangeCoalescer(fn) {
|
|
3173
|
+
return typeof fn === "function" && fn[COALESCER_BRAND] === true;
|
|
3174
|
+
}
|
|
3175
|
+
function freshStats() {
|
|
3176
|
+
return {
|
|
3177
|
+
ranges: 0,
|
|
3178
|
+
requests: 0,
|
|
3179
|
+
bytesRequested: 0,
|
|
3180
|
+
bytesFetched: 0,
|
|
3181
|
+
bytesDiscarded: 0,
|
|
3182
|
+
fallbacks: 0
|
|
3183
|
+
};
|
|
3184
|
+
}
|
|
3185
|
+
function nonNegative(name, value, allowInfinite = false) {
|
|
3186
|
+
const valid = allowInfinite ? !Number.isNaN(value) && value >= 0 : Number.isFinite(value) && value >= 0;
|
|
3187
|
+
if (!valid) {
|
|
3188
|
+
throw new RangeError(`coalesce.${name} must be a non-negative number`);
|
|
3189
|
+
}
|
|
3190
|
+
return value;
|
|
3191
|
+
}
|
|
3192
|
+
function abortReason(signal) {
|
|
3193
|
+
return signal.reason ?? new DOMException("The operation was aborted.", "AbortError");
|
|
3194
|
+
}
|
|
3195
|
+
function settle(waiter, outcome) {
|
|
3196
|
+
if (waiter.settled) return;
|
|
3197
|
+
waiter.settled = true;
|
|
3198
|
+
waiter.detach?.();
|
|
3199
|
+
waiter.detach = void 0;
|
|
3200
|
+
waiter.onAbort = void 0;
|
|
3201
|
+
outcome();
|
|
3202
|
+
}
|
|
3203
|
+
function failsWholeObject(error) {
|
|
3204
|
+
if (error instanceof NotFoundError) return true;
|
|
3205
|
+
const status = error?.status;
|
|
3206
|
+
return typeof status === "number" && WHOLE_OBJECT_STATUSES.has(status);
|
|
3207
|
+
}
|
|
3208
|
+
function createRangeCoalescer(options = {}) {
|
|
3209
|
+
const resolved = {
|
|
3210
|
+
maxGap: nonNegative("maxGap", options.maxGap ?? DEFAULT_MAX_GAP),
|
|
3211
|
+
maxBytes: nonNegative("maxBytes", options.maxBytes ?? DEFAULT_MAX_BYTES),
|
|
3212
|
+
maxWasteRatio: nonNegative(
|
|
3213
|
+
"maxWasteRatio",
|
|
3214
|
+
options.maxWasteRatio ?? DEFAULT_MAX_WASTE_RATIO,
|
|
3215
|
+
true
|
|
3216
|
+
),
|
|
3217
|
+
window: nonNegative("window", options.window ?? 0)
|
|
3218
|
+
};
|
|
3219
|
+
const stats = freshStats();
|
|
3220
|
+
const wrap = (store) => wrapStore(store, resolved, stats);
|
|
3221
|
+
return Object.assign(wrap, {
|
|
3222
|
+
[COALESCER_BRAND]: true,
|
|
3223
|
+
stats,
|
|
3224
|
+
resetStats() {
|
|
3225
|
+
Object.assign(stats, freshStats());
|
|
3226
|
+
}
|
|
3227
|
+
});
|
|
3228
|
+
}
|
|
3229
|
+
function wrapStore(store, options, stats) {
|
|
3230
|
+
if (typeof store.getRange !== "function") {
|
|
3231
|
+
throw new Error("Range coalescing requires a store with getRange");
|
|
3232
|
+
}
|
|
3233
|
+
const inner = store.getRange.bind(store);
|
|
3234
|
+
const pending = /* @__PURE__ */ new Map();
|
|
3235
|
+
let scheduled = false;
|
|
3236
|
+
function flush() {
|
|
3237
|
+
scheduled = false;
|
|
3238
|
+
const work = [...pending];
|
|
3239
|
+
pending.clear();
|
|
3240
|
+
for (const [key, reads] of work) {
|
|
3241
|
+
const live = reads.filter((read) => !read.waiter.settled);
|
|
3242
|
+
for (const run of planRuns(live, options)) {
|
|
3243
|
+
const length = run.end - run.offset;
|
|
3244
|
+
stats.bytesDiscarded += length - run.delivered;
|
|
3245
|
+
void issue(inner, stats, key, run.offset, length, run.members);
|
|
3246
|
+
}
|
|
3247
|
+
}
|
|
3248
|
+
}
|
|
3249
|
+
function schedule() {
|
|
3250
|
+
if (scheduled) return;
|
|
3251
|
+
scheduled = true;
|
|
3252
|
+
if (options.window > 0) setTimeout(flush, options.window);
|
|
3253
|
+
else queueMicrotask(flush);
|
|
3254
|
+
}
|
|
3255
|
+
return {
|
|
3256
|
+
get: (key, getOptions) => store.get(key, getOptions),
|
|
3257
|
+
getRange(key, range, getOptions) {
|
|
3258
|
+
if ("suffixLength" in range) return inner(key, range, getOptions);
|
|
3259
|
+
const signal = getOptions?.signal;
|
|
3260
|
+
if (signal?.aborted) return Promise.reject(abortReason(signal));
|
|
3261
|
+
stats.ranges++;
|
|
3262
|
+
stats.bytesRequested += range.length;
|
|
3263
|
+
return new Promise((resolve, reject) => {
|
|
3264
|
+
const waiter = { signal, settled: false, resolve, reject };
|
|
3265
|
+
if (signal) {
|
|
3266
|
+
const onAbort = () => {
|
|
3267
|
+
const hook = waiter.onAbort;
|
|
3268
|
+
settle(waiter, () => waiter.reject(abortReason(signal)));
|
|
3269
|
+
hook?.();
|
|
3270
|
+
};
|
|
3271
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
3272
|
+
waiter.detach = () => signal.removeEventListener("abort", onAbort);
|
|
3273
|
+
}
|
|
3274
|
+
let reads = pending.get(key);
|
|
3275
|
+
if (!reads) {
|
|
3276
|
+
reads = [];
|
|
3277
|
+
pending.set(key, reads);
|
|
3278
|
+
}
|
|
3279
|
+
reads.push({ offset: range.offset, length: range.length, waiter });
|
|
3280
|
+
schedule();
|
|
3281
|
+
});
|
|
3282
|
+
}
|
|
3283
|
+
};
|
|
3284
|
+
}
|
|
3285
|
+
function planRuns(reads, options) {
|
|
3286
|
+
reads.sort((a, b) => a.offset - b.offset || a.length - b.length);
|
|
3287
|
+
const members = [];
|
|
3288
|
+
for (const read of reads) {
|
|
3289
|
+
const last = members[members.length - 1];
|
|
3290
|
+
if (last && last.offset === read.offset && last.length === read.length) {
|
|
3291
|
+
last.waiters.push(read.waiter);
|
|
3292
|
+
} else {
|
|
3293
|
+
members.push({
|
|
3294
|
+
offset: read.offset,
|
|
3295
|
+
length: read.length,
|
|
3296
|
+
waiters: [read.waiter]
|
|
3297
|
+
});
|
|
3298
|
+
}
|
|
3299
|
+
}
|
|
3300
|
+
const runs = [];
|
|
3301
|
+
let run;
|
|
3302
|
+
for (const member of members) {
|
|
3303
|
+
const memberEnd = member.offset + member.length;
|
|
3304
|
+
if (run) {
|
|
3305
|
+
const added = Math.max(0, memberEnd - Math.max(member.offset, run.end));
|
|
3306
|
+
const end = Math.max(run.end, memberEnd);
|
|
3307
|
+
const length = end - run.offset;
|
|
3308
|
+
const delivered = run.delivered + added;
|
|
3309
|
+
if (member.offset - run.end <= options.maxGap && length <= options.maxBytes && length - delivered <= options.maxWasteRatio * delivered) {
|
|
3310
|
+
run.end = end;
|
|
3311
|
+
run.delivered = delivered;
|
|
3312
|
+
run.members.push(member);
|
|
3313
|
+
continue;
|
|
3314
|
+
}
|
|
3315
|
+
}
|
|
3316
|
+
run = {
|
|
3317
|
+
offset: member.offset,
|
|
3318
|
+
end: memberEnd,
|
|
3319
|
+
members: [member],
|
|
3320
|
+
delivered: member.length
|
|
3321
|
+
};
|
|
3322
|
+
runs.push(run);
|
|
3323
|
+
}
|
|
3324
|
+
return runs;
|
|
3325
|
+
}
|
|
3326
|
+
async function issue(inner, stats, key, offset, length, members) {
|
|
3327
|
+
const waiters = [];
|
|
3328
|
+
for (const member of members) {
|
|
3329
|
+
for (const waiter of member.waiters) {
|
|
3330
|
+
if (!waiter.settled) waiters.push(waiter);
|
|
3331
|
+
}
|
|
3332
|
+
}
|
|
3333
|
+
if (waiters.length === 0) return;
|
|
3334
|
+
let signal;
|
|
3335
|
+
let controller;
|
|
3336
|
+
if (waiters.length === 1) {
|
|
3337
|
+
signal = waiters[0].signal;
|
|
3338
|
+
} else if (waiters.some((waiter) => waiter.signal)) {
|
|
3339
|
+
const shared = new AbortController();
|
|
3340
|
+
controller = shared;
|
|
3341
|
+
signal = shared.signal;
|
|
3342
|
+
let live = waiters.length;
|
|
3343
|
+
for (const waiter of waiters) {
|
|
3344
|
+
const waiterSignal = waiter.signal;
|
|
3345
|
+
if (!waiterSignal) continue;
|
|
3346
|
+
waiter.onAbort = () => {
|
|
3347
|
+
if (--live === 0) shared.abort(waiterSignal.reason);
|
|
3348
|
+
};
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
stats.requests++;
|
|
3352
|
+
stats.bytesFetched += length;
|
|
3353
|
+
let data;
|
|
3354
|
+
try {
|
|
3355
|
+
data = await inner(
|
|
3356
|
+
key,
|
|
3357
|
+
{ offset, length },
|
|
3358
|
+
signal ? { signal } : void 0
|
|
3359
|
+
);
|
|
3360
|
+
if (members.length > 1 && data && data.length < length) {
|
|
3361
|
+
throw new Error(
|
|
3362
|
+
`Short read for ${key} at ${offset}: expected ${length} bytes, got ${data.length}`
|
|
3363
|
+
);
|
|
3364
|
+
}
|
|
3365
|
+
} catch (error) {
|
|
3366
|
+
if (members.length > 1 && !controller?.signal.aborted && !failsWholeObject(error)) {
|
|
3367
|
+
stats.fallbacks++;
|
|
3368
|
+
for (const waiter of waiters) waiter.onAbort = void 0;
|
|
3369
|
+
await Promise.all(
|
|
3370
|
+
members.map(
|
|
3371
|
+
(member) => issue(inner, stats, key, member.offset, member.length, [member])
|
|
3372
|
+
)
|
|
3373
|
+
);
|
|
3374
|
+
return;
|
|
3375
|
+
}
|
|
3376
|
+
for (const waiter of waiters) settle(waiter, () => waiter.reject(error));
|
|
3377
|
+
return;
|
|
3378
|
+
}
|
|
3379
|
+
for (const member of members) {
|
|
3380
|
+
const start = member.offset - offset;
|
|
3381
|
+
let view;
|
|
3382
|
+
for (const waiter of member.waiters) {
|
|
3383
|
+
if (waiter.settled) continue;
|
|
3384
|
+
if (!data) {
|
|
3385
|
+
settle(waiter, () => waiter.resolve(void 0));
|
|
3386
|
+
continue;
|
|
3387
|
+
}
|
|
3388
|
+
if (!view) {
|
|
3389
|
+
view = members.length === 1 ? data : copyBytes(data, start, start + member.length);
|
|
3390
|
+
const first = view;
|
|
3391
|
+
settle(waiter, () => waiter.resolve(first));
|
|
3392
|
+
} else {
|
|
3393
|
+
const copy = new Uint8Array(view);
|
|
3394
|
+
settle(waiter, () => waiter.resolve(copy));
|
|
3395
|
+
}
|
|
3396
|
+
}
|
|
3397
|
+
}
|
|
3398
|
+
}
|
|
3053
3399
|
|
|
3054
3400
|
// src/reader/session.ts
|
|
3055
3401
|
var RANGE_COALESCE_SIZE = 32 * 1024;
|
|
@@ -3727,6 +4073,7 @@ var Repository = class _Repository {
|
|
|
3727
4073
|
storage;
|
|
3728
4074
|
repoInfo = null;
|
|
3729
4075
|
repoInfoAttempted = false;
|
|
4076
|
+
v1Containers = null;
|
|
3730
4077
|
constructor(storage) {
|
|
3731
4078
|
this.storage = storage;
|
|
3732
4079
|
}
|
|
@@ -3748,11 +4095,10 @@ var Repository = class _Repository {
|
|
|
3748
4095
|
}
|
|
3749
4096
|
this.repoInfoAttempted = true;
|
|
3750
4097
|
try {
|
|
3751
|
-
const data = await this.storage.getObject(
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
);
|
|
4098
|
+
const data = await this.storage.getObject(REPO_INFO_PATH, void 0, {
|
|
4099
|
+
...options,
|
|
4100
|
+
cache: "no-cache"
|
|
4101
|
+
});
|
|
3756
4102
|
this.repoInfo = parseRepo(data);
|
|
3757
4103
|
return this.repoInfo;
|
|
3758
4104
|
} catch (error) {
|
|
@@ -3768,6 +4114,42 @@ var Repository = class _Repository {
|
|
|
3768
4114
|
);
|
|
3769
4115
|
}
|
|
3770
4116
|
}
|
|
4117
|
+
/**
|
|
4118
|
+
* Load and cache the virtual chunk containers of a v1 repository, which
|
|
4119
|
+
* keeps its config in `config.yaml` rather than in a repo file.
|
|
4120
|
+
*
|
|
4121
|
+
* Best-effort, like the v2 config. A repository that never saved a config
|
|
4122
|
+
* has no such file (S3 answers 403 rather than 404 for a missing key of a
|
|
4123
|
+
* bucket that cannot be listed), and a file that cannot be parsed declares
|
|
4124
|
+
* no containers; chunk locations then resolve as they would without a
|
|
4125
|
+
* config. Any other failure also yields no containers but is not cached, so
|
|
4126
|
+
* the next checkout asks again.
|
|
4127
|
+
*
|
|
4128
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
4129
|
+
*/
|
|
4130
|
+
async loadV1Containers(options) {
|
|
4131
|
+
if (this.v1Containers) {
|
|
4132
|
+
return this.v1Containers;
|
|
4133
|
+
}
|
|
4134
|
+
let containers = [];
|
|
4135
|
+
let settled = true;
|
|
4136
|
+
try {
|
|
4137
|
+
const data = await this.storage.getObject(CONFIG_PATH, void 0, {
|
|
4138
|
+
...options,
|
|
4139
|
+
cache: "no-cache"
|
|
4140
|
+
});
|
|
4141
|
+
containers = parseConfigYamlContainers(new TextDecoder().decode(data));
|
|
4142
|
+
} catch (error) {
|
|
4143
|
+
if (isAbortError(error)) {
|
|
4144
|
+
throw error;
|
|
4145
|
+
}
|
|
4146
|
+
settled = error instanceof NotFoundError || error instanceof StorageError && error.status === 403;
|
|
4147
|
+
}
|
|
4148
|
+
if (settled) {
|
|
4149
|
+
this.v1Containers = containers;
|
|
4150
|
+
}
|
|
4151
|
+
return containers;
|
|
4152
|
+
}
|
|
3771
4153
|
/**
|
|
3772
4154
|
* Open an icechunk repository.
|
|
3773
4155
|
*
|
|
@@ -3972,16 +4354,25 @@ var Repository = class _Repository {
|
|
|
3972
4354
|
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3973
4355
|
});
|
|
3974
4356
|
}
|
|
3975
|
-
const
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
3979
|
-
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
|
|
3984
|
-
|
|
4357
|
+
const resolveSnapshotId = async () => {
|
|
4358
|
+
const refDirPath = getBranchRefDirPath(name);
|
|
4359
|
+
const refPath = await this.findLatestRefFile(
|
|
4360
|
+
refDirPath,
|
|
4361
|
+
getBranchRefPath(name)
|
|
4362
|
+
);
|
|
4363
|
+
if (!refPath) {
|
|
4364
|
+
throw new Error(`Reference not found: ${refDirPath}`);
|
|
4365
|
+
}
|
|
4366
|
+
return this.readSnapshotIdFromRef(refPath, options);
|
|
4367
|
+
};
|
|
4368
|
+
const [snapshotId, virtualChunkContainers] = await Promise.all([
|
|
4369
|
+
resolveSnapshotId(),
|
|
4370
|
+
this.loadV1Containers(options)
|
|
4371
|
+
]);
|
|
4372
|
+
return ReadSession.open(this.storage, snapshotId, {
|
|
4373
|
+
...options,
|
|
4374
|
+
virtualChunkContainers
|
|
4375
|
+
});
|
|
3985
4376
|
}
|
|
3986
4377
|
/**
|
|
3987
4378
|
* Checkout a tag to get a read session.
|
|
@@ -4002,17 +4393,26 @@ var Repository = class _Repository {
|
|
|
4002
4393
|
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
4003
4394
|
});
|
|
4004
4395
|
}
|
|
4005
|
-
const
|
|
4006
|
-
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
|
|
4012
|
-
|
|
4013
|
-
|
|
4014
|
-
|
|
4015
|
-
|
|
4396
|
+
const resolveSnapshotId = async () => {
|
|
4397
|
+
const refDirPath = getTagRefDirPath(name);
|
|
4398
|
+
const legacyPath = getTagRefPath(name);
|
|
4399
|
+
const refPath = await this.findLatestRefFile(refDirPath, legacyPath);
|
|
4400
|
+
if (!refPath) {
|
|
4401
|
+
throw new Error(`Reference not found: ${refDirPath}`);
|
|
4402
|
+
}
|
|
4403
|
+
if (refPath === legacyPath && await this.storage.exists(`${refPath}.deleted`, options)) {
|
|
4404
|
+
throw new Error(`Tag not found: ${name}`);
|
|
4405
|
+
}
|
|
4406
|
+
return this.readSnapshotIdFromRef(refPath, options);
|
|
4407
|
+
};
|
|
4408
|
+
const [snapshotId, virtualChunkContainers] = await Promise.all([
|
|
4409
|
+
resolveSnapshotId(),
|
|
4410
|
+
this.loadV1Containers(options)
|
|
4411
|
+
]);
|
|
4412
|
+
return ReadSession.open(this.storage, snapshotId, {
|
|
4413
|
+
...options,
|
|
4414
|
+
virtualChunkContainers
|
|
4415
|
+
});
|
|
4016
4416
|
}
|
|
4017
4417
|
/**
|
|
4018
4418
|
* Checkout a specific snapshot by ID.
|
|
@@ -4024,10 +4424,9 @@ var Repository = class _Repository {
|
|
|
4024
4424
|
async checkoutSnapshot(snapshotId, options) {
|
|
4025
4425
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
4026
4426
|
const repoInfo = await this.loadRepoInfo(options);
|
|
4027
|
-
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
4028
4427
|
return ReadSession.open(this.storage, id, {
|
|
4029
4428
|
...options,
|
|
4030
|
-
|
|
4429
|
+
virtualChunkContainers: repoInfo ? repoInfo.virtualChunkContainers : await this.loadV1Containers(options)
|
|
4031
4430
|
});
|
|
4032
4431
|
}
|
|
4033
4432
|
/**
|
|
@@ -4080,7 +4479,10 @@ var Repository = class _Repository {
|
|
|
4080
4479
|
}
|
|
4081
4480
|
/** Read and parse a ref file */
|
|
4082
4481
|
async readRef(path, options) {
|
|
4083
|
-
const data = await this.storage.getObject(path, void 0,
|
|
4482
|
+
const data = await this.storage.getObject(path, void 0, {
|
|
4483
|
+
...options,
|
|
4484
|
+
cache: "no-cache"
|
|
4485
|
+
});
|
|
4084
4486
|
const json = new TextDecoder().decode(data);
|
|
4085
4487
|
return JSON.parse(json);
|
|
4086
4488
|
}
|
|
@@ -4135,7 +4537,7 @@ var HttpStorage = class {
|
|
|
4135
4537
|
method: "GET",
|
|
4136
4538
|
headers,
|
|
4137
4539
|
credentials: this.options.credentials,
|
|
4138
|
-
cache: this.options.cache,
|
|
4540
|
+
cache: options?.cache ?? this.options.cache,
|
|
4139
4541
|
signal: options?.signal
|
|
4140
4542
|
});
|
|
4141
4543
|
} catch (error) {
|
|
@@ -4150,11 +4552,28 @@ var HttpStorage = class {
|
|
|
4150
4552
|
}
|
|
4151
4553
|
if (response.status !== 200 && response.status !== 206) {
|
|
4152
4554
|
throw new StorageError(
|
|
4153
|
-
`HTTP ${response.status} ${response.statusText} for ${url}
|
|
4555
|
+
`HTTP ${response.status} ${response.statusText} for ${url}`,
|
|
4556
|
+
void 0,
|
|
4557
|
+
response.status
|
|
4154
4558
|
);
|
|
4155
4559
|
}
|
|
4156
|
-
const
|
|
4157
|
-
|
|
4560
|
+
const body = new Uint8Array(await response.arrayBuffer());
|
|
4561
|
+
if (!range) return body;
|
|
4562
|
+
if (response.status === 206) {
|
|
4563
|
+
const got = parseContentRange(response);
|
|
4564
|
+
if (got && (got.start !== range.start || got.end !== range.end - 1)) {
|
|
4565
|
+
throw new StorageError(
|
|
4566
|
+
`Range response for ${url} covers bytes ${got.start}-${got.end}, not the requested ${range.start}-${range.end - 1}`
|
|
4567
|
+
);
|
|
4568
|
+
}
|
|
4569
|
+
return body;
|
|
4570
|
+
}
|
|
4571
|
+
if (body.length >= range.end) {
|
|
4572
|
+
return new Uint8Array(body.subarray(range.start, range.end));
|
|
4573
|
+
}
|
|
4574
|
+
throw new StorageError(
|
|
4575
|
+
`Range request not honored for ${url}: need at least ${range.end} bytes to slice the requested range, got ${body.length}`
|
|
4576
|
+
);
|
|
4158
4577
|
}
|
|
4159
4578
|
async exists(path, options) {
|
|
4160
4579
|
options?.signal?.throwIfAborted();
|
|
@@ -4201,13 +4620,14 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
4201
4620
|
this.withRangeCoalescing = withRangeCoalescing;
|
|
4202
4621
|
}
|
|
4203
4622
|
static async open(arg, options = {}) {
|
|
4623
|
+
const withRangeCoalescing = resolveRangeCoalescing(options);
|
|
4204
4624
|
if (arg instanceof ReadSession) {
|
|
4205
4625
|
return new _IcechunkStore(
|
|
4206
4626
|
arg,
|
|
4207
4627
|
options.fetchClient,
|
|
4208
4628
|
options.validateChecksums,
|
|
4209
4629
|
options.azureAccount,
|
|
4210
|
-
|
|
4630
|
+
withRangeCoalescing
|
|
4211
4631
|
);
|
|
4212
4632
|
}
|
|
4213
4633
|
const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
|
|
@@ -4236,9 +4656,17 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
4236
4656
|
options.fetchClient,
|
|
4237
4657
|
options.validateChecksums,
|
|
4238
4658
|
options.azureAccount,
|
|
4239
|
-
|
|
4659
|
+
withRangeCoalescing
|
|
4240
4660
|
);
|
|
4241
4661
|
}
|
|
4662
|
+
/**
|
|
4663
|
+
* The built-in range coalescer this store reads chunks through, with its
|
|
4664
|
+
* counters (`stats`: requests issued, bytes fetched and discarded, ...).
|
|
4665
|
+
* `undefined` when coalescing is off or a custom wrapper was supplied.
|
|
4666
|
+
*/
|
|
4667
|
+
get rangeCoalescer() {
|
|
4668
|
+
return isRangeCoalescer(this.withRangeCoalescing) ? this.withRangeCoalescing : void 0;
|
|
4669
|
+
}
|
|
4242
4670
|
/**
|
|
4243
4671
|
* Get data for a zarr key.
|
|
4244
4672
|
*
|
|
@@ -4402,6 +4830,11 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
4402
4830
|
return this.session.getMetadata(fullPath);
|
|
4403
4831
|
}
|
|
4404
4832
|
};
|
|
4833
|
+
function resolveRangeCoalescing(options) {
|
|
4834
|
+
if (options.withRangeCoalescing) return options.withRangeCoalescing;
|
|
4835
|
+
if (options.coalesce === false) return void 0;
|
|
4836
|
+
return createRangeCoalescer(options.coalesce);
|
|
4837
|
+
}
|
|
4405
4838
|
function parseZarrKey(key) {
|
|
4406
4839
|
const path = key.slice(1);
|
|
4407
4840
|
if (path === "zarr.json") {
|
|
@@ -4433,6 +4866,7 @@ export {
|
|
|
4433
4866
|
Repository,
|
|
4434
4867
|
SpecVersion,
|
|
4435
4868
|
StorageError,
|
|
4869
|
+
createRangeCoalescer,
|
|
4436
4870
|
decodeBase32,
|
|
4437
4871
|
decodeObjectId12,
|
|
4438
4872
|
decodeObjectId8,
|