icechunk-js 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/index.cjs +157 -23
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +15 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +157 -23
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -163,6 +163,13 @@ a region (repos written by Icechunk do), reads go straight to that regional
|
|
|
163
163
|
endpoint — which serves CORS and works in both Node and the browser, with no
|
|
164
164
|
extra configuration.
|
|
165
165
|
|
|
166
|
+
The config is read from the repo file of a v2 repository and from `config.yaml`
|
|
167
|
+
of a v1 repository. The `config.yaml` reader covers the layout Icechunk writes
|
|
168
|
+
for ordinary values, not YAML in general: a containers section it cannot read
|
|
169
|
+
(one edited by hand into another style, or holding a value with control
|
|
170
|
+
characters or a line break) is ignored as a whole, as is a missing or
|
|
171
|
+
unreadable file.
|
|
172
|
+
|
|
166
173
|
If no region is known, dotted-name buckets fall back to the global endpoint,
|
|
167
174
|
whose region redirect (for buckets outside `us-east-1`) is resolved
|
|
168
175
|
automatically in Node but not in browsers — the cross-origin redirect carries no
|
package/dist/index.cjs
CHANGED
|
@@ -105,6 +105,7 @@ var PATHS = {
|
|
|
105
105
|
TRANSACTIONS: "transactions"
|
|
106
106
|
};
|
|
107
107
|
var REPO_INFO_PATH = "repo";
|
|
108
|
+
var CONFIG_PATH = "config.yaml";
|
|
108
109
|
function getBranchRefDirPath(name) {
|
|
109
110
|
return `${PATHS.REFS}/branch.${name}/`;
|
|
110
111
|
}
|
|
@@ -1305,6 +1306,10 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1305
1306
|
} catch {
|
|
1306
1307
|
return result;
|
|
1307
1308
|
}
|
|
1309
|
+
return virtualChunkContainersFromConfig(config);
|
|
1310
|
+
}
|
|
1311
|
+
function virtualChunkContainersFromConfig(config) {
|
|
1312
|
+
const result = [];
|
|
1308
1313
|
if (!isRecord(config)) return result;
|
|
1309
1314
|
const containers = config.virtual_chunk_containers;
|
|
1310
1315
|
if (!isRecord(containers)) return result;
|
|
@@ -1456,6 +1461,81 @@ function resolveSnapshotParent(parsedRepo, snapshotId) {
|
|
|
1456
1461
|
return getSnapshotIdByIndex(parsedRepo, parentOffset);
|
|
1457
1462
|
}
|
|
1458
1463
|
|
|
1464
|
+
// src/format/config-yaml.ts
|
|
1465
|
+
var SECTION = "virtual_chunk_containers";
|
|
1466
|
+
var ENTRY = /^('(?:[^']|'')*'|"(?:[^"\\]|\\.)*"|.+?):(?: +(.*))?$/;
|
|
1467
|
+
function parseConfigYamlContainers(yaml) {
|
|
1468
|
+
const lines = yaml.split(/\r?\n/);
|
|
1469
|
+
const start = lines.findIndex((line) => line.startsWith(`${SECTION}:`));
|
|
1470
|
+
if (start < 0) return [];
|
|
1471
|
+
const body = [];
|
|
1472
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
1473
|
+
if (/^\S/.test(lines[i])) break;
|
|
1474
|
+
if (lines[i].trim() !== "") body.push(lines[i]);
|
|
1475
|
+
}
|
|
1476
|
+
for (let i = 0; i + 1 < body.length; i++) {
|
|
1477
|
+
const indent = indentOf(body[i]);
|
|
1478
|
+
const value = body[i + 1].slice(indent);
|
|
1479
|
+
if (body[i].startsWith("? ", indent) && indentOf(body[i + 1]) === indent && value.startsWith(": ")) {
|
|
1480
|
+
body[i] = `${" ".repeat(indent)}${JSON.stringify(body[i].slice(indent + 2))}:`;
|
|
1481
|
+
body[i + 1] = `${" ".repeat(indent)} ${value.slice(2)}`;
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
try {
|
|
1485
|
+
const [containers] = parseMapping(body, 0);
|
|
1486
|
+
return virtualChunkContainersFromConfig({ [SECTION]: containers });
|
|
1487
|
+
} catch {
|
|
1488
|
+
return [];
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
function parseMapping(lines, from) {
|
|
1492
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
1493
|
+
if (from >= lines.length) return [result, from];
|
|
1494
|
+
const indent = indentOf(lines[from]);
|
|
1495
|
+
let i = from;
|
|
1496
|
+
while (i < lines.length && indentOf(lines[i]) >= indent) {
|
|
1497
|
+
const entry = indentOf(lines[i]) === indent ? ENTRY.exec(lines[i].slice(indent)) : null;
|
|
1498
|
+
if (!entry || /^[-?:] /.test(entry[1])) {
|
|
1499
|
+
throw new Error(`Unsupported config.yaml line: ${lines[i]}`);
|
|
1500
|
+
}
|
|
1501
|
+
i++;
|
|
1502
|
+
const key = unquote(entry[1]);
|
|
1503
|
+
const tagged = /^!(\S+)(?: +(.*))?$/.exec(entry[2] ?? "");
|
|
1504
|
+
const rest = (tagged ? tagged[2] ?? "" : entry[2] ?? "").trim();
|
|
1505
|
+
let value;
|
|
1506
|
+
if (rest === "" && i < lines.length && indentOf(lines[i]) > indent) {
|
|
1507
|
+
[value, i] = parseMapping(lines, i);
|
|
1508
|
+
} else {
|
|
1509
|
+
value = parseScalar(rest);
|
|
1510
|
+
}
|
|
1511
|
+
result[key] = tagged ? { [tagged[1]]: value } : value;
|
|
1512
|
+
}
|
|
1513
|
+
return [result, i];
|
|
1514
|
+
}
|
|
1515
|
+
function indentOf(line) {
|
|
1516
|
+
return line.length - line.trimStart().length;
|
|
1517
|
+
}
|
|
1518
|
+
function parseScalar(text) {
|
|
1519
|
+
if (text === "" || text === "null" || text === "~") return null;
|
|
1520
|
+
if (text === "true") return true;
|
|
1521
|
+
if (text === "false") return false;
|
|
1522
|
+
if (text === "{}") return {};
|
|
1523
|
+
const quoted = /^['"]/.test(text);
|
|
1524
|
+
if (/^[[{|>&*]/.test(text) || !quoted && /(^| )#/.test(text)) {
|
|
1525
|
+
throw new Error(`Unsupported config.yaml value: ${text}`);
|
|
1526
|
+
}
|
|
1527
|
+
return unquote(text);
|
|
1528
|
+
}
|
|
1529
|
+
function unquote(text) {
|
|
1530
|
+
if (text.length >= 2 && text.startsWith("'") && text.endsWith("'")) {
|
|
1531
|
+
return text.slice(1, -1).replace(/''/g, "'");
|
|
1532
|
+
}
|
|
1533
|
+
if (text.length >= 2 && text.startsWith('"') && text.endsWith('"')) {
|
|
1534
|
+
return JSON.parse(text);
|
|
1535
|
+
}
|
|
1536
|
+
return text;
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1459
1539
|
// src/cache/lru.ts
|
|
1460
1540
|
var LRUCache = class {
|
|
1461
1541
|
cache;
|
|
@@ -4047,6 +4127,7 @@ var Repository = class _Repository {
|
|
|
4047
4127
|
storage;
|
|
4048
4128
|
repoInfo = null;
|
|
4049
4129
|
repoInfoAttempted = false;
|
|
4130
|
+
v1Containers = null;
|
|
4050
4131
|
constructor(storage) {
|
|
4051
4132
|
this.storage = storage;
|
|
4052
4133
|
}
|
|
@@ -4087,6 +4168,42 @@ var Repository = class _Repository {
|
|
|
4087
4168
|
);
|
|
4088
4169
|
}
|
|
4089
4170
|
}
|
|
4171
|
+
/**
|
|
4172
|
+
* Load and cache the virtual chunk containers of a v1 repository, which
|
|
4173
|
+
* keeps its config in `config.yaml` rather than in a repo file.
|
|
4174
|
+
*
|
|
4175
|
+
* Best-effort, like the v2 config. A repository that never saved a config
|
|
4176
|
+
* has no such file (S3 answers 403 rather than 404 for a missing key of a
|
|
4177
|
+
* bucket that cannot be listed), and a file that cannot be parsed declares
|
|
4178
|
+
* no containers; chunk locations then resolve as they would without a
|
|
4179
|
+
* config. Any other failure also yields no containers but is not cached, so
|
|
4180
|
+
* the next checkout asks again.
|
|
4181
|
+
*
|
|
4182
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
4183
|
+
*/
|
|
4184
|
+
async loadV1Containers(options) {
|
|
4185
|
+
if (this.v1Containers) {
|
|
4186
|
+
return this.v1Containers;
|
|
4187
|
+
}
|
|
4188
|
+
let containers = [];
|
|
4189
|
+
let settled = true;
|
|
4190
|
+
try {
|
|
4191
|
+
const data = await this.storage.getObject(CONFIG_PATH, void 0, {
|
|
4192
|
+
...options,
|
|
4193
|
+
cache: "no-cache"
|
|
4194
|
+
});
|
|
4195
|
+
containers = parseConfigYamlContainers(new TextDecoder().decode(data));
|
|
4196
|
+
} catch (error) {
|
|
4197
|
+
if (isAbortError(error)) {
|
|
4198
|
+
throw error;
|
|
4199
|
+
}
|
|
4200
|
+
settled = error instanceof NotFoundError || error instanceof StorageError && error.status === 403;
|
|
4201
|
+
}
|
|
4202
|
+
if (settled) {
|
|
4203
|
+
this.v1Containers = containers;
|
|
4204
|
+
}
|
|
4205
|
+
return containers;
|
|
4206
|
+
}
|
|
4090
4207
|
/**
|
|
4091
4208
|
* Open an icechunk repository.
|
|
4092
4209
|
*
|
|
@@ -4291,16 +4408,25 @@ var Repository = class _Repository {
|
|
|
4291
4408
|
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
4292
4409
|
});
|
|
4293
4410
|
}
|
|
4294
|
-
const
|
|
4295
|
-
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
|
|
4299
|
-
|
|
4300
|
-
|
|
4301
|
-
|
|
4302
|
-
|
|
4303
|
-
|
|
4411
|
+
const resolveSnapshotId = async () => {
|
|
4412
|
+
const refDirPath = getBranchRefDirPath(name);
|
|
4413
|
+
const refPath = await this.findLatestRefFile(
|
|
4414
|
+
refDirPath,
|
|
4415
|
+
getBranchRefPath(name)
|
|
4416
|
+
);
|
|
4417
|
+
if (!refPath) {
|
|
4418
|
+
throw new Error(`Reference not found: ${refDirPath}`);
|
|
4419
|
+
}
|
|
4420
|
+
return this.readSnapshotIdFromRef(refPath, options);
|
|
4421
|
+
};
|
|
4422
|
+
const [snapshotId, virtualChunkContainers] = await Promise.all([
|
|
4423
|
+
resolveSnapshotId(),
|
|
4424
|
+
this.loadV1Containers(options)
|
|
4425
|
+
]);
|
|
4426
|
+
return ReadSession.open(this.storage, snapshotId, {
|
|
4427
|
+
...options,
|
|
4428
|
+
virtualChunkContainers
|
|
4429
|
+
});
|
|
4304
4430
|
}
|
|
4305
4431
|
/**
|
|
4306
4432
|
* Checkout a tag to get a read session.
|
|
@@ -4321,17 +4447,26 @@ var Repository = class _Repository {
|
|
|
4321
4447
|
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
4322
4448
|
});
|
|
4323
4449
|
}
|
|
4324
|
-
const
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4450
|
+
const resolveSnapshotId = async () => {
|
|
4451
|
+
const refDirPath = getTagRefDirPath(name);
|
|
4452
|
+
const legacyPath = getTagRefPath(name);
|
|
4453
|
+
const refPath = await this.findLatestRefFile(refDirPath, legacyPath);
|
|
4454
|
+
if (!refPath) {
|
|
4455
|
+
throw new Error(`Reference not found: ${refDirPath}`);
|
|
4456
|
+
}
|
|
4457
|
+
if (refPath === legacyPath && await this.storage.exists(`${refPath}.deleted`, options)) {
|
|
4458
|
+
throw new Error(`Tag not found: ${name}`);
|
|
4459
|
+
}
|
|
4460
|
+
return this.readSnapshotIdFromRef(refPath, options);
|
|
4461
|
+
};
|
|
4462
|
+
const [snapshotId, virtualChunkContainers] = await Promise.all([
|
|
4463
|
+
resolveSnapshotId(),
|
|
4464
|
+
this.loadV1Containers(options)
|
|
4465
|
+
]);
|
|
4466
|
+
return ReadSession.open(this.storage, snapshotId, {
|
|
4467
|
+
...options,
|
|
4468
|
+
virtualChunkContainers
|
|
4469
|
+
});
|
|
4335
4470
|
}
|
|
4336
4471
|
/**
|
|
4337
4472
|
* Checkout a specific snapshot by ID.
|
|
@@ -4343,10 +4478,9 @@ var Repository = class _Repository {
|
|
|
4343
4478
|
async checkoutSnapshot(snapshotId, options) {
|
|
4344
4479
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
4345
4480
|
const repoInfo = await this.loadRepoInfo(options);
|
|
4346
|
-
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
4347
4481
|
return ReadSession.open(this.storage, id, {
|
|
4348
4482
|
...options,
|
|
4349
|
-
|
|
4483
|
+
virtualChunkContainers: repoInfo ? repoInfo.virtualChunkContainers : await this.loadV1Containers(options)
|
|
4350
4484
|
});
|
|
4351
4485
|
}
|
|
4352
4486
|
/**
|