icechunk-js 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/index.cjs +157 -23
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +15 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +157 -23
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -902,6 +902,7 @@ declare class Repository {
|
|
|
902
902
|
private storage;
|
|
903
903
|
private repoInfo;
|
|
904
904
|
private repoInfoAttempted;
|
|
905
|
+
private v1Containers;
|
|
905
906
|
private constructor();
|
|
906
907
|
/**
|
|
907
908
|
* Load and cache the v2 repo info file.
|
|
@@ -916,6 +917,20 @@ declare class Repository {
|
|
|
916
917
|
* @throws An error named "AbortError" if the operation was aborted
|
|
917
918
|
*/
|
|
918
919
|
private loadRepoInfo;
|
|
920
|
+
/**
|
|
921
|
+
* Load and cache the virtual chunk containers of a v1 repository, which
|
|
922
|
+
* keeps its config in `config.yaml` rather than in a repo file.
|
|
923
|
+
*
|
|
924
|
+
* Best-effort, like the v2 config. A repository that never saved a config
|
|
925
|
+
* has no such file (S3 answers 403 rather than 404 for a missing key of a
|
|
926
|
+
* bucket that cannot be listed), and a file that cannot be parsed declares
|
|
927
|
+
* no containers; chunk locations then resolve as they would without a
|
|
928
|
+
* config. Any other failure also yields no containers but is not cached, so
|
|
929
|
+
* the next checkout asks again.
|
|
930
|
+
*
|
|
931
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
932
|
+
*/
|
|
933
|
+
private loadV1Containers;
|
|
919
934
|
/**
|
|
920
935
|
* Open an icechunk repository.
|
|
921
936
|
*
|
package/dist/index.d.ts
CHANGED
|
@@ -902,6 +902,7 @@ declare class Repository {
|
|
|
902
902
|
private storage;
|
|
903
903
|
private repoInfo;
|
|
904
904
|
private repoInfoAttempted;
|
|
905
|
+
private v1Containers;
|
|
905
906
|
private constructor();
|
|
906
907
|
/**
|
|
907
908
|
* Load and cache the v2 repo info file.
|
|
@@ -916,6 +917,20 @@ declare class Repository {
|
|
|
916
917
|
* @throws An error named "AbortError" if the operation was aborted
|
|
917
918
|
*/
|
|
918
919
|
private loadRepoInfo;
|
|
920
|
+
/**
|
|
921
|
+
* Load and cache the virtual chunk containers of a v1 repository, which
|
|
922
|
+
* keeps its config in `config.yaml` rather than in a repo file.
|
|
923
|
+
*
|
|
924
|
+
* Best-effort, like the v2 config. A repository that never saved a config
|
|
925
|
+
* has no such file (S3 answers 403 rather than 404 for a missing key of a
|
|
926
|
+
* bucket that cannot be listed), and a file that cannot be parsed declares
|
|
927
|
+
* no containers; chunk locations then resolve as they would without a
|
|
928
|
+
* config. Any other failure also yields no containers but is not cached, so
|
|
929
|
+
* the next checkout asks again.
|
|
930
|
+
*
|
|
931
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
932
|
+
*/
|
|
933
|
+
private loadV1Containers;
|
|
919
934
|
/**
|
|
920
935
|
* Open an icechunk repository.
|
|
921
936
|
*
|
package/dist/index.js
CHANGED
|
@@ -51,6 +51,7 @@ var PATHS = {
|
|
|
51
51
|
TRANSACTIONS: "transactions"
|
|
52
52
|
};
|
|
53
53
|
var REPO_INFO_PATH = "repo";
|
|
54
|
+
var CONFIG_PATH = "config.yaml";
|
|
54
55
|
function getBranchRefDirPath(name) {
|
|
55
56
|
return `${PATHS.REFS}/branch.${name}/`;
|
|
56
57
|
}
|
|
@@ -1251,6 +1252,10 @@ function parseVirtualChunkContainers(repo) {
|
|
|
1251
1252
|
} catch {
|
|
1252
1253
|
return result;
|
|
1253
1254
|
}
|
|
1255
|
+
return virtualChunkContainersFromConfig(config);
|
|
1256
|
+
}
|
|
1257
|
+
function virtualChunkContainersFromConfig(config) {
|
|
1258
|
+
const result = [];
|
|
1254
1259
|
if (!isRecord(config)) return result;
|
|
1255
1260
|
const containers = config.virtual_chunk_containers;
|
|
1256
1261
|
if (!isRecord(containers)) return result;
|
|
@@ -1402,6 +1407,81 @@ function resolveSnapshotParent(parsedRepo, snapshotId) {
|
|
|
1402
1407
|
return getSnapshotIdByIndex(parsedRepo, parentOffset);
|
|
1403
1408
|
}
|
|
1404
1409
|
|
|
1410
|
+
// src/format/config-yaml.ts
|
|
1411
|
+
var SECTION = "virtual_chunk_containers";
|
|
1412
|
+
var ENTRY = /^('(?:[^']|'')*'|"(?:[^"\\]|\\.)*"|.+?):(?: +(.*))?$/;
|
|
1413
|
+
function parseConfigYamlContainers(yaml) {
|
|
1414
|
+
const lines = yaml.split(/\r?\n/);
|
|
1415
|
+
const start = lines.findIndex((line) => line.startsWith(`${SECTION}:`));
|
|
1416
|
+
if (start < 0) return [];
|
|
1417
|
+
const body = [];
|
|
1418
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
1419
|
+
if (/^\S/.test(lines[i])) break;
|
|
1420
|
+
if (lines[i].trim() !== "") body.push(lines[i]);
|
|
1421
|
+
}
|
|
1422
|
+
for (let i = 0; i + 1 < body.length; i++) {
|
|
1423
|
+
const indent = indentOf(body[i]);
|
|
1424
|
+
const value = body[i + 1].slice(indent);
|
|
1425
|
+
if (body[i].startsWith("? ", indent) && indentOf(body[i + 1]) === indent && value.startsWith(": ")) {
|
|
1426
|
+
body[i] = `${" ".repeat(indent)}${JSON.stringify(body[i].slice(indent + 2))}:`;
|
|
1427
|
+
body[i + 1] = `${" ".repeat(indent)} ${value.slice(2)}`;
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
try {
|
|
1431
|
+
const [containers] = parseMapping(body, 0);
|
|
1432
|
+
return virtualChunkContainersFromConfig({ [SECTION]: containers });
|
|
1433
|
+
} catch {
|
|
1434
|
+
return [];
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
function parseMapping(lines, from) {
|
|
1438
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
1439
|
+
if (from >= lines.length) return [result, from];
|
|
1440
|
+
const indent = indentOf(lines[from]);
|
|
1441
|
+
let i = from;
|
|
1442
|
+
while (i < lines.length && indentOf(lines[i]) >= indent) {
|
|
1443
|
+
const entry = indentOf(lines[i]) === indent ? ENTRY.exec(lines[i].slice(indent)) : null;
|
|
1444
|
+
if (!entry || /^[-?:] /.test(entry[1])) {
|
|
1445
|
+
throw new Error(`Unsupported config.yaml line: ${lines[i]}`);
|
|
1446
|
+
}
|
|
1447
|
+
i++;
|
|
1448
|
+
const key = unquote(entry[1]);
|
|
1449
|
+
const tagged = /^!(\S+)(?: +(.*))?$/.exec(entry[2] ?? "");
|
|
1450
|
+
const rest = (tagged ? tagged[2] ?? "" : entry[2] ?? "").trim();
|
|
1451
|
+
let value;
|
|
1452
|
+
if (rest === "" && i < lines.length && indentOf(lines[i]) > indent) {
|
|
1453
|
+
[value, i] = parseMapping(lines, i);
|
|
1454
|
+
} else {
|
|
1455
|
+
value = parseScalar(rest);
|
|
1456
|
+
}
|
|
1457
|
+
result[key] = tagged ? { [tagged[1]]: value } : value;
|
|
1458
|
+
}
|
|
1459
|
+
return [result, i];
|
|
1460
|
+
}
|
|
1461
|
+
function indentOf(line) {
|
|
1462
|
+
return line.length - line.trimStart().length;
|
|
1463
|
+
}
|
|
1464
|
+
function parseScalar(text) {
|
|
1465
|
+
if (text === "" || text === "null" || text === "~") return null;
|
|
1466
|
+
if (text === "true") return true;
|
|
1467
|
+
if (text === "false") return false;
|
|
1468
|
+
if (text === "{}") return {};
|
|
1469
|
+
const quoted = /^['"]/.test(text);
|
|
1470
|
+
if (/^[[{|>&*]/.test(text) || !quoted && /(^| )#/.test(text)) {
|
|
1471
|
+
throw new Error(`Unsupported config.yaml value: ${text}`);
|
|
1472
|
+
}
|
|
1473
|
+
return unquote(text);
|
|
1474
|
+
}
|
|
1475
|
+
function unquote(text) {
|
|
1476
|
+
if (text.length >= 2 && text.startsWith("'") && text.endsWith("'")) {
|
|
1477
|
+
return text.slice(1, -1).replace(/''/g, "'");
|
|
1478
|
+
}
|
|
1479
|
+
if (text.length >= 2 && text.startsWith('"') && text.endsWith('"')) {
|
|
1480
|
+
return JSON.parse(text);
|
|
1481
|
+
}
|
|
1482
|
+
return text;
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1405
1485
|
// src/cache/lru.ts
|
|
1406
1486
|
var LRUCache = class {
|
|
1407
1487
|
cache;
|
|
@@ -3993,6 +4073,7 @@ var Repository = class _Repository {
|
|
|
3993
4073
|
storage;
|
|
3994
4074
|
repoInfo = null;
|
|
3995
4075
|
repoInfoAttempted = false;
|
|
4076
|
+
v1Containers = null;
|
|
3996
4077
|
constructor(storage) {
|
|
3997
4078
|
this.storage = storage;
|
|
3998
4079
|
}
|
|
@@ -4033,6 +4114,42 @@ var Repository = class _Repository {
|
|
|
4033
4114
|
);
|
|
4034
4115
|
}
|
|
4035
4116
|
}
|
|
4117
|
+
/**
|
|
4118
|
+
* Load and cache the virtual chunk containers of a v1 repository, which
|
|
4119
|
+
* keeps its config in `config.yaml` rather than in a repo file.
|
|
4120
|
+
*
|
|
4121
|
+
* Best-effort, like the v2 config. A repository that never saved a config
|
|
4122
|
+
* has no such file (S3 answers 403 rather than 404 for a missing key of a
|
|
4123
|
+
* bucket that cannot be listed), and a file that cannot be parsed declares
|
|
4124
|
+
* no containers; chunk locations then resolve as they would without a
|
|
4125
|
+
* config. Any other failure also yields no containers but is not cached, so
|
|
4126
|
+
* the next checkout asks again.
|
|
4127
|
+
*
|
|
4128
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
4129
|
+
*/
|
|
4130
|
+
async loadV1Containers(options) {
|
|
4131
|
+
if (this.v1Containers) {
|
|
4132
|
+
return this.v1Containers;
|
|
4133
|
+
}
|
|
4134
|
+
let containers = [];
|
|
4135
|
+
let settled = true;
|
|
4136
|
+
try {
|
|
4137
|
+
const data = await this.storage.getObject(CONFIG_PATH, void 0, {
|
|
4138
|
+
...options,
|
|
4139
|
+
cache: "no-cache"
|
|
4140
|
+
});
|
|
4141
|
+
containers = parseConfigYamlContainers(new TextDecoder().decode(data));
|
|
4142
|
+
} catch (error) {
|
|
4143
|
+
if (isAbortError(error)) {
|
|
4144
|
+
throw error;
|
|
4145
|
+
}
|
|
4146
|
+
settled = error instanceof NotFoundError || error instanceof StorageError && error.status === 403;
|
|
4147
|
+
}
|
|
4148
|
+
if (settled) {
|
|
4149
|
+
this.v1Containers = containers;
|
|
4150
|
+
}
|
|
4151
|
+
return containers;
|
|
4152
|
+
}
|
|
4036
4153
|
/**
|
|
4037
4154
|
* Open an icechunk repository.
|
|
4038
4155
|
*
|
|
@@ -4237,16 +4354,25 @@ var Repository = class _Repository {
|
|
|
4237
4354
|
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
4238
4355
|
});
|
|
4239
4356
|
}
|
|
4240
|
-
const
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4357
|
+
const resolveSnapshotId = async () => {
|
|
4358
|
+
const refDirPath = getBranchRefDirPath(name);
|
|
4359
|
+
const refPath = await this.findLatestRefFile(
|
|
4360
|
+
refDirPath,
|
|
4361
|
+
getBranchRefPath(name)
|
|
4362
|
+
);
|
|
4363
|
+
if (!refPath) {
|
|
4364
|
+
throw new Error(`Reference not found: ${refDirPath}`);
|
|
4365
|
+
}
|
|
4366
|
+
return this.readSnapshotIdFromRef(refPath, options);
|
|
4367
|
+
};
|
|
4368
|
+
const [snapshotId, virtualChunkContainers] = await Promise.all([
|
|
4369
|
+
resolveSnapshotId(),
|
|
4370
|
+
this.loadV1Containers(options)
|
|
4371
|
+
]);
|
|
4372
|
+
return ReadSession.open(this.storage, snapshotId, {
|
|
4373
|
+
...options,
|
|
4374
|
+
virtualChunkContainers
|
|
4375
|
+
});
|
|
4250
4376
|
}
|
|
4251
4377
|
/**
|
|
4252
4378
|
* Checkout a tag to get a read session.
|
|
@@ -4267,17 +4393,26 @@ var Repository = class _Repository {
|
|
|
4267
4393
|
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
4268
4394
|
});
|
|
4269
4395
|
}
|
|
4270
|
-
const
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4396
|
+
const resolveSnapshotId = async () => {
|
|
4397
|
+
const refDirPath = getTagRefDirPath(name);
|
|
4398
|
+
const legacyPath = getTagRefPath(name);
|
|
4399
|
+
const refPath = await this.findLatestRefFile(refDirPath, legacyPath);
|
|
4400
|
+
if (!refPath) {
|
|
4401
|
+
throw new Error(`Reference not found: ${refDirPath}`);
|
|
4402
|
+
}
|
|
4403
|
+
if (refPath === legacyPath && await this.storage.exists(`${refPath}.deleted`, options)) {
|
|
4404
|
+
throw new Error(`Tag not found: ${name}`);
|
|
4405
|
+
}
|
|
4406
|
+
return this.readSnapshotIdFromRef(refPath, options);
|
|
4407
|
+
};
|
|
4408
|
+
const [snapshotId, virtualChunkContainers] = await Promise.all([
|
|
4409
|
+
resolveSnapshotId(),
|
|
4410
|
+
this.loadV1Containers(options)
|
|
4411
|
+
]);
|
|
4412
|
+
return ReadSession.open(this.storage, snapshotId, {
|
|
4413
|
+
...options,
|
|
4414
|
+
virtualChunkContainers
|
|
4415
|
+
});
|
|
4281
4416
|
}
|
|
4282
4417
|
/**
|
|
4283
4418
|
* Checkout a specific snapshot by ID.
|
|
@@ -4289,10 +4424,9 @@ var Repository = class _Repository {
|
|
|
4289
4424
|
async checkoutSnapshot(snapshotId, options) {
|
|
4290
4425
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
4291
4426
|
const repoInfo = await this.loadRepoInfo(options);
|
|
4292
|
-
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
4293
4427
|
return ReadSession.open(this.storage, id, {
|
|
4294
4428
|
...options,
|
|
4295
|
-
|
|
4429
|
+
virtualChunkContainers: repoInfo ? repoInfo.virtualChunkContainers : await this.loadV1Containers(options)
|
|
4296
4430
|
});
|
|
4297
4431
|
}
|
|
4298
4432
|
/**
|