icechunk-js 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -902,6 +902,7 @@ declare class Repository {
902
902
  private storage;
903
903
  private repoInfo;
904
904
  private repoInfoAttempted;
905
+ private v1Containers;
905
906
  private constructor();
906
907
  /**
907
908
  * Load and cache the v2 repo info file.
@@ -916,6 +917,20 @@ declare class Repository {
916
917
  * @throws An error named "AbortError" if the operation was aborted
917
918
  */
918
919
  private loadRepoInfo;
920
+ /**
921
+ * Load and cache the virtual chunk containers of a v1 repository, which
922
+ * keeps its config in `config.yaml` rather than in a repo file.
923
+ *
924
+ * Best-effort, like the v2 config. A repository that never saved a config
925
+ * has no such file (S3 answers 403 rather than 404 for a missing key of a
926
+ * bucket that cannot be listed), and a file that cannot be parsed declares
927
+ * no containers; chunk locations then resolve as they would without a
928
+ * config. Any other failure also yields no containers but is not cached, so
929
+ * the next checkout asks again.
930
+ *
931
+ * @throws An error named "AbortError" if the operation was aborted
932
+ */
933
+ private loadV1Containers;
919
934
  /**
920
935
  * Open an icechunk repository.
921
936
  *
package/dist/index.d.ts CHANGED
@@ -902,6 +902,7 @@ declare class Repository {
902
902
  private storage;
903
903
  private repoInfo;
904
904
  private repoInfoAttempted;
905
+ private v1Containers;
905
906
  private constructor();
906
907
  /**
907
908
  * Load and cache the v2 repo info file.
@@ -916,6 +917,20 @@ declare class Repository {
916
917
  * @throws An error named "AbortError" if the operation was aborted
917
918
  */
918
919
  private loadRepoInfo;
920
+ /**
921
+ * Load and cache the virtual chunk containers of a v1 repository, which
922
+ * keeps its config in `config.yaml` rather than in a repo file.
923
+ *
924
+ * Best-effort, like the v2 config. A repository that never saved a config
925
+ * has no such file (S3 answers 403 rather than 404 for a missing key of a
926
+ * bucket that cannot be listed), and a file that cannot be parsed declares
927
+ * no containers; chunk locations then resolve as they would without a
928
+ * config. Any other failure also yields no containers but is not cached, so
929
+ * the next checkout asks again.
930
+ *
931
+ * @throws An error named "AbortError" if the operation was aborted
932
+ */
933
+ private loadV1Containers;
919
934
  /**
920
935
  * Open an icechunk repository.
921
936
  *
package/dist/index.js CHANGED
@@ -51,6 +51,7 @@ var PATHS = {
51
51
  TRANSACTIONS: "transactions"
52
52
  };
53
53
  var REPO_INFO_PATH = "repo";
54
+ var CONFIG_PATH = "config.yaml";
54
55
  function getBranchRefDirPath(name) {
55
56
  return `${PATHS.REFS}/branch.${name}/`;
56
57
  }
@@ -1251,6 +1252,10 @@ function parseVirtualChunkContainers(repo) {
1251
1252
  } catch {
1252
1253
  return result;
1253
1254
  }
1255
+ return virtualChunkContainersFromConfig(config);
1256
+ }
1257
+ function virtualChunkContainersFromConfig(config) {
1258
+ const result = [];
1254
1259
  if (!isRecord(config)) return result;
1255
1260
  const containers = config.virtual_chunk_containers;
1256
1261
  if (!isRecord(containers)) return result;
@@ -1402,6 +1407,81 @@ function resolveSnapshotParent(parsedRepo, snapshotId) {
1402
1407
  return getSnapshotIdByIndex(parsedRepo, parentOffset);
1403
1408
  }
1404
1409
 
1410
+ // src/format/config-yaml.ts
1411
+ var SECTION = "virtual_chunk_containers";
1412
+ var ENTRY = /^('(?:[^']|'')*'|"(?:[^"\\]|\\.)*"|.+?):(?: +(.*))?$/;
1413
+ function parseConfigYamlContainers(yaml) {
1414
+ const lines = yaml.split(/\r?\n/);
1415
+ const start = lines.findIndex((line) => line.startsWith(`${SECTION}:`));
1416
+ if (start < 0) return [];
1417
+ const body = [];
1418
+ for (let i = start + 1; i < lines.length; i++) {
1419
+ if (/^\S/.test(lines[i])) break;
1420
+ if (lines[i].trim() !== "") body.push(lines[i]);
1421
+ }
1422
+ for (let i = 0; i + 1 < body.length; i++) {
1423
+ const indent = indentOf(body[i]);
1424
+ const value = body[i + 1].slice(indent);
1425
+ if (body[i].startsWith("? ", indent) && indentOf(body[i + 1]) === indent && value.startsWith(": ")) {
1426
+ body[i] = `${" ".repeat(indent)}${JSON.stringify(body[i].slice(indent + 2))}:`;
1427
+ body[i + 1] = `${" ".repeat(indent)} ${value.slice(2)}`;
1428
+ }
1429
+ }
1430
+ try {
1431
+ const [containers] = parseMapping(body, 0);
1432
+ return virtualChunkContainersFromConfig({ [SECTION]: containers });
1433
+ } catch {
1434
+ return [];
1435
+ }
1436
+ }
1437
+ function parseMapping(lines, from) {
1438
+ const result = /* @__PURE__ */ Object.create(null);
1439
+ if (from >= lines.length) return [result, from];
1440
+ const indent = indentOf(lines[from]);
1441
+ let i = from;
1442
+ while (i < lines.length && indentOf(lines[i]) >= indent) {
1443
+ const entry = indentOf(lines[i]) === indent ? ENTRY.exec(lines[i].slice(indent)) : null;
1444
+ if (!entry || /^[-?:] /.test(entry[1])) {
1445
+ throw new Error(`Unsupported config.yaml line: ${lines[i]}`);
1446
+ }
1447
+ i++;
1448
+ const key = unquote(entry[1]);
1449
+ const tagged = /^!(\S+)(?: +(.*))?$/.exec(entry[2] ?? "");
1450
+ const rest = (tagged ? tagged[2] ?? "" : entry[2] ?? "").trim();
1451
+ let value;
1452
+ if (rest === "" && i < lines.length && indentOf(lines[i]) > indent) {
1453
+ [value, i] = parseMapping(lines, i);
1454
+ } else {
1455
+ value = parseScalar(rest);
1456
+ }
1457
+ result[key] = tagged ? { [tagged[1]]: value } : value;
1458
+ }
1459
+ return [result, i];
1460
+ }
1461
+ function indentOf(line) {
1462
+ return line.length - line.trimStart().length;
1463
+ }
1464
+ function parseScalar(text) {
1465
+ if (text === "" || text === "null" || text === "~") return null;
1466
+ if (text === "true") return true;
1467
+ if (text === "false") return false;
1468
+ if (text === "{}") return {};
1469
+ const quoted = /^['"]/.test(text);
1470
+ if (/^[[{|>&*]/.test(text) || !quoted && /(^| )#/.test(text)) {
1471
+ throw new Error(`Unsupported config.yaml value: ${text}`);
1472
+ }
1473
+ return unquote(text);
1474
+ }
1475
+ function unquote(text) {
1476
+ if (text.length >= 2 && text.startsWith("'") && text.endsWith("'")) {
1477
+ return text.slice(1, -1).replace(/''/g, "'");
1478
+ }
1479
+ if (text.length >= 2 && text.startsWith('"') && text.endsWith('"')) {
1480
+ return JSON.parse(text);
1481
+ }
1482
+ return text;
1483
+ }
1484
+
1405
1485
  // src/cache/lru.ts
1406
1486
  var LRUCache = class {
1407
1487
  cache;
@@ -3993,6 +4073,7 @@ var Repository = class _Repository {
3993
4073
  storage;
3994
4074
  repoInfo = null;
3995
4075
  repoInfoAttempted = false;
4076
+ v1Containers = null;
3996
4077
  constructor(storage) {
3997
4078
  this.storage = storage;
3998
4079
  }
@@ -4033,6 +4114,42 @@ var Repository = class _Repository {
4033
4114
  );
4034
4115
  }
4035
4116
  }
4117
+ /**
4118
+ * Load and cache the virtual chunk containers of a v1 repository, which
4119
+ * keeps its config in `config.yaml` rather than in a repo file.
4120
+ *
4121
+ * Best-effort, like the v2 config. A repository that never saved a config
4122
+ * has no such file (S3 answers 403 rather than 404 for a missing key of a
4123
+ * bucket that cannot be listed), and a file that cannot be parsed declares
4124
+ * no containers; chunk locations then resolve as they would without a
4125
+ * config. Any other failure also yields no containers but is not cached, so
4126
+ * the next checkout asks again.
4127
+ *
4128
+ * @throws An error named "AbortError" if the operation was aborted
4129
+ */
4130
+ async loadV1Containers(options) {
4131
+ if (this.v1Containers) {
4132
+ return this.v1Containers;
4133
+ }
4134
+ let containers = [];
4135
+ let settled = true;
4136
+ try {
4137
+ const data = await this.storage.getObject(CONFIG_PATH, void 0, {
4138
+ ...options,
4139
+ cache: "no-cache"
4140
+ });
4141
+ containers = parseConfigYamlContainers(new TextDecoder().decode(data));
4142
+ } catch (error) {
4143
+ if (isAbortError(error)) {
4144
+ throw error;
4145
+ }
4146
+ settled = error instanceof NotFoundError || error instanceof StorageError && error.status === 403;
4147
+ }
4148
+ if (settled) {
4149
+ this.v1Containers = containers;
4150
+ }
4151
+ return containers;
4152
+ }
4036
4153
  /**
4037
4154
  * Open an icechunk repository.
4038
4155
  *
@@ -4237,16 +4354,25 @@ var Repository = class _Repository {
4237
4354
  virtualChunkContainers: repoInfo.virtualChunkContainers
4238
4355
  });
4239
4356
  }
4240
- const refDirPath = getBranchRefDirPath(name);
4241
- const refPath = await this.findLatestRefFile(
4242
- refDirPath,
4243
- getBranchRefPath(name)
4244
- );
4245
- if (!refPath) {
4246
- throw new Error(`Reference not found: ${refDirPath}`);
4247
- }
4248
- const snapshotId = await this.readSnapshotIdFromRef(refPath, options);
4249
- return ReadSession.open(this.storage, snapshotId, options);
4357
+ const resolveSnapshotId = async () => {
4358
+ const refDirPath = getBranchRefDirPath(name);
4359
+ const refPath = await this.findLatestRefFile(
4360
+ refDirPath,
4361
+ getBranchRefPath(name)
4362
+ );
4363
+ if (!refPath) {
4364
+ throw new Error(`Reference not found: ${refDirPath}`);
4365
+ }
4366
+ return this.readSnapshotIdFromRef(refPath, options);
4367
+ };
4368
+ const [snapshotId, virtualChunkContainers] = await Promise.all([
4369
+ resolveSnapshotId(),
4370
+ this.loadV1Containers(options)
4371
+ ]);
4372
+ return ReadSession.open(this.storage, snapshotId, {
4373
+ ...options,
4374
+ virtualChunkContainers
4375
+ });
4250
4376
  }
4251
4377
  /**
4252
4378
  * Checkout a tag to get a read session.
@@ -4267,17 +4393,26 @@ var Repository = class _Repository {
4267
4393
  virtualChunkContainers: repoInfo.virtualChunkContainers
4268
4394
  });
4269
4395
  }
4270
- const refDirPath = getTagRefDirPath(name);
4271
- const legacyPath = getTagRefPath(name);
4272
- const refPath = await this.findLatestRefFile(refDirPath, legacyPath);
4273
- if (!refPath) {
4274
- throw new Error(`Reference not found: ${refDirPath}`);
4275
- }
4276
- if (refPath === legacyPath && await this.storage.exists(`${refPath}.deleted`, options)) {
4277
- throw new Error(`Tag not found: ${name}`);
4278
- }
4279
- const snapshotId = await this.readSnapshotIdFromRef(refPath, options);
4280
- return ReadSession.open(this.storage, snapshotId, options);
4396
+ const resolveSnapshotId = async () => {
4397
+ const refDirPath = getTagRefDirPath(name);
4398
+ const legacyPath = getTagRefPath(name);
4399
+ const refPath = await this.findLatestRefFile(refDirPath, legacyPath);
4400
+ if (!refPath) {
4401
+ throw new Error(`Reference not found: ${refDirPath}`);
4402
+ }
4403
+ if (refPath === legacyPath && await this.storage.exists(`${refPath}.deleted`, options)) {
4404
+ throw new Error(`Tag not found: ${name}`);
4405
+ }
4406
+ return this.readSnapshotIdFromRef(refPath, options);
4407
+ };
4408
+ const [snapshotId, virtualChunkContainers] = await Promise.all([
4409
+ resolveSnapshotId(),
4410
+ this.loadV1Containers(options)
4411
+ ]);
4412
+ return ReadSession.open(this.storage, snapshotId, {
4413
+ ...options,
4414
+ virtualChunkContainers
4415
+ });
4281
4416
  }
4282
4417
  /**
4283
4418
  * Checkout a specific snapshot by ID.
@@ -4289,10 +4424,9 @@ var Repository = class _Repository {
4289
4424
  async checkoutSnapshot(snapshotId, options) {
4290
4425
  const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
4291
4426
  const repoInfo = await this.loadRepoInfo(options);
4292
- const virtualChunkContainers = repoInfo?.virtualChunkContainers;
4293
4427
  return ReadSession.open(this.storage, id, {
4294
4428
  ...options,
4295
- ...virtualChunkContainers && { virtualChunkContainers }
4429
+ virtualChunkContainers: repoInfo ? repoInfo.virtualChunkContainers : await this.loadV1Containers(options)
4296
4430
  });
4297
4431
  }
4298
4432
  /**