icechunk-js 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -163,6 +163,13 @@ a region (repos written by Icechunk do), reads go straight to that regional
163
163
  endpoint — which serves CORS and works in both Node and the browser, with no
164
164
  extra configuration.
165
165
 
166
+ The config is read from the repo file of a v2 repository and from `config.yaml`
167
+ of a v1 repository. The `config.yaml` reader covers the layout Icechunk writes
168
+ for ordinary values, not YAML in general: a containers section it cannot read
169
+ (one edited by hand into another style, or holding a value with control
170
+ characters or a line break) is ignored as a whole, as is a missing or
171
+ unreadable file.
172
+
166
173
  If no region is known, dotted-name buckets fall back to the global endpoint,
167
174
  whose region redirect (for buckets outside `us-east-1`) is resolved
168
175
  automatically in Node but not in browsers — the cross-origin redirect carries no
package/dist/index.cjs CHANGED
@@ -105,6 +105,7 @@ var PATHS = {
105
105
  TRANSACTIONS: "transactions"
106
106
  };
107
107
  var REPO_INFO_PATH = "repo";
108
+ var CONFIG_PATH = "config.yaml";
108
109
  function getBranchRefDirPath(name) {
109
110
  return `${PATHS.REFS}/branch.${name}/`;
110
111
  }
@@ -1305,6 +1306,10 @@ function parseVirtualChunkContainers(repo) {
1305
1306
  } catch {
1306
1307
  return result;
1307
1308
  }
1309
+ return virtualChunkContainersFromConfig(config);
1310
+ }
1311
+ function virtualChunkContainersFromConfig(config) {
1312
+ const result = [];
1308
1313
  if (!isRecord(config)) return result;
1309
1314
  const containers = config.virtual_chunk_containers;
1310
1315
  if (!isRecord(containers)) return result;
@@ -1456,6 +1461,81 @@ function resolveSnapshotParent(parsedRepo, snapshotId) {
1456
1461
  return getSnapshotIdByIndex(parsedRepo, parentOffset);
1457
1462
  }
1458
1463
 
1464
+ // src/format/config-yaml.ts
1465
+ var SECTION = "virtual_chunk_containers";
1466
+ var ENTRY = /^('(?:[^']|'')*'|"(?:[^"\\]|\\.)*"|.+?):(?: +(.*))?$/;
1467
+ function parseConfigYamlContainers(yaml) {
1468
+ const lines = yaml.split(/\r?\n/);
1469
+ const start = lines.findIndex((line) => line.startsWith(`${SECTION}:`));
1470
+ if (start < 0) return [];
1471
+ const body = [];
1472
+ for (let i = start + 1; i < lines.length; i++) {
1473
+ if (/^\S/.test(lines[i])) break;
1474
+ if (lines[i].trim() !== "") body.push(lines[i]);
1475
+ }
1476
+ for (let i = 0; i + 1 < body.length; i++) {
1477
+ const indent = indentOf(body[i]);
1478
+ const value = body[i + 1].slice(indent);
1479
+ if (body[i].startsWith("? ", indent) && indentOf(body[i + 1]) === indent && value.startsWith(": ")) {
1480
+ body[i] = `${" ".repeat(indent)}${JSON.stringify(body[i].slice(indent + 2))}:`;
1481
+ body[i + 1] = `${" ".repeat(indent)} ${value.slice(2)}`;
1482
+ }
1483
+ }
1484
+ try {
1485
+ const [containers] = parseMapping(body, 0);
1486
+ return virtualChunkContainersFromConfig({ [SECTION]: containers });
1487
+ } catch {
1488
+ return [];
1489
+ }
1490
+ }
1491
+ function parseMapping(lines, from) {
1492
+ const result = /* @__PURE__ */ Object.create(null);
1493
+ if (from >= lines.length) return [result, from];
1494
+ const indent = indentOf(lines[from]);
1495
+ let i = from;
1496
+ while (i < lines.length && indentOf(lines[i]) >= indent) {
1497
+ const entry = indentOf(lines[i]) === indent ? ENTRY.exec(lines[i].slice(indent)) : null;
1498
+ if (!entry || /^[-?:] /.test(entry[1])) {
1499
+ throw new Error(`Unsupported config.yaml line: ${lines[i]}`);
1500
+ }
1501
+ i++;
1502
+ const key = unquote(entry[1]);
1503
+ const tagged = /^!(\S+)(?: +(.*))?$/.exec(entry[2] ?? "");
1504
+ const rest = (tagged ? tagged[2] ?? "" : entry[2] ?? "").trim();
1505
+ let value;
1506
+ if (rest === "" && i < lines.length && indentOf(lines[i]) > indent) {
1507
+ [value, i] = parseMapping(lines, i);
1508
+ } else {
1509
+ value = parseScalar(rest);
1510
+ }
1511
+ result[key] = tagged ? { [tagged[1]]: value } : value;
1512
+ }
1513
+ return [result, i];
1514
+ }
1515
+ function indentOf(line) {
1516
+ return line.length - line.trimStart().length;
1517
+ }
1518
+ function parseScalar(text) {
1519
+ if (text === "" || text === "null" || text === "~") return null;
1520
+ if (text === "true") return true;
1521
+ if (text === "false") return false;
1522
+ if (text === "{}") return {};
1523
+ const quoted = /^['"]/.test(text);
1524
+ if (/^[[{|>&*]/.test(text) || !quoted && /(^| )#/.test(text)) {
1525
+ throw new Error(`Unsupported config.yaml value: ${text}`);
1526
+ }
1527
+ return unquote(text);
1528
+ }
1529
+ function unquote(text) {
1530
+ if (text.length >= 2 && text.startsWith("'") && text.endsWith("'")) {
1531
+ return text.slice(1, -1).replace(/''/g, "'");
1532
+ }
1533
+ if (text.length >= 2 && text.startsWith('"') && text.endsWith('"')) {
1534
+ return JSON.parse(text);
1535
+ }
1536
+ return text;
1537
+ }
1538
+
1459
1539
  // src/cache/lru.ts
1460
1540
  var LRUCache = class {
1461
1541
  cache;
@@ -4047,6 +4127,7 @@ var Repository = class _Repository {
4047
4127
  storage;
4048
4128
  repoInfo = null;
4049
4129
  repoInfoAttempted = false;
4130
+ v1Containers = null;
4050
4131
  constructor(storage) {
4051
4132
  this.storage = storage;
4052
4133
  }
@@ -4087,6 +4168,42 @@ var Repository = class _Repository {
4087
4168
  );
4088
4169
  }
4089
4170
  }
4171
+ /**
4172
+ * Load and cache the virtual chunk containers of a v1 repository, which
4173
+ * keeps its config in `config.yaml` rather than in a repo file.
4174
+ *
4175
+ * Best-effort, like the v2 config. A repository that never saved a config
4176
+ * has no such file (S3 answers 403 rather than 404 for a missing key of a
4177
+ * bucket that cannot be listed), and a file that cannot be parsed declares
4178
+ * no containers; chunk locations then resolve as they would without a
4179
+ * config. Any other failure also yields no containers but is not cached, so
4180
+ * the next checkout asks again.
4181
+ *
4182
+ * @throws An error named "AbortError" if the operation was aborted
4183
+ */
4184
+ async loadV1Containers(options) {
4185
+ if (this.v1Containers) {
4186
+ return this.v1Containers;
4187
+ }
4188
+ let containers = [];
4189
+ let settled = true;
4190
+ try {
4191
+ const data = await this.storage.getObject(CONFIG_PATH, void 0, {
4192
+ ...options,
4193
+ cache: "no-cache"
4194
+ });
4195
+ containers = parseConfigYamlContainers(new TextDecoder().decode(data));
4196
+ } catch (error) {
4197
+ if (isAbortError(error)) {
4198
+ throw error;
4199
+ }
4200
+ settled = error instanceof NotFoundError || error instanceof StorageError && error.status === 403;
4201
+ }
4202
+ if (settled) {
4203
+ this.v1Containers = containers;
4204
+ }
4205
+ return containers;
4206
+ }
4090
4207
  /**
4091
4208
  * Open an icechunk repository.
4092
4209
  *
@@ -4291,16 +4408,25 @@ var Repository = class _Repository {
4291
4408
  virtualChunkContainers: repoInfo.virtualChunkContainers
4292
4409
  });
4293
4410
  }
4294
- const refDirPath = getBranchRefDirPath(name);
4295
- const refPath = await this.findLatestRefFile(
4296
- refDirPath,
4297
- getBranchRefPath(name)
4298
- );
4299
- if (!refPath) {
4300
- throw new Error(`Reference not found: ${refDirPath}`);
4301
- }
4302
- const snapshotId = await this.readSnapshotIdFromRef(refPath, options);
4303
- return ReadSession.open(this.storage, snapshotId, options);
4411
+ const resolveSnapshotId = async () => {
4412
+ const refDirPath = getBranchRefDirPath(name);
4413
+ const refPath = await this.findLatestRefFile(
4414
+ refDirPath,
4415
+ getBranchRefPath(name)
4416
+ );
4417
+ if (!refPath) {
4418
+ throw new Error(`Reference not found: ${refDirPath}`);
4419
+ }
4420
+ return this.readSnapshotIdFromRef(refPath, options);
4421
+ };
4422
+ const [snapshotId, virtualChunkContainers] = await Promise.all([
4423
+ resolveSnapshotId(),
4424
+ this.loadV1Containers(options)
4425
+ ]);
4426
+ return ReadSession.open(this.storage, snapshotId, {
4427
+ ...options,
4428
+ virtualChunkContainers
4429
+ });
4304
4430
  }
4305
4431
  /**
4306
4432
  * Checkout a tag to get a read session.
@@ -4321,17 +4447,26 @@ var Repository = class _Repository {
4321
4447
  virtualChunkContainers: repoInfo.virtualChunkContainers
4322
4448
  });
4323
4449
  }
4324
- const refDirPath = getTagRefDirPath(name);
4325
- const legacyPath = getTagRefPath(name);
4326
- const refPath = await this.findLatestRefFile(refDirPath, legacyPath);
4327
- if (!refPath) {
4328
- throw new Error(`Reference not found: ${refDirPath}`);
4329
- }
4330
- if (refPath === legacyPath && await this.storage.exists(`${refPath}.deleted`, options)) {
4331
- throw new Error(`Tag not found: ${name}`);
4332
- }
4333
- const snapshotId = await this.readSnapshotIdFromRef(refPath, options);
4334
- return ReadSession.open(this.storage, snapshotId, options);
4450
+ const resolveSnapshotId = async () => {
4451
+ const refDirPath = getTagRefDirPath(name);
4452
+ const legacyPath = getTagRefPath(name);
4453
+ const refPath = await this.findLatestRefFile(refDirPath, legacyPath);
4454
+ if (!refPath) {
4455
+ throw new Error(`Reference not found: ${refDirPath}`);
4456
+ }
4457
+ if (refPath === legacyPath && await this.storage.exists(`${refPath}.deleted`, options)) {
4458
+ throw new Error(`Tag not found: ${name}`);
4459
+ }
4460
+ return this.readSnapshotIdFromRef(refPath, options);
4461
+ };
4462
+ const [snapshotId, virtualChunkContainers] = await Promise.all([
4463
+ resolveSnapshotId(),
4464
+ this.loadV1Containers(options)
4465
+ ]);
4466
+ return ReadSession.open(this.storage, snapshotId, {
4467
+ ...options,
4468
+ virtualChunkContainers
4469
+ });
4335
4470
  }
4336
4471
  /**
4337
4472
  * Checkout a specific snapshot by ID.
@@ -4343,10 +4478,9 @@ var Repository = class _Repository {
4343
4478
  async checkoutSnapshot(snapshotId, options) {
4344
4479
  const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
4345
4480
  const repoInfo = await this.loadRepoInfo(options);
4346
- const virtualChunkContainers = repoInfo?.virtualChunkContainers;
4347
4481
  return ReadSession.open(this.storage, id, {
4348
4482
  ...options,
4349
- ...virtualChunkContainers && { virtualChunkContainers }
4483
+ virtualChunkContainers: repoInfo ? repoInfo.virtualChunkContainers : await this.loadV1Containers(options)
4350
4484
  });
4351
4485
  }
4352
4486
  /**