keepset 0.39.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { $ as ArchiveError, H as getCollectionTarget, L as getOwn, U as getTarget, at as RecipeError, it as NativeParseError, rt as KeepsetError, t as getDataset, z as countTarget } from "../bundled-6dYCPVD-.js";
1
+ import { J as getTarget, K as getCollectionTarget, U as countTarget, V as getOwn, Y as isStreamable, dt as NativeParseError, ft as RecipeError, ot as ArchiveError, q as getStreamedShape, t as getDataset, ut as KeepsetError } from "../bundled-CL1eRNRR.js";
2
2
  import { RoundTripMismatchError, assertRoundTrip, encode } from "crossmap";
3
3
  import { z } from "zod";
4
4
  import { createHash } from "node:crypto";
@@ -222,8 +222,8 @@ function parseEdgeList(text, options) {
222
222
  }
223
223
  //#endregion
224
224
  //#region src/parsers/gd-contest.ts
225
- const FORMAT$15 = "gd-contest";
226
- const NUMBER$9 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
225
+ const FORMAT$18 = "gd-contest";
226
+ const NUMBER$10 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
227
227
  /**
228
228
  * Reads a vertex number, which the format numbers from 0 to N - 1.
229
229
  * @param token - The token.
@@ -231,7 +231,7 @@ const NUMBER$9 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
231
231
  * @returns The vertex id as text.
232
232
  */
233
233
  function vertex$1(token, count) {
234
- if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$15, token.line, `expected a vertex number from 0 to ${String(count - 1)}, found ${JSON.stringify(token.text)}`);
234
+ if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$18, token.line, `expected a vertex number from 0 to ${String(count - 1)}, found ${JSON.stringify(token.text)}`);
235
235
  return token.text;
236
236
  }
237
237
  /**
@@ -251,17 +251,17 @@ function parseGdContest(text) {
251
251
  });
252
252
  }
253
253
  const [first] = tokens;
254
- if (first === void 0) throw new NativeParseError(FORMAT$15, 1, "the file holds no vertex count");
255
- if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$15, first.line, `the vertex count must be a non-negative integer, found ${JSON.stringify(first.text)}`);
254
+ if (first === void 0) throw new NativeParseError(FORMAT$18, 1, "the file holds no vertex count");
255
+ if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$18, first.line, `the vertex count must be a non-negative integer, found ${JSON.stringify(first.text)}`);
256
256
  const count = Number(first.text);
257
257
  const coordinates = tokens.slice(1, 1 + 2 * count);
258
- if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$15, lines(text).length, `expected ${String(2 * count)} coordinates for ${String(count)} vertices, found ${String(coordinates.length)}`);
259
- for (const coordinate of coordinates) if (!NUMBER$9.test(coordinate.text)) throw new NativeParseError(FORMAT$15, coordinate.line, `coordinate ${JSON.stringify(coordinate.text)} is not a number`);
258
+ if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$18, lines(text).length, `expected ${String(2 * count)} coordinates for ${String(count)} vertices, found ${String(coordinates.length)}`);
259
+ for (const coordinate of coordinates) if (!NUMBER$10.test(coordinate.text)) throw new NativeParseError(FORMAT$18, coordinate.line, `coordinate ${JSON.stringify(coordinate.text)} is not a number`);
260
260
  const ends = tokens.slice(1 + 2 * count);
261
261
  const edges = [];
262
262
  for (let source = ends.shift(); source !== void 0; source = ends.shift()) {
263
263
  const target = ends.shift();
264
- if (target === void 0) throw new NativeParseError(FORMAT$15, source.line, `edge from vertex ${source.text} has no second vertex`);
264
+ if (target === void 0) throw new NativeParseError(FORMAT$18, source.line, `edge from vertex ${source.text} has no second vertex`);
265
265
  edges.push({
266
266
  source: vertex$1(source, count),
267
267
  target: vertex$1(target, count)
@@ -275,7 +275,7 @@ function parseGdContest(text) {
275
275
  //#endregion
276
276
  //#region src/parsers/gml.ts
277
277
  const KEY = /[A-Za-z_][A-Za-z0-9_]*/y;
278
- const NUMBER$8 = /[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?/y;
278
+ const NUMBER$9 = /[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?/y;
279
279
  /**
280
280
  * Matches a sticky pattern at a position.
281
281
  * @param pattern - A sticky regular expression.
@@ -326,7 +326,7 @@ function tokenise(text) {
326
326
  position = end + 1;
327
327
  } else {
328
328
  const key = matchAt(KEY, text, position);
329
- const number = key === void 0 ? matchAt(NUMBER$8, text, position) : void 0;
329
+ const number = key === void 0 ? matchAt(NUMBER$9, text, position) : void 0;
330
330
  const match = key ?? number;
331
331
  if (match === void 0) throw new NativeParseError("gml", line, `unexpected character ${JSON.stringify(char)}`);
332
332
  tokens.push({
@@ -431,7 +431,7 @@ function parseGml(text) {
431
431
  }
432
432
  //#endregion
433
433
  //#region src/parsers/hgb.ts
434
- const FORMAT$14 = "hgb";
434
+ const FORMAT$17 = "hgb";
435
435
  /** A type code or node id as the HGB files write them: a non-negative decimal integer with no sign or leading zero. */
436
436
  const CODE = /^(?:0|[1-9]\d*)$/;
437
437
  /** A number as a decimal literal with an optional sign and exponent, so `Number` cannot read hex, an empty string or infinity. */
@@ -451,7 +451,7 @@ const LINK_FIELDS = 4;
451
451
  */
452
452
  function resolve(field, types, where) {
453
453
  const type = types[CODE.test(field) ? Number(field) : -1];
454
- if (type === void 0) throw new NativeParseError(FORMAT$14, where.line, `${where.file}: the ${where.kind} type ${JSON.stringify(field)} is not one of the ${String(types.length)} the recipe names`);
454
+ if (type === void 0) throw new NativeParseError(FORMAT$17, where.line, `${where.file}: the ${where.kind} type ${JSON.stringify(field)} is not one of the ${String(types.length)} the recipe names`);
455
455
  return type;
456
456
  }
457
457
  /**
@@ -469,9 +469,9 @@ function parseHgb(parts, options) {
469
469
  forEachLine(parts.nodes, (line, number) => {
470
470
  const fields = line.split(" ");
471
471
  const [id, , type] = fields;
472
- if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$14, number, `node.dat: expected ${String(NODE_FIELDS)} or ${String(NODE_FIELDS_WITH_ATTRIBUTES)} fields separated by tabs, found ${String(fields.length)}`);
473
- if (!CODE.test(id)) throw new NativeParseError(FORMAT$14, number, `node.dat: the node id ${JSON.stringify(id)} is not a non-negative integer`);
474
- if (typeOf.has(id)) throw new NativeParseError(FORMAT$14, number, `node.dat: the node id ${id} is repeated`);
472
+ if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$17, number, `node.dat: expected ${String(NODE_FIELDS)} or ${String(NODE_FIELDS_WITH_ATTRIBUTES)} fields separated by tabs, found ${String(fields.length)}`);
473
+ if (!CODE.test(id)) throw new NativeParseError(FORMAT$17, number, `node.dat: the node id ${JSON.stringify(id)} is not a non-negative integer`);
474
+ if (typeOf.has(id)) throw new NativeParseError(FORMAT$17, number, `node.dat: the node id ${id} is repeated`);
475
475
  const nodeType = resolve(type, options.nodeTypes, {
476
476
  file: "node.dat",
477
477
  line: number,
@@ -483,12 +483,12 @@ function parseHgb(parts, options) {
483
483
  type: nodeType
484
484
  });
485
485
  });
486
- if (nodes.length === 0) throw new NativeParseError(FORMAT$14, 1, "node.dat: the file holds no nodes");
486
+ if (nodes.length === 0) throw new NativeParseError(FORMAT$17, 1, "node.dat: the file holds no nodes");
487
487
  const edges = [];
488
488
  forEachLine(parts.links, (line, number) => {
489
489
  const fields = line.split(" ");
490
490
  const [source, target, type, weight] = fields;
491
- if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$14, number, `link.dat: expected ${String(LINK_FIELDS)} fields separated by tabs, found ${String(fields.length)}`);
491
+ if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$17, number, `link.dat: expected ${String(LINK_FIELDS)} fields separated by tabs, found ${String(fields.length)}`);
492
492
  const linkType = resolve(type, options.linkTypes, {
493
493
  file: "link.dat",
494
494
  line: number,
@@ -504,17 +504,17 @@ function parseHgb(parts, options) {
504
504
  linkType.target
505
505
  ]]) {
506
506
  const actual = typeOf.get(id);
507
- if (actual === void 0) throw new NativeParseError(FORMAT$14, number, `link.dat: the ${end} ${JSON.stringify(id)} is not a node of node.dat`);
508
- if (actual !== expected) throw new NativeParseError(FORMAT$14, number, `link.dat: a ${linkType.name} link's ${end} must be a ${expected} node, and ${id} is a ${actual} node`);
507
+ if (actual === void 0) throw new NativeParseError(FORMAT$17, number, `link.dat: the ${end} ${JSON.stringify(id)} is not a node of node.dat`);
508
+ if (actual !== expected) throw new NativeParseError(FORMAT$17, number, `link.dat: a ${linkType.name} link's ${end} must be a ${expected} node, and ${id} is a ${actual} node`);
509
509
  }
510
- if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$14, number, `link.dat: the weight ${JSON.stringify(weight)} is not 1; a weighted HGB file would lose its weights`);
510
+ if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$17, number, `link.dat: the weight ${JSON.stringify(weight)} is not 1; a weighted HGB file would lose its weights`);
511
511
  edges.push({
512
512
  source,
513
513
  target,
514
514
  type: linkType.name
515
515
  });
516
516
  });
517
- if (edges.length === 0) throw new NativeParseError(FORMAT$14, 1, "link.dat: the file holds no links");
517
+ if (edges.length === 0) throw new NativeParseError(FORMAT$17, 1, "link.dat: the file holds no links");
518
518
  return {
519
519
  nodes,
520
520
  edges
@@ -522,7 +522,7 @@ function parseHgb(parts, options) {
522
522
  }
523
523
  //#endregion
524
524
  //#region src/parsers/kg-triples.ts
525
- const FORMAT$13 = "kg-triples";
525
+ const FORMAT$16 = "kg-triples";
526
526
  /** The character each delimiter option splits a line on. A line is split on exactly that character, so a field cannot hide a second delimiter. */
527
527
  const DELIMITERS$1 = {
528
528
  tab: " ",
@@ -544,8 +544,8 @@ const QUOTE$2 = "\"";
544
544
  */
545
545
  function value(field, quoted, where) {
546
546
  const inner = quoted ? field.slice(1, -1) : field;
547
- if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$13, where.line, `${where.part}the field ${JSON.stringify(field)} is not one value wrapped in double quotes`);
548
- if (inner === "") throw new NativeParseError(FORMAT$13, where.line, `${where.part}a field is empty`);
547
+ if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$16, where.line, `${where.part}the field ${JSON.stringify(field)} is not one value wrapped in double quotes`);
548
+ if (inner === "") throw new NativeParseError(FORMAT$16, where.line, `${where.part}a field is empty`);
549
549
  return inner;
550
550
  }
551
551
  /**
@@ -569,16 +569,16 @@ function parseKgTriples(parts, options) {
569
569
  let lastLine = 0;
570
570
  forEachLine(part.pieces, (line, number) => {
571
571
  lastLine = number;
572
- if (line.trim() === "") throw new NativeParseError(FORMAT$13, number, `${label}the line is blank`);
572
+ if (line.trim() === "") throw new NativeParseError(FORMAT$16, number, `${label}the line is blank`);
573
573
  if (number === 1 && options.header === "names") return;
574
574
  if (number === 1 && options.header === "count") {
575
- if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$13, number, `${label}expected the number of triples on the first line, found ${JSON.stringify(line)}`);
575
+ if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$16, number, `${label}expected the number of triples on the first line, found ${JSON.stringify(line)}`);
576
576
  declared = Number(line);
577
577
  return;
578
578
  }
579
579
  const fields = line.split(delimiter);
580
580
  const [head, middle, last] = fields;
581
- if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$13, number, `${label}expected ${String(TRIPLE_FIELDS)} fields separated by ${options.delimiter}, found ${String(fields.length)}`);
581
+ if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$16, number, `${label}expected ${String(TRIPLE_FIELDS)} fields separated by ${options.delimiter}, found ${String(fields.length)}`);
582
582
  const where = {
583
583
  line: number,
584
584
  part: label
@@ -604,8 +604,8 @@ function parseKgTriples(parts, options) {
604
604
  });
605
605
  triples += 1;
606
606
  });
607
- if (triples === 0) throw new NativeParseError(FORMAT$13, lastLine, `${label}the file holds no triples`);
608
- if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$13, lastLine, `${label}the first line declares ${String(declared)} triples, the file holds ${String(triples)}`);
607
+ if (triples === 0) throw new NativeParseError(FORMAT$16, lastLine, `${label}the file holds no triples`);
608
+ if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$16, lastLine, `${label}the first line declares ${String(declared)} triples, the file holds ${String(triples)}`);
609
609
  }
610
610
  return {
611
611
  nodes: order.nodes(),
@@ -614,7 +614,7 @@ function parseKgTriples(parts, options) {
614
614
  }
615
615
  //#endregion
616
616
  //#region src/parsers/konect.ts
617
- const FORMAT$12 = "konect";
617
+ const FORMAT$15 = "konect";
618
618
  /** The network formats of KONECT's first line (Kunegis, "The KONECT Handbook", table 1): undirected, directed and bipartite. */
619
619
  const NETWORK_FORMATS = /* @__PURE__ */ new Set([
620
620
  "sym",
@@ -689,9 +689,9 @@ function parseKonect(pieces, options) {
689
689
  }));
690
690
  });
691
691
  const { header } = state;
692
- if (header === void 0) throw new NativeParseError(FORMAT$12, lastLine, "the file has no first line naming its format and weights");
693
- if (edges.length === 0) throw new NativeParseError(FORMAT$12, lastLine, "the file holds no edges");
694
- if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$12, lastLine, `the second line declares ${String(header.counts.edges)} edges, the file holds ${String(edges.length)}`);
692
+ if (header === void 0) throw new NativeParseError(FORMAT$15, lastLine, "the file has no first line naming its format and weights");
693
+ if (edges.length === 0) throw new NativeParseError(FORMAT$15, lastLine, "the file holds no edges");
694
+ if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$15, lastLine, `the second line declares ${String(header.counts.edges)} edges, the file holds ${String(edges.length)}`);
695
695
  return {
696
696
  nodes: order.nodes(),
697
697
  edges
@@ -709,9 +709,9 @@ function readHeader(content, number, options) {
709
709
  const match = FORMAT_LINE.exec(content);
710
710
  const network = match?.[1];
711
711
  const weights = match?.[2];
712
- if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$12, number, `expected a first line "% FORMAT WEIGHTS" with FORMAT one of ${[...NETWORK_FORMATS].join(", ")}, found ${JSON.stringify(content)}`);
713
- if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$12, number, `the weight type ${JSON.stringify(weights)} is not one this parser reads (its third column is not an edge weight or a repetition count)`);
714
- if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$12, number, `the file's weight type is ${weights}, so a recipe with weighted ${String(WEIGHTED_TYPES.has(weights))} reads it, not one with weighted ${String(options.weighted)}`);
712
+ if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$15, number, `expected a first line "% FORMAT WEIGHTS" with FORMAT one of ${[...NETWORK_FORMATS].join(", ")}, found ${JSON.stringify(content)}`);
713
+ if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$15, number, `the weight type ${JSON.stringify(weights)} is not one this parser reads (its third column is not an edge weight or a repetition count)`);
714
+ if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$15, number, `the file's weight type is ${weights}, so a recipe with weighted ${String(WEIGHTED_TYPES.has(weights))} reads it, not one with weighted ${String(options.weighted)}`);
715
715
  return {
716
716
  line: number,
717
717
  network
@@ -731,15 +731,15 @@ function readEdge(content, number, context) {
731
731
  const [from, to, third, time] = fields;
732
732
  const expected = options.timestamped ? TIMESTAMPED_COLUMNS : options.weighted ? WEIGHTED_COLUMNS : void 0;
733
733
  const fits = expected === void 0 ? fields.length === ENDPOINT_COLUMNS || fields.length === WEIGHTED_COLUMNS : fields.length === expected;
734
- if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$12, number, `expected ${expected === void 0 ? `${String(ENDPOINT_COLUMNS)} or ${String(WEIGHTED_COLUMNS)}` : String(expected)} columns for this recipe, found ${String(fields.length)}`);
734
+ if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$15, number, `expected ${expected === void 0 ? `${String(ENDPOINT_COLUMNS)} or ${String(WEIGHTED_COLUMNS)}` : String(expected)} columns for this recipe, found ${String(fields.length)}`);
735
735
  const bipartite = header.network === "bip";
736
736
  const source = endpoint(from, bipartite ? LEFT_PREFIX : "", header.counts?.left, number);
737
737
  const target = endpoint(to, bipartite ? RIGHT_PREFIX : "", header.counts?.right, number);
738
738
  order.add(source);
739
739
  order.add(target);
740
- if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$12, number, `the third column ${JSON.stringify(third)} is not 1; an edge that stands for several would be read as one`);
741
- if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$12, number, `the weight ${JSON.stringify(third)} is not a number`);
742
- if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$12, number, `the time ${JSON.stringify(time)} is not a number`);
740
+ if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$15, number, `the third column ${JSON.stringify(third)} is not 1; an edge that stands for several would be read as one`);
741
+ if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$15, number, `the weight ${JSON.stringify(third)} is not a number`);
742
+ if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$15, number, `the time ${JSON.stringify(time)} is not a number`);
743
743
  const edge = {
744
744
  source,
745
745
  target
@@ -758,12 +758,12 @@ function readEdge(content, number, context) {
758
758
  * @throws NativeParseError when the token is not a positive integer or exceeds the set's size.
759
759
  */
760
760
  function endpoint(token, prefix, size, number) {
761
- if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$12, number, `${JSON.stringify(token)} is not a node id from 1${size === void 0 ? "" : ` to ${String(size)}`}`);
761
+ if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$15, number, `${JSON.stringify(token)} is not a node id from 1${size === void 0 ? "" : ` to ${String(size)}`}`);
762
762
  return `${prefix}${token}`;
763
763
  }
764
764
  //#endregion
765
765
  //#region src/parsers/ogb.ts
766
- const FORMAT$11 = "ogb";
766
+ const FORMAT$14 = "ogb";
767
767
  /** The files of an OGB raw directory this reader takes, by their path under the dataset's root directory. A relation's files sit in `raw/relations/<head>___<relation>___<tail>/`. */
768
768
  const RAW = /^(?<root>(?:[^/]+\/)*?)raw\/(?<name>num-node-list|num-node-dict|edge|num-edge-list|triplet-type-list|relations\/[^/]+\/(?:edge|num-edge-list))\.csv\.gz$/u;
769
769
  const INDEX = /^(?:0|[1-9]\d*)$/;
@@ -812,7 +812,7 @@ function rows(file) {
812
812
  function onlyLine(file) {
813
813
  const all = rows(file);
814
814
  const [only] = all;
815
- if (all.length !== 1 || only === void 0) throw new NativeParseError(FORMAT$11, all[1]?.line ?? 1, `${file.path} must hold one line, found ${String(all.length)}`);
815
+ if (all.length !== 1 || only === void 0) throw new NativeParseError(FORMAT$14, all[1]?.line ?? 1, `${file.path} must hold one line, found ${String(all.length)}`);
816
816
  return only.text.trim();
817
817
  }
818
818
  /**
@@ -825,7 +825,7 @@ function onlyLine(file) {
825
825
  */
826
826
  function index(text, file, line) {
827
827
  const trimmed = text.trim();
828
- if (!INDEX.test(trimmed)) throw new NativeParseError(FORMAT$11, line, `${file.path}: expected a non-negative integer, found ${JSON.stringify(text)}`);
828
+ if (!INDEX.test(trimmed)) throw new NativeParseError(FORMAT$14, line, `${file.path}: expected a non-negative integer, found ${JSON.stringify(text)}`);
829
829
  return Number(trimmed);
830
830
  }
831
831
  /**
@@ -842,17 +842,17 @@ function readEdges(edges, count, ends, emit) {
842
842
  forEachRow(edges, ({ text, line }) => {
843
843
  const fields = text.split(",");
844
844
  const [first, second] = fields;
845
- if (fields.length !== 2 || first === void 0 || second === void 0) throw new NativeParseError(FORMAT$11, line, `${edges.path}: expected a source and a target index, found ${JSON.stringify(text)}`);
845
+ if (fields.length !== 2 || first === void 0 || second === void 0) throw new NativeParseError(FORMAT$14, line, `${edges.path}: expected a source and a target index, found ${JSON.stringify(text)}`);
846
846
  const source = index(first, edges, line);
847
847
  const target = index(second, edges, line);
848
- if (source >= ends.sources || target >= ends.targets) throw new NativeParseError(FORMAT$11, line, `${edges.path}: edge ${text} names a node outside the ${String(ends.sources)} sources and ${String(ends.targets)} targets the dataset records`);
848
+ if (source >= ends.sources || target >= ends.targets) throw new NativeParseError(FORMAT$14, line, `${edges.path}: edge ${text} names a node outside the ${String(ends.sources)} sources and ${String(ends.targets)} targets the dataset records`);
849
849
  emit({
850
850
  source: ends.source(source),
851
851
  target: ends.target(target)
852
852
  });
853
853
  read += 1;
854
854
  });
855
- if (read !== recorded) throw new NativeParseError(FORMAT$11, read, `${edges.path} holds ${String(read)} edges, but ${count.path} records ${String(recorded)}`);
855
+ if (read !== recorded) throw new NativeParseError(FORMAT$14, read, `${edges.path} holds ${String(read)} edges, but ${count.path} records ${String(recorded)}`);
856
856
  }
857
857
  /**
858
858
  * Takes one file of the dataset directory by its name under `raw/`.
@@ -864,7 +864,7 @@ function readEdges(edges, count, ends, emit) {
864
864
  */
865
865
  function required$1(files, name, root) {
866
866
  const file = files.get(name);
867
- if (file === void 0) throw new NativeParseError(FORMAT$11, 1, `the dataset directory ${root} has no raw/${name}.csv.gz among the files the recipe selects`);
867
+ if (file === void 0) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has no raw/${name}.csv.gz among the files the recipe selects`);
868
868
  return file;
869
869
  }
870
870
  /**
@@ -880,20 +880,20 @@ function parseOgb(files) {
880
880
  const match = RAW.exec(file.path);
881
881
  const name = match?.groups?.["name"];
882
882
  const root = match?.groups?.["root"];
883
- if (name === void 0 || root === void 0) throw new NativeParseError(FORMAT$11, 1, `${file.path} is not a file of an OGB raw directory this reader takes`);
883
+ if (name === void 0 || root === void 0) throw new NativeParseError(FORMAT$14, 1, `${file.path} is not a file of an OGB raw directory this reader takes`);
884
884
  roots.add(root);
885
885
  byName.set(name, file);
886
886
  }
887
887
  const [root] = roots;
888
- if (roots.size !== 1 || root === void 0) throw new NativeParseError(FORMAT$11, 1, `the recipe must select the raw files of one dataset, found ${roots.size === 0 ? "none" : `the directories ${[...roots].join(", ")}`}`);
888
+ if (roots.size !== 1 || root === void 0) throw new NativeParseError(FORMAT$14, 1, `the recipe must select the raw files of one dataset, found ${roots.size === 0 ? "none" : `the directories ${[...roots].join(", ")}`}`);
889
889
  const homogeneous = byName.get("num-node-list");
890
890
  const heterogeneous = byName.get("num-node-dict");
891
891
  const neitherOrBoth = `the dataset directory ${root} must have exactly one of raw/num-node-list.csv.gz (one node type) and raw/num-node-dict.csv.gz (several)`;
892
892
  const edges = [];
893
893
  if (homogeneous !== void 0) {
894
- if (heterogeneous !== void 0) throw new NativeParseError(FORMAT$11, 1, neitherOrBoth);
894
+ if (heterogeneous !== void 0) throw new NativeParseError(FORMAT$14, 1, neitherOrBoth);
895
895
  const [relation] = [...byName.keys()].filter((name) => name.startsWith("relations/"));
896
- if (relation !== void 0 || byName.has("triplet-type-list")) throw new NativeParseError(FORMAT$11, 1, `the dataset directory ${root} has raw/num-node-list.csv.gz, so it has one node type, but the recipe selects ${relation === void 0 ? "raw/triplet-type-list.csv.gz" : `raw/${relation}.csv.gz`} as well`);
896
+ if (relation !== void 0 || byName.has("triplet-type-list")) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has raw/num-node-list.csv.gz, so it has one node type, but the recipe selects ${relation === void 0 ? "raw/triplet-type-list.csv.gz" : `raw/${relation}.csv.gz`} as well`);
897
897
  const count = index(onlyLine(homogeneous), homogeneous, 1);
898
898
  const name = (node) => String(node);
899
899
  readEdges(required$1(byName, "edge", root), required$1(byName, "num-edge-list", root), {
@@ -909,17 +909,17 @@ function parseOgb(files) {
909
909
  edges
910
910
  };
911
911
  }
912
- if (heterogeneous === void 0) throw new NativeParseError(FORMAT$11, 1, neitherOrBoth);
913
- for (const stray of ["edge", "num-edge-list"]) if (byName.has(stray)) throw new NativeParseError(FORMAT$11, 1, `the dataset directory ${root} has raw/num-node-dict.csv.gz, so its edges are in raw/relations/, but the recipe selects raw/${stray}.csv.gz as well`);
912
+ if (heterogeneous === void 0) throw new NativeParseError(FORMAT$14, 1, neitherOrBoth);
913
+ for (const stray of ["edge", "num-edge-list"]) if (byName.has(stray)) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has raw/num-node-dict.csv.gz, so its edges are in raw/relations/, but the recipe selects raw/${stray}.csv.gz as well`);
914
914
  const header = rows(heterogeneous);
915
915
  const [types, counts] = header;
916
- if (header.length !== 2 || types === void 0 || counts === void 0) throw new NativeParseError(FORMAT$11, header[2]?.line ?? 1, `${heterogeneous.path} must hold a header of node types and one row of their counts, found ${String(header.length)} lines`);
916
+ if (header.length !== 2 || types === void 0 || counts === void 0) throw new NativeParseError(FORMAT$14, header[2]?.line ?? 1, `${heterogeneous.path} must hold a header of node types and one row of their counts, found ${String(header.length)} lines`);
917
917
  const typeNames = types.text.split(",").map((type) => type.trim());
918
918
  const typeCounts = counts.text.split(",");
919
- if (typeNames.length !== typeCounts.length || typeNames.some((type) => type === "") || new Set(typeNames).size !== typeNames.length) throw new NativeParseError(FORMAT$11, counts.line, `${heterogeneous.path} must name each node type once, over one count per type`);
919
+ if (typeNames.length !== typeCounts.length || typeNames.some((type) => type === "") || new Set(typeNames).size !== typeNames.length) throw new NativeParseError(FORMAT$14, counts.line, `${heterogeneous.path} must name each node type once, over one count per type`);
920
920
  const nodeTypes = typeNames.map((type, position) => {
921
921
  const count = typeCounts[position];
922
- if (count === void 0) throw new NativeParseError(FORMAT$11, counts.line, `${heterogeneous.path} gives node type ${type} no count`);
922
+ if (count === void 0) throw new NativeParseError(FORMAT$14, counts.line, `${heterogeneous.path} gives node type ${type} no count`);
923
923
  return {
924
924
  type,
925
925
  count: index(count, heterogeneous, counts.line)
@@ -932,10 +932,10 @@ function parseOgb(files) {
932
932
  for (const { text, line } of rows(triplets)) {
933
933
  const parts = text.split(",").map((part) => part.trim());
934
934
  const [head, relation, tail] = parts;
935
- if (parts.length !== TRIPLET_PARTS || head === void 0 || relation === void 0 || tail === void 0 || relation === "") throw new NativeParseError(FORMAT$11, line, `${triplets.path}: expected a head type, a relation and a tail type, found ${JSON.stringify(text)}`);
935
+ if (parts.length !== TRIPLET_PARTS || head === void 0 || relation === void 0 || tail === void 0 || relation === "") throw new NativeParseError(FORMAT$14, line, `${triplets.path}: expected a head type, a relation and a tail type, found ${JSON.stringify(text)}`);
936
936
  const sources = nodeCounts.get(head);
937
937
  const targets = nodeCounts.get(tail);
938
- if (sources === void 0 || targets === void 0) throw new NativeParseError(FORMAT$11, line, `${triplets.path}: ${text} names a node type that ${heterogeneous.path} does not count`);
938
+ if (sources === void 0 || targets === void 0) throw new NativeParseError(FORMAT$14, line, `${triplets.path}: ${text} names a node type that ${heterogeneous.path} does not count`);
939
939
  relations.push({
940
940
  head,
941
941
  relation,
@@ -948,7 +948,7 @@ function parseOgb(files) {
948
948
  const listed = new Set(relations.map(({ head, relation, tail }) => `relations/${head}___${relation}___${tail}`));
949
949
  for (const name of byName.keys()) {
950
950
  const directory = name.slice(0, name.lastIndexOf("/"));
951
- if (name.startsWith("relations/") && !listed.has(directory)) throw new NativeParseError(FORMAT$11, 1, `raw/${directory}/ is not one of the relations ${triplets.path} lists`);
951
+ if (name.startsWith("relations/") && !listed.has(directory)) throw new NativeParseError(FORMAT$14, 1, `raw/${directory}/ is not one of the relations ${triplets.path} lists`);
952
952
  }
953
953
  for (const { head, relation, tail, sources, targets } of relations) {
954
954
  const directory = `relations/${head}___${relation}___${tail}`;
@@ -1201,69 +1201,71 @@ var RecordScanner = class {
1201
1201
  }
1202
1202
  };
1203
1203
  /**
1204
- * Splits text into records and fields by RFC 4180 as the text arrives in pieces, one piece's records at a time, so neither the whole file nor every cell is held at once: a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR. A piece may end anywhere, even between the two quotes of a doubled quote or between a CR and its LF.
1205
- * @param pieces - The file text in order, after the lines the recipe skips.
1206
- * @param delimiter - The field delimiter.
1207
- * @param firstLine - The 1-based line of the file the text starts on.
1208
- * @returns The records in order, blank lines included as a record of one empty field.
1209
- * @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
1210
- */
1211
- function* records$1(pieces, delimiter, firstLine) {
1212
- const scanner = new RecordScanner(delimiter.charCodeAt(0), firstLine);
1213
- let carry = "";
1214
- for (const piece of pieces) {
1215
- const scanned = scanner.scan(carry + piece, false);
1216
- carry = scanned.carry;
1217
- yield* scanned.completed;
1204
+ * Reads a table file's records as its text arrives in pieces, so neither the whole file nor every cell is held at once: a leading byte order mark is removed, the physical lines the recipe skips are dropped, and the rest is split into records and fields by RFC 4180 (a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR). Blank lines are not records. A piece may end anywhere, even between the two quotes of a doubled quote or between a CR and its LF.
1205
+ */
1206
+ var RecordReader = class {
1207
+ options;
1208
+ scanner;
1209
+ carry = "";
1210
+ first = true;
1211
+ toSkip;
1212
+ constructor(options) {
1213
+ this.options = options;
1214
+ this.scanner = new RecordScanner(DELIMITERS[options.delimiter].charCodeAt(0), options.skipLines + 1);
1215
+ this.toSkip = options.skipLines;
1218
1216
  }
1219
- yield* scanner.scan(carry, true).completed;
1220
- const last = scanner.finish();
1221
- if (last !== void 0) yield last;
1222
- }
1217
+ /**
1218
+ * Reads the next piece of the file's text.
1219
+ * @param piece - The text, which follows the pieces read before it.
1220
+ * @returns The records the piece completes, in order.
1221
+ * @throws NativeParseError when a quote opens inside an unquoted field.
1222
+ */
1223
+ push(piece) {
1224
+ let rest = this.first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
1225
+ this.first = false;
1226
+ while (this.toSkip > 0) {
1227
+ const next = rest.indexOf("\n");
1228
+ if (next === -1) return [];
1229
+ rest = rest.slice(next + 1);
1230
+ this.toSkip -= 1;
1231
+ }
1232
+ if (rest === "") return [];
1233
+ const scanned = this.scanner.scan(this.carry + rest, false);
1234
+ this.carry = scanned.carry;
1235
+ return scanned.completed.filter(isNotBlank);
1236
+ }
1237
+ /**
1238
+ * Ends the file.
1239
+ * @returns The records left when the text ends, in order.
1240
+ * @throws NativeParseError when a quoted field is never closed, a quote opens inside an unquoted field, or the file has fewer lines than the recipe skips.
1241
+ */
1242
+ end() {
1243
+ if (this.toSkip > 0) throw new NativeParseError("csv-table", this.options.skipLines - this.toSkip + 1, `the file has fewer than ${String(this.options.skipLines)} lines to skip before the header`);
1244
+ const completed = this.scanner.scan(this.carry, true).completed;
1245
+ this.carry = "";
1246
+ const last = this.scanner.finish();
1247
+ return [...completed, ...last === void 0 ? [] : [last]].filter(isNotBlank);
1248
+ }
1249
+ };
1223
1250
  /**
1224
- * Splits comma-separated text into its records' fields by RFC 4180, as `records` does, keeping every record (blank lines included) at its position: the grid a workbook sheet's CSV makes, for a parser that reads a layout of blocks and not one table.
1251
+ * Splits comma-separated text into its records' fields by RFC 4180, as the record reader does, keeping every record (blank lines included) at its position: the grid a workbook sheet's CSV makes, for a parser that reads a layout of blocks and not one table.
1225
1252
  * @param text - The text.
1226
1253
  * @returns Each record's fields, in order.
1227
1254
  * @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
1228
1255
  */
1229
1256
  function csvGrid(text) {
1230
- return [...records$1([text], ",", 1)].map((record) => record.fields);
1231
- }
1232
- /**
1233
- * Drops the first physical lines of a file given in pieces.
1234
- * @param pieces - The file text in order.
1235
- * @param count - How many physical lines to drop.
1236
- * @returns The rest of the text, in pieces.
1237
- * @throws NativeParseError when the file has fewer lines than that.
1238
- */
1239
- function* afterLines(pieces, count) {
1240
- let remaining = count;
1241
- for (const piece of pieces) {
1242
- let rest = piece;
1243
- while (remaining > 0) {
1244
- const next = rest.indexOf("\n");
1245
- if (next === -1) {
1246
- rest = "";
1247
- break;
1248
- }
1249
- rest = rest.slice(next + 1);
1250
- remaining -= 1;
1251
- }
1252
- if (remaining === 0 && rest !== "") yield rest;
1253
- }
1254
- if (remaining > 0) throw new NativeParseError("csv-table", count - remaining + 1, `the file has fewer than ${String(count)} lines to skip before the header`);
1257
+ const scanner = new RecordScanner(DELIMITERS.comma.charCodeAt(0), 1);
1258
+ const { completed } = scanner.scan(text, true);
1259
+ const last = scanner.finish();
1260
+ return [...completed, ...last === void 0 ? [] : [last]].map((record) => record.fields);
1255
1261
  }
1256
1262
  /**
1257
- * The text of a file as its pieces, with a leading byte order mark removed.
1258
- * @param text - The file text, or its pieces in order.
1259
- * @returns The pieces.
1263
+ * Whether a record holds anything: a blank line is a record of one empty field.
1264
+ * @param record - The record.
1265
+ * @returns False for a blank line.
1260
1266
  */
1261
- function* withoutByteOrderMark(text) {
1262
- let first = true;
1263
- for (const piece of typeof text === "string" ? [text] : text) {
1264
- yield first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
1265
- first = false;
1266
- }
1267
+ function isNotBlank(record) {
1268
+ return record.fields.length > 1 || record.fields[0]?.trim() !== "";
1267
1269
  }
1268
1270
  /**
1269
1271
  * Reads a numeric cell.
@@ -1280,27 +1282,81 @@ function numeric$1(cell, factor, column, line) {
1280
1282
  return Number(cell) * factor;
1281
1283
  }
1282
1284
  /**
1283
- * The records of a table file after the lines the recipe skips and without its blank lines: the header first, then the rows, one at a time.
1284
- * @param text - The file text, or its pieces in order, which may begin with a byte order mark.
1285
- * @param options - The recipe's table parser, of which the delimiter and the lines to skip are read.
1286
- * @returns Each record.
1287
- * @throws NativeParseError when a quote is never closed or opens inside an unquoted field, or the file has fewer lines than the recipe skips.
1285
+ * Reads a delimited table into rows as its text arrives in pieces, holding only the record in progress between pieces, so a caller that hands each row on as it is returned holds no more of the table than that. The rows are exactly those `parseCsvTable` gives, in order, with the same errors on the same lines however the text is cut.
1288
1286
  */
1289
- function* tableRecords(text, options) {
1290
- const body = afterLines(withoutByteOrderMark(text), options.skipLines);
1291
- for (const record of records$1(body, DELIMITERS[options.delimiter], options.skipLines + 1)) if (record.fields.length > 1 || record.fields[0]?.trim() !== "") yield record;
1292
- }
1287
+ var TableReader = class {
1288
+ options;
1289
+ records;
1290
+ layout;
1291
+ rows = 0;
1292
+ missing;
1293
+ constructor(options) {
1294
+ this.options = options;
1295
+ this.records = new RecordReader(options);
1296
+ this.missing = new Set(options.nullValues ?? []);
1297
+ }
1298
+ /**
1299
+ * Reads the next piece of the file's text.
1300
+ * @param piece - The text, which follows the pieces read before it.
1301
+ * @returns The rows the piece completes, in file order.
1302
+ * @throws NativeParseError as `parseCsvTable` does, naming the line of the problem.
1303
+ */
1304
+ push(piece) {
1305
+ return this.rowsOf(this.records.push(piece));
1306
+ }
1307
+ /**
1308
+ * Ends the file.
1309
+ * @returns The rows left when the text ends, in file order.
1310
+ * @throws NativeParseError as `parseCsvTable` does, including for a file with no header row.
1311
+ */
1312
+ end() {
1313
+ const rows = this.rowsOf(this.records.end());
1314
+ if (this.layout === void 0) throw new NativeParseError("csv-table", this.options.skipLines + 1, "the file has no header row");
1315
+ return rows;
1316
+ }
1317
+ /**
1318
+ * Builds the rows of records, the first record of the file being its header.
1319
+ * @param records - Records in file order.
1320
+ * @returns The rows among them.
1321
+ */
1322
+ rowsOf(records) {
1323
+ const rows = [];
1324
+ for (const record of records) if (this.layout === void 0) this.layout = layoutOf(record, this.options);
1325
+ else rows.push(this.rowOf(record, this.layout));
1326
+ return rows;
1327
+ }
1328
+ /**
1329
+ * Builds one row.
1330
+ * @param record - A record after the header.
1331
+ * @param layout - The header's layout.
1332
+ * @returns The row.
1333
+ * @throws NativeParseError when the record's width differs from the header's, its id is empty or a numeric cell is not a number.
1334
+ */
1335
+ rowOf(record, layout) {
1336
+ const { idColumn } = this.options;
1337
+ if (record.fields.length !== layout.width) throw new NativeParseError("csv-table", record.line, `the row has ${String(record.fields.length)} fields, the header has ${String(layout.width)}`);
1338
+ this.rows += 1;
1339
+ const id = idColumn === null ? String(this.rows) : (record.fields[layout.idPosition] ?? "").trim();
1340
+ if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
1341
+ const cells = layout.kept.map(({ name, position, factor }) => {
1342
+ const trimmed = (record.fields[position] ?? "").trim();
1343
+ const cell = this.missing.has(trimmed) ? "" : trimmed;
1344
+ return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
1345
+ });
1346
+ return {
1347
+ ...Object.fromEntries(cells),
1348
+ id
1349
+ };
1350
+ }
1351
+ };
1293
1352
  /**
1294
- * Parses a delimited table into rows (a cell whose text is one of the recipe's `nullValues` is empty, so null): the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
1295
- * @param text - The file text, or its pieces in order.
1353
+ * Reads a table's header into the layout its rows are built by, checking the recipe against it.
1354
+ * @param header - The header record.
1296
1355
  * @param options - The recipe's table parser.
1297
- * @returns The rows, in file order.
1298
- * @throws NativeParseError naming the line of the first problem, including a listed or numeric column the header lacks or a numeric column that is not listed.
1356
+ * @returns The layout.
1357
+ * @throws NativeParseError when two header cells get the same name, the recipe names a column the header lacks, lists the id column among the columns, or declares numeric a column it does not keep.
1299
1358
  */
1300
- function parseCsvTable(text, options) {
1301
- const table = tableRecords(text, options);
1302
- const { value: header } = table.next();
1303
- if (header === void 0) throw new NativeParseError("csv-table", options.skipLines + 1, "the file has no header row");
1359
+ function layoutOf(header, options) {
1304
1360
  const names = columnNames(header.fields, options.idColumn, header.line);
1305
1361
  const { idColumn, numericColumns, columns } = options;
1306
1362
  const wanted = new Set(columns ?? names.filter((name) => name !== idColumn));
@@ -1312,28 +1368,28 @@ function parseCsvTable(text, options) {
1312
1368
  if (idColumn !== null && wanted.has(idColumn)) throw new NativeParseError("csv-table", header.line, `the id column ${idColumn} is the row's id, so it is not listed among the columns`);
1313
1369
  for (const { column } of numericColumns) if (!wanted.has(column)) throw new NativeParseError("csv-table", header.line, `the column ${column} is declared numeric but is not among the columns kept`);
1314
1370
  const factors = new Map(numericColumns.map((entry) => [entry.column, entry.factor]));
1315
- const missing = new Set(options.nullValues ?? []);
1316
- const idPosition = idColumn === null ? -1 : names.indexOf(idColumn);
1317
- const kept = names.flatMap((name, position) => name !== idColumn && wanted.has(name) ? [{
1318
- name,
1319
- position,
1320
- factor: factors.get(name)
1321
- }] : []);
1371
+ return {
1372
+ width: names.length,
1373
+ idPosition: idColumn === null ? -1 : names.indexOf(idColumn),
1374
+ kept: names.flatMap((name, position) => name !== idColumn && wanted.has(name) ? [{
1375
+ name,
1376
+ position,
1377
+ factor: factors.get(name)
1378
+ }] : [])
1379
+ };
1380
+ }
1381
+ /**
1382
+ * Parses a delimited table into rows (a cell whose text is one of the recipe's `nullValues` is empty, so null): the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
1383
+ * @param text - The file text, or its pieces in order.
1384
+ * @param options - The recipe's table parser.
1385
+ * @returns The rows, in file order.
1386
+ * @throws NativeParseError naming the line of the first problem, including a listed or numeric column the header lacks or a numeric column that is not listed.
1387
+ */
1388
+ function parseCsvTable(text, options) {
1389
+ const reader = new TableReader(options);
1322
1390
  const nodes = [];
1323
- for (const record of table) {
1324
- if (record.fields.length !== names.length) throw new NativeParseError("csv-table", record.line, `the row has ${String(record.fields.length)} fields, the header has ${String(names.length)}`);
1325
- const id = idColumn === null ? String(nodes.length + 1) : (record.fields[idPosition] ?? "").trim();
1326
- if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
1327
- const cells = kept.map(({ name, position, factor }) => {
1328
- const trimmed = (record.fields[position] ?? "").trim();
1329
- const cell = missing.has(trimmed) ? "" : trimmed;
1330
- return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
1331
- });
1332
- nodes.push({
1333
- ...Object.fromEntries(cells),
1334
- id
1335
- });
1336
- }
1391
+ for (const piece of typeof text === "string" ? [text] : text) for (const row of reader.push(piece)) nodes.push(row);
1392
+ for (const row of reader.end()) nodes.push(row);
1337
1393
  return { nodes };
1338
1394
  }
1339
1395
  //#endregion
@@ -1438,7 +1494,7 @@ function parseGeojson(text, options) {
1438
1494
  }
1439
1495
  //#endregion
1440
1496
  //#region src/parsers/hhcrsp.ts
1441
- const FORMAT$10 = "hhcrsp";
1497
+ const FORMAT$13 = "hhcrsp";
1442
1498
  /** The labels of the sheet's blocks, each the top-left cell of a block whose header runs to its right and whose rows run below it. */
1443
1499
  const LABEL = {
1444
1500
  patients: "Patients",
@@ -1450,9 +1506,9 @@ const LABEL = {
1450
1506
  };
1451
1507
  /** A service's header, `Service <k>`, a patient's row label and a caregiver's. */
1452
1508
  const SERVICE = /^Service (\d+)$/;
1453
- const NUMBER$7 = /^-?\d+(?:\.\d+)?$/;
1509
+ const NUMBER$8 = /^-?\d+(?:\.\d+)?$/;
1454
1510
  /** How many required services a patient may have: the instances ask for one, or two that must be synchronised, and the parser has a duration column for each. */
1455
- const MOST_SERVICES = 2;
1511
+ const MOST_SERVICES$1 = 2;
1456
1512
  /**
1457
1513
  * Finds a block by its label: the one cell of the sheet that holds it, its header the cells to its right up to the first empty one, and its rows those below it up to the first whose label and cells are all empty. Past its header a block has an empty column, so the cell there must be empty in every row and a value the header does not label is refused rather than left unread; `Pat / Ser` instead keeps its count there, with the next block starting right after it, and its count is checked on its own.
1458
1514
  * @param grid - The sheet's cells by row.
@@ -1467,7 +1523,7 @@ function blockOf(grid, label, pastHeader = "empty") {
1467
1523
  column
1468
1524
  }] : []));
1469
1525
  const [only] = found;
1470
- if (only === void 0 || found.length !== 1) throw new NativeParseError(FORMAT$10, 1, `expected one block labelled ${JSON.stringify(label)}, found ${String(found.length)}`);
1526
+ if (only === void 0 || found.length !== 1) throw new NativeParseError(FORMAT$13, 1, `expected one block labelled ${JSON.stringify(label)}, found ${String(found.length)}`);
1471
1527
  const headerRow = grid[only.line] ?? [];
1472
1528
  const end = headerRow.findIndex((cell, column) => column > only.column && cell.trim() === "");
1473
1529
  const header = headerRow.slice(only.column + 1, end === -1 ? headerRow.length : end).map((cell) => cell.trim());
@@ -1476,7 +1532,7 @@ function blockOf(grid, label, pastHeader = "empty") {
1476
1532
  const cells = (grid[line] ?? []).slice(only.column, only.column + 1 + header.length).map((cell) => cell.trim());
1477
1533
  if (cells.every((cell) => cell === "")) break;
1478
1534
  const past = (grid[line] ?? [])[only.column + 1 + header.length]?.trim();
1479
- if (pastHeader === "empty" && past !== void 0 && past !== "") throw new NativeParseError(FORMAT$10, line + 1, `the ${label} block's row holds ${JSON.stringify(past)} past its header ${JSON.stringify(header)}, which does not label it`);
1535
+ if (pastHeader === "empty" && past !== void 0 && past !== "") throw new NativeParseError(FORMAT$13, line + 1, `the ${label} block's row holds ${JSON.stringify(past)} past its header ${JSON.stringify(header)}, which does not label it`);
1480
1536
  rows.push({
1481
1537
  line: line + 1,
1482
1538
  label: cells[0] ?? "",
@@ -1499,7 +1555,7 @@ function blockOf(grid, label, pastHeader = "empty") {
1499
1555
  * @throws NativeParseError when the cell is not a decimal number.
1500
1556
  */
1501
1557
  function numberOf$4(cell, line, what) {
1502
- if (cell === void 0 || !NUMBER$7.test(cell)) throw new NativeParseError(FORMAT$10, line, `expected ${what}, found ${JSON.stringify(cell)}`);
1558
+ if (cell === void 0 || !NUMBER$8.test(cell)) throw new NativeParseError(FORMAT$13, line, `expected ${what}, found ${JSON.stringify(cell)}`);
1503
1559
  return Number(cell);
1504
1560
  }
1505
1561
  /**
@@ -1511,7 +1567,7 @@ function numberOf$4(cell, line, what) {
1511
1567
  */
1512
1568
  function checkNumbered(block, noun, label) {
1513
1569
  block.rows.forEach((row, index) => {
1514
- if (row.label !== `${noun} ${String(index + 1)}`) throw new NativeParseError(FORMAT$10, row.line, `the ${label} block's row ${String(index + 1)} is labelled ${JSON.stringify(row.label)}, not ${noun} ${String(index + 1)}`);
1570
+ if (row.label !== `${noun} ${String(index + 1)}`) throw new NativeParseError(FORMAT$13, row.line, `the ${label} block's row ${String(index + 1)} is labelled ${JSON.stringify(row.label)}, not ${noun} ${String(index + 1)}`);
1515
1571
  });
1516
1572
  }
1517
1573
  /**
@@ -1523,7 +1579,7 @@ function checkNumbered(block, noun, label) {
1523
1579
  */
1524
1580
  function serviceCount(block, label) {
1525
1581
  block.header.forEach((cell, index) => {
1526
- if (SERVICE.exec(cell)?.[1] !== String(index + 1)) throw new NativeParseError(FORMAT$10, block.line, `the ${label} block's header cell ${String(index + 1)} is ${JSON.stringify(cell)}, not Service ${String(index + 1)}`);
1582
+ if (SERVICE.exec(cell)?.[1] !== String(index + 1)) throw new NativeParseError(FORMAT$13, block.line, `the ${label} block's header cell ${String(index + 1)} is ${JSON.stringify(cell)}, not Service ${String(index + 1)}`);
1527
1583
  });
1528
1584
  return block.header.length;
1529
1585
  }
@@ -1542,7 +1598,7 @@ function parseHhcrsp(text) {
1542
1598
  const caregiverWindows = blockOf(grid, LABEL.caregiverWindows);
1543
1599
  const windows = blockOf(grid, LABEL.patientWindows);
1544
1600
  const serviceKinds = serviceCount(requests, LABEL.requests);
1545
- if (serviceCount(times, LABEL.times) !== serviceKinds || serviceCount(skills, LABEL.skills) !== serviceKinds) throw new NativeParseError(FORMAT$10, times.line, `the ${LABEL.requests}, ${LABEL.times} and ${LABEL.skills} blocks name different numbers of services`);
1601
+ if (serviceCount(times, LABEL.times) !== serviceKinds || serviceCount(skills, LABEL.skills) !== serviceKinds) throw new NativeParseError(FORMAT$13, times.line, `the ${LABEL.requests}, ${LABEL.times} and ${LABEL.skills} blocks name different numbers of services`);
1546
1602
  checkNumbered(requests, "Patient", LABEL.requests);
1547
1603
  checkNumbered(windows, "Patient", LABEL.patientWindows);
1548
1604
  checkNumbered(skills, "Caregiver", LABEL.skills);
@@ -1557,11 +1613,11 @@ function parseHhcrsp(text) {
1557
1613
  "",
1558
1614
  "center",
1559
1615
  "centre"
1560
- ].includes(centreTimes.label.toLowerCase()) || windows.rows.length !== patients || patientTimes.length !== patients || skills.rows.length !== caregiverWindows.rows.length) throw new NativeParseError(FORMAT$10, requests.line, `the blocks disagree: ${String(patients)} patients requesting services, ${String(patientTimes.length)} with durations after the centre's row, ${String(windows.rows.length)} with time windows, and ${String(skills.rows.length)} caregivers with skills against ${String(caregiverWindows.rows.length)} with windows`);
1561
- if (positions.header.join(",") !== "X,Y" || positions.rows.length !== patients + 2 || positions.rows.some((row, index) => row.label !== String(index))) throw new NativeParseError(FORMAT$10, positions.line, `the ${LABEL.patients} block must give X and Y for nodes 0 to ${String(patients + 1)}, the centre, ${String(patients)} patients and the centre again`);
1616
+ ].includes(centreTimes.label.toLowerCase()) || windows.rows.length !== patients || patientTimes.length !== patients || skills.rows.length !== caregiverWindows.rows.length) throw new NativeParseError(FORMAT$13, requests.line, `the blocks disagree: ${String(patients)} patients requesting services, ${String(patientTimes.length)} with durations after the centre's row, ${String(windows.rows.length)} with time windows, and ${String(skills.rows.length)} caregivers with skills against ${String(caregiverWindows.rows.length)} with windows`);
1617
+ if (positions.header.join(",") !== "X,Y" || positions.rows.length !== patients + 2 || positions.rows.some((row, index) => row.label !== String(index))) throw new NativeParseError(FORMAT$13, positions.line, `the ${LABEL.patients} block must give X and Y for nodes 0 to ${String(patients + 1)}, the centre, ${String(patients)} patients and the centre again`);
1562
1618
  const position = (index) => {
1563
1619
  const row = positions.rows[index];
1564
- if (row === void 0) throw new NativeParseError(FORMAT$10, positions.line, `the ${LABEL.patients} block has no node ${String(index)}`);
1620
+ if (row === void 0) throw new NativeParseError(FORMAT$13, positions.line, `the ${LABEL.patients} block has no node ${String(index)}`);
1565
1621
  return {
1566
1622
  x: numberOf$4(row.cells[0], row.line, "an X"),
1567
1623
  y: numberOf$4(row.cells[1], row.line, "a Y"),
@@ -1570,7 +1626,7 @@ function parseHhcrsp(text) {
1570
1626
  };
1571
1627
  const { x, y } = position(0);
1572
1628
  const closing = position(patients + 1);
1573
- if (closing.x !== x || closing.y !== y) throw new NativeParseError(FORMAT$10, closing.line, `the last node of the ${LABEL.patients} block is not at the centre's position, so it is not the route's end`);
1629
+ if (closing.x !== x || closing.y !== y) throw new NativeParseError(FORMAT$13, closing.line, `the last node of the ${LABEL.patients} block is not at the centre's position, so it is not the route's end`);
1574
1630
  const windowColumn = (name) => {
1575
1631
  const index = windows.header.indexOf(name);
1576
1632
  return index === -1 ? void 0 : index;
@@ -1581,7 +1637,7 @@ function parseHhcrsp(text) {
1581
1637
  earliest2: windowColumn("ETW 2"),
1582
1638
  latest2: windowColumn("LTW 2")
1583
1639
  };
1584
- if (columns.earliest1 === void 0 || columns.latest1 === void 0 || columns.earliest2 === void 0 !== (columns.latest2 === void 0) || windows.header.length !== (columns.earliest2 === void 0 ? 2 : 4)) throw new NativeParseError(FORMAT$10, windows.line, `the ${LABEL.patientWindows} header ${JSON.stringify(windows.header)} is not ETW 1 and LTW 1, with or without ETW 2 and LTW 2`);
1640
+ if (columns.earliest1 === void 0 || columns.latest1 === void 0 || columns.earliest2 === void 0 !== (columns.latest2 === void 0) || windows.header.length !== (columns.earliest2 === void 0 ? 2 : 4)) throw new NativeParseError(FORMAT$13, windows.line, `the ${LABEL.patientWindows} header ${JSON.stringify(windows.header)} is not ETW 1 and LTW 1, with or without ETW 2 and LTW 2`);
1585
1641
  const nodes = [{
1586
1642
  id: "0",
1587
1643
  role: "centre",
@@ -1597,15 +1653,15 @@ function parseHhcrsp(text) {
1597
1653
  }];
1598
1654
  requests.rows.forEach((request, index) => {
1599
1655
  const required = request.cells.flatMap((cell, service) => {
1600
- if (cell !== "0" && cell !== "1" && cell !== "") throw new NativeParseError(FORMAT$10, request.line, `${request.label} has ${JSON.stringify(cell)} for Service ${String(service + 1)}, not 0, 1 or empty`);
1656
+ if (cell !== "0" && cell !== "1" && cell !== "") throw new NativeParseError(FORMAT$13, request.line, `${request.label} has ${JSON.stringify(cell)} for Service ${String(service + 1)}, not 0, 1 or empty`);
1601
1657
  return cell === "1" ? [service] : [];
1602
1658
  });
1603
1659
  const stated = (grid[request.line - 1] ?? [])[requests.column + serviceKinds + 1]?.trim();
1604
- if (stated !== String(required.length)) throw new NativeParseError(FORMAT$10, request.line, `${request.label} requires ${String(required.length)} services by its marks, and the count beside them is ${JSON.stringify(stated)}`);
1605
- if (required.length === 0 || required.length > MOST_SERVICES) throw new NativeParseError(FORMAT$10, request.line, `${request.label} requires ${String(required.length)} services, and a patient requires one or two`);
1660
+ if (stated !== String(required.length)) throw new NativeParseError(FORMAT$13, request.line, `${request.label} requires ${String(required.length)} services by its marks, and the count beside them is ${JSON.stringify(stated)}`);
1661
+ if (required.length === 0 || required.length > MOST_SERVICES$1) throw new NativeParseError(FORMAT$13, request.line, `${request.label} requires ${String(required.length)} services, and a patient requires one or two`);
1606
1662
  const timeRow = patientTimes[index];
1607
1663
  const windowRow = windows.rows[index];
1608
- if (timeRow === void 0 || windowRow === void 0) throw new NativeParseError(FORMAT$10, request.line, `${request.label} has no durations or time windows`);
1664
+ if (timeRow === void 0 || windowRow === void 0) throw new NativeParseError(FORMAT$13, request.line, `${request.label} has no durations or time windows`);
1609
1665
  const duration = (which) => {
1610
1666
  const service = required[which];
1611
1667
  return service === void 0 ? null : numberOf$4(timeRow.cells[service], timeRow.line, `the duration of Service ${String(service + 1)}`);
@@ -1636,6 +1692,433 @@ function parseHhcrsp(text) {
1636
1692
  };
1637
1693
  }
1638
1694
  //#endregion
1695
+ //#region src/parsers/mankowska.ts
1696
+ const FORMAT$12 = "mankowska";
1697
+ /** The sections of the file, each a keyword on a line of its own followed by its numbers. */
1698
+ const SECTIONS$1 = [
1699
+ "nbNodes",
1700
+ "nbVehi",
1701
+ "nbServi",
1702
+ "r",
1703
+ "DS",
1704
+ "a",
1705
+ "x",
1706
+ "y",
1707
+ "d",
1708
+ "p",
1709
+ "mind",
1710
+ "maxd",
1711
+ "e",
1712
+ "l"
1713
+ ];
1714
+ /** How many patients a double service has, and so the most required services a patient can have. */
1715
+ const MOST_SERVICES = 2;
1716
+ /** The depot copies: the first and the last node. */
1717
+ const DEPOTS = 2;
1718
+ const NUMBER$7 = /^-?\d+(?:\.\d+)?$/;
1719
+ const KEYWORD = /^[A-Za-z]+$/;
1720
+ /**
1721
+ * Splits the file into its sections: a line holding one word starts a section, and every number on the lines after it, up to the next such line, is the section's.
1722
+ * @param text - The file text.
1723
+ * @returns Each section by its keyword.
1724
+ * @throws NativeParseError for a keyword the format does not have, one given twice, a value that is not a number, or a number before the first keyword.
1725
+ */
1726
+ function sectionsOf$1(text) {
1727
+ const sections = /* @__PURE__ */ new Map();
1728
+ let current;
1729
+ for (const [index, raw] of lines(text).entries()) {
1730
+ const tokens = raw.trim().split(/\s+/).filter((token) => token !== "");
1731
+ const [only] = tokens;
1732
+ if (tokens.length === 1 && only !== void 0 && KEYWORD.test(only)) {
1733
+ const name = SECTIONS$1.find((section) => section === only);
1734
+ if (name === void 0 || sections.has(name)) throw new NativeParseError(FORMAT$12, index + 1, name === void 0 ? `the section ${JSON.stringify(only)} is not one of ${SECTIONS$1.join(", ")}` : `the section ${name} is given twice`);
1735
+ current = [];
1736
+ sections.set(name, {
1737
+ line: index + 1,
1738
+ values: current
1739
+ });
1740
+ continue;
1741
+ }
1742
+ for (const token of tokens) {
1743
+ if (current === void 0 || !NUMBER$7.test(token)) throw new NativeParseError(FORMAT$12, index + 1, current === void 0 ? "a number comes before the first section" : `expected a number, found ${JSON.stringify(token)}`);
1744
+ current.push(Number(token));
1745
+ }
1746
+ }
1747
+ return sections;
1748
+ }
1749
+ /**
1750
+ * Reads a section that must hold a given count of numbers.
1751
+ * @param sections - The sections.
1752
+ * @param name - The section.
1753
+ * @param count - How many numbers it must hold.
1754
+ * @returns Its numbers.
1755
+ * @throws NativeParseError when the section is missing or holds another count.
1756
+ */
1757
+ function sized(sections, name, count) {
1758
+ const section = sections.get(name);
1759
+ if (section === void 0) throw new NativeParseError(FORMAT$12, 1, `the file has no ${name} section`);
1760
+ if (section.values.length !== count) throw new NativeParseError(FORMAT$12, section.line, `the ${name} section holds ${String(section.values.length)} numbers where ${String(count)} are due`);
1761
+ return section.values;
1762
+ }
1763
+ /**
1764
+ * Reads a count section (`nbNodes`, `nbVehi`, `nbServi`): one whole number of at least a least value.
1765
+ * @param sections - The sections.
1766
+ * @param name - The section.
1767
+ * @param least - The smallest count that makes an instance.
1768
+ * @returns The count.
1769
+ * @throws NativeParseError when the section is missing, holds more than one number, or the count is not a whole number of at least `least`.
1770
+ */
1771
+ function countOf(sections, name, least) {
1772
+ const [count] = sized(sections, name, 1);
1773
+ if (count === void 0 || !Number.isInteger(count) || count < least) throw new NativeParseError(FORMAT$12, sections.get(name)?.line ?? 1, `${name} must be a whole number of at least ${String(least)}, found ${String(count)}`);
1774
+ return count;
1775
+ }
1776
+ /**
1777
+ * Reads one cell of a row-major matrix section.
1778
+ * @param values - The section's numbers.
1779
+ * @param columns - The matrix's width.
1780
+ * @param row - The row.
1781
+ * @param column - The column.
1782
+ * @returns The cell.
1783
+ * @throws NativeParseError when the cell is outside the section, which `sized` has already ruled out.
1784
+ */
1785
+ function cell(values, columns, row, column) {
1786
+ const value = values[row * columns + column];
1787
+ if (value === void 0) throw new NativeParseError(FORMAT$12, 1, `cell ${String(row)}, ${String(column)} is outside its section`);
1788
+ return value;
1789
+ }
1790
+ /**
1791
+ * Parses a home health care routing and scheduling instance in the keyword text format of Mankowska, Meisel and Bierwirth (2014), as the OVIG generator writes it: the counts of nodes (`nbNodes`, the patients and two copies of the depot, the first and the last), vehicles (`nbVehi`, the caregivers) and services (`nbServi`); the services each node requires (`r`, a node by service matrix of zeros and ones), the double service nodes (`DS`, node numbers from one), the services each caregiver can give (`a`), the positions (`x`, `y`), the travel matrix (`d`), the processing time of each service at each node by each caregiver (`p`, a row per node and caregiver), the least and most time between a double service's two services (`mind`, `maxd`), and the time windows (`e`, `l`). A node's id is its number from zero. The depot copies are marked as requiring every service, which is the format's convention and not a visit, so they carry no services or processing times. A patient's required services are their numbers from one, joined by spaces, and its first and second processing time are those of its required services in that order. The travel matrix and the caregivers' qualifications are not node fields and are not kept; the processing times must not differ between caregivers, since a node column holds one value. The gaps are kept for a double service node only. The units of time are the generator's own.
1792
+ * @param text - The file text.
1793
+ * @returns The instance: its patient, caregiver and service kind counts as `meta`, and its nodes in node order.
1794
+ * @throws NativeParseError naming the line for a section that is missing, given twice or of the wrong size, a value that is not a number, a requirement that is not zero or one, a node with more than two required services, a double service node with fewer than two or out of range, or processing times that differ between caregivers.
1795
+ */
1796
+ function parseMankowska(text) {
1797
+ const sections = sectionsOf$1(text);
1798
+ const nodeCount = countOf(sections, "nbNodes", DEPOTS);
1799
+ const caregivers = countOf(sections, "nbVehi", 1);
1800
+ const serviceKinds = countOf(sections, "nbServi", 1);
1801
+ const required = sized(sections, "r", nodeCount * serviceKinds);
1802
+ sized(sections, "a", caregivers * serviceKinds);
1803
+ const xs = sized(sections, "x", nodeCount);
1804
+ const ys = sized(sections, "y", nodeCount);
1805
+ sized(sections, "d", nodeCount * nodeCount);
1806
+ const times = sized(sections, "p", nodeCount * caregivers * serviceKinds);
1807
+ const least = sized(sections, "mind", nodeCount);
1808
+ const most = sized(sections, "maxd", nodeCount);
1809
+ const earliest = sized(sections, "e", nodeCount);
1810
+ const latest = sized(sections, "l", nodeCount);
1811
+ const line = (name) => sections.get(name)?.line ?? 1;
1812
+ const doubles = new Set((sections.get("DS")?.values ?? []).map((number) => {
1813
+ if (!Number.isInteger(number) || number < 1 || number > nodeCount) throw new NativeParseError(FORMAT$12, line("DS"), `the double service node ${String(number)} is not a node number from 1 to ${String(nodeCount)}`);
1814
+ return number - 1;
1815
+ }));
1816
+ const nodes = Array.from({ length: nodeCount }, (_, node) => {
1817
+ const at = (values) => cell(values, 1, node, 0);
1818
+ if (node === 0 || node === nodeCount - 1) return {
1819
+ id: String(node),
1820
+ role: "depot",
1821
+ x: at(xs),
1822
+ y: at(ys),
1823
+ services: null,
1824
+ duration1: null,
1825
+ duration2: null,
1826
+ earliest: at(earliest),
1827
+ latest: at(latest),
1828
+ minGap: null,
1829
+ maxGap: null
1830
+ };
1831
+ const services = Array.from({ length: serviceKinds }, (__, service) => service).filter((service) => {
1832
+ const flag = cell(required, serviceKinds, node, service);
1833
+ if (flag !== 0 && flag !== 1) throw new NativeParseError(FORMAT$12, line("r"), `node ${String(node)} has the requirement ${String(flag)} for service ${String(service + 1)}, not 0 or 1`);
1834
+ return flag === 1;
1835
+ });
1836
+ if (services.length > MOST_SERVICES || doubles.has(node) && services.length !== MOST_SERVICES) throw new NativeParseError(FORMAT$12, line("r"), `node ${String(node)} requires ${String(services.length)} services${doubles.has(node) ? " but is a double service node" : ""}`);
1837
+ const [first, second] = services.map((service) => {
1838
+ const byCaregiver = Array.from({ length: caregivers }, (__, caregiver) => cell(times, serviceKinds, node * caregivers + caregiver, service));
1839
+ const [time] = byCaregiver;
1840
+ if (time === void 0 || byCaregiver.some((other) => other !== time)) throw new NativeParseError(FORMAT$12, line("p"), `node ${String(node)} has processing times for service ${String(service + 1)} that differ between caregivers (${byCaregiver.join(", ")})`);
1841
+ return time;
1842
+ });
1843
+ return {
1844
+ id: String(node),
1845
+ role: "patient",
1846
+ x: at(xs),
1847
+ y: at(ys),
1848
+ services: services.length === 0 ? null : services.map((service) => String(service + 1)).join(" "),
1849
+ duration1: first ?? null,
1850
+ duration2: second ?? null,
1851
+ earliest: at(earliest),
1852
+ latest: at(latest),
1853
+ minGap: doubles.has(node) ? at(least) : null,
1854
+ maxGap: doubles.has(node) ? at(most) : null
1855
+ };
1856
+ });
1857
+ return {
1858
+ meta: {
1859
+ patients: nodeCount - DEPOTS,
1860
+ caregivers,
1861
+ serviceKinds
1862
+ },
1863
+ nodes
1864
+ };
1865
+ }
1866
+ //#endregion
1867
+ //#region src/parsers/tdc.ts
1868
+ const FORMAT$11 = "tdc";
1869
+ /** A technician line: its id, skills, time window and whether it is an investment option. */
1870
+ const TECHNICIAN = /^Technician id \d+, skills \[[^\]]*\], time window \[[^\]]*\], investment option (?:true|false)$/;
1871
+ /** A task line, with the fields the parser keeps captured: id, skills, duration, window opening and closing, priority, penalty, creation, modification (or `nothing`) and cancellation; its relevant technicians follow and are not kept. */
1872
+ const TASK = /^Task id (\d+), skills \[([^\]]*)\], duration (\d+), time window \[([^,\]]+), ([^\]]+)\], priority (\d+), penalty (-?\d+(?:\.\d+)?), created ([^,]+), modified ([^,]+), cancelled (true|false), relevant technician ids (?:Int64)?\[[\d, ]*\]$/;
1873
+ /** A travel time line, between two tasks or between a technician and a task, in either direction. */
1874
+ const TRAVEL = /^(?:task|technician)\d+(?:task|technician)\d+: -?\d+(?:\.\d+)?$/;
1875
+ /** The section headings, in the order the file gives them. */
1876
+ const HEADINGS = [
1877
+ "Technicians:",
1878
+ "Tasks:",
1879
+ "Travel time:"
1880
+ ];
1881
+ /** What a line before any heading and a line of each section is, for errors: indexed by the section's position plus one, so the lines before the first heading are first. */
1882
+ const SECTION_NOUNS = [
1883
+ "heading",
1884
+ "technician",
1885
+ "task",
1886
+ "travel time"
1887
+ ];
1888
+ /**
1889
+ * Names what a position's lines are, for an error.
1890
+ * @param position - A section's position plus one.
1891
+ * @returns The noun.
1892
+ */
1893
+ function nounOf(position) {
1894
+ return SECTION_NOUNS[position] ?? `section ${String(position)}`;
1895
+ }
1896
+ /** Where the task pattern captures each field it keeps. */
1897
+ const TASK_AT = {
1898
+ id: 1,
1899
+ skills: 2,
1900
+ duration: 3,
1901
+ earliest: 4,
1902
+ latest: 5,
1903
+ priority: 6,
1904
+ penalty: 7,
1905
+ created: 8,
1906
+ modified: 9,
1907
+ cancelled: 10
1908
+ };
1909
+ /** How much of a refused line an error shows. */
1910
+ const SHOWN = 120;
1911
+ /**
1912
+ * Parses a technician routing and scheduling instance in the sectioned text format of Gamst and Pisinger (2024): a `Technicians:` heading and a line per technician, a `Tasks:` heading and a line per task (`Task id <n>, skills ["<skill>", ...], duration <minutes>, time window [<opening>, <closing>], priority <n>, penalty <n>, created <time>, modified <time or nothing>, cancelled <true or false>, relevant technician ids [<ids>]`), then a `Travel time:` heading and a line per pair, the travel minutes between two tasks or between a technician and a task. The file is read a line at a time and the travel time lines, almost all of a file, are checked and let go, as the tasks/v1 target has no arcs; a technician's line is counted. A task's id is its id, and a modification of `nothing` is null. Times are as the file writes them.
1913
+ * @param pieces - The file's text, in pieces.
1914
+ * @returns The instance: its technician count as `meta`, and its tasks in file order.
1915
+ * @throws NativeParseError naming the line for a heading out of order or missing, a line that is not of its section's shape, or a task id given twice.
1916
+ */
1917
+ function parseTdc(pieces) {
1918
+ let section = -1;
1919
+ let technicians = 0;
1920
+ const nodes = [];
1921
+ const ids = /* @__PURE__ */ new Set();
1922
+ forEachLine(pieces, (raw, line) => {
1923
+ const text = raw.trim();
1924
+ if (text === "") return;
1925
+ const heading = HEADINGS.findIndex((candidate) => candidate === text);
1926
+ if (heading !== -1) {
1927
+ if (heading !== section + 1) throw new NativeParseError(FORMAT$11, line, `the heading ${text} is out of order`);
1928
+ section = heading;
1929
+ return;
1930
+ }
1931
+ if (section === 0 && TECHNICIAN.test(text)) {
1932
+ technicians += 1;
1933
+ return;
1934
+ }
1935
+ if (section === 2 && TRAVEL.test(text)) return;
1936
+ const task = section === 1 ? TASK.exec(text) : null;
1937
+ if (task === null) throw new NativeParseError(FORMAT$11, line, `the line is not a ${nounOf(section + 1)} line: ${JSON.stringify(text.slice(0, SHOWN))}`);
1938
+ const field = (index) => {
1939
+ const value = task[index];
1940
+ if (value === void 0) throw new NativeParseError(FORMAT$11, line, `the task line has no field ${String(index)}`);
1941
+ return value;
1942
+ };
1943
+ const id = field(TASK_AT.id);
1944
+ if (ids.has(id)) throw new NativeParseError(FORMAT$11, line, `the task ${id} is given twice`);
1945
+ ids.add(id);
1946
+ const skills = field(TASK_AT.skills).split(",").map((skill) => skill.trim().replace(/^"|"$/g, "")).filter((skill) => skill !== "");
1947
+ const modified = field(TASK_AT.modified);
1948
+ nodes.push({
1949
+ id,
1950
+ skills: skills.length === 0 ? null : skills.join(" "),
1951
+ durationMinutes: Number(field(TASK_AT.duration)),
1952
+ earliest: field(TASK_AT.earliest).trim(),
1953
+ latest: field(TASK_AT.latest).trim(),
1954
+ priority: Number(field(TASK_AT.priority)),
1955
+ penalty: Number(field(TASK_AT.penalty)),
1956
+ created: field(TASK_AT.created),
1957
+ modified: modified === "nothing" ? null : modified,
1958
+ cancelled: field(TASK_AT.cancelled)
1959
+ });
1960
+ });
1961
+ if (section !== HEADINGS.length - 1) throw new NativeParseError(FORMAT$11, 1, `the file has no ${nounOf(section + 2)} heading`);
1962
+ return {
1963
+ meta: { technicians },
1964
+ nodes
1965
+ };
1966
+ }
1967
+ //#endregion
1968
+ //#region src/parsers/verolog-2017.ts
1969
+ const FORMAT$10 = "verolog-2017";
1970
+ /** The header lines, with the key of `meta` each fills and whether its value is text. */
1971
+ const HEADER$3 = /* @__PURE__ */ new Map([
1972
+ ["DATASET", ["dataset", "text"]],
1973
+ ["NAME", ["name", "text"]],
1974
+ ["DAYS", ["days", "number"]],
1975
+ ["CAPACITY", ["capacity", "number"]],
1976
+ ["MAX_TRIP_DISTANCE", ["maxTripDistance", "number"]],
1977
+ ["DEPOT_COORDINATE", ["depotCoordinate", "number"]],
1978
+ ["VEHICLE_COST", ["vehicleCost", "number"]],
1979
+ ["VEHICLE_DAY_COST", ["vehicleDayCost", "number"]],
1980
+ ["DISTANCE_COST", ["distanceCost", "number"]]
1981
+ ]);
1982
+ /** The sections that hold rows, and how many numbers each row has. */
1983
+ const SECTION_WIDTHS = {
1984
+ TOOLS: 4,
1985
+ COORDINATES: 3,
1986
+ REQUESTS: 7
1987
+ };
1988
+ /** Where a request row keeps its fields. */
1989
+ const REQUEST_AT$1 = {
1990
+ id: 0,
1991
+ location: 1,
1992
+ fromDay: 2,
1993
+ toDay: 3,
1994
+ stayDays: 4,
1995
+ tool: 5,
1996
+ amount: 6
1997
+ };
1998
+ const WHOLE$3 = /^-?\d+$/;
1999
+ /**
2000
+ * Reads a field of a row.
2001
+ * @param row - The row.
2002
+ * @param index - The field's position.
2003
+ * @returns The number.
2004
+ * @throws NativeParseError when the row has no such field.
2005
+ */
2006
+ function at(row, index) {
2007
+ const value = row.values[index];
2008
+ if (value === void 0) throw new NativeParseError(FORMAT$10, row.line, `the row has no field ${String(index + 1)}`);
2009
+ return value;
2010
+ }
2011
+ /**
2012
+ * Parses an instance of the ORTEC VeRoLog Solver Challenge 2016 to 2017 (Kheiri et al. 2019): `KEY = value` header lines (the dataset and instance names, the planning days, the vehicle capacity, the longest trip, the depot's coordinate id and the three costs), then three sections each opened by `NAME = <count>` and followed by that many rows of whole numbers: `TOOLS` (id, size, how many are available, cost a day), `COORDINATES` (id, x, y) and `REQUESTS` (id, location, first day, last day, days the tools stay, tool kind, how many). Fields are separated by runs of spaces or tabs, and blank lines are skipped. The nodes are the depot, at its coordinate, with the id `depot`, then one node per request under the request's id at the position of its location. The tool table's sizes, availability and costs are not node fields and are not kept; its row count is.
2013
+ * @param text - The file text.
2014
+ * @returns The instance: its header values and tool count as `meta`, and its nodes, the depot first and the requests in file order.
2015
+ * @throws NativeParseError naming the line for an unknown or repeated key, a header value that is not a number where one is due, a missing header line or section, a section with a different number of rows than it says, a row that is not its section's count of whole numbers, a coordinate given twice, a request at a location the file does not give, or a request id given twice.
2016
+ */
2017
+ function parseVerolog2017(text) {
2018
+ const meta = /* @__PURE__ */ new Map();
2019
+ const sections = /* @__PURE__ */ new Map();
2020
+ let open;
2021
+ const close = () => {
2022
+ if (open !== void 0 && open.rows.length !== open.count) throw new NativeParseError(FORMAT$10, open.line, `the ${open.name} section says ${String(open.count)} rows and has ${String(open.rows.length)}`);
2023
+ open = void 0;
2024
+ };
2025
+ for (const [index, raw] of lines(text).entries()) {
2026
+ const line = index + 1;
2027
+ const trimmed = raw.trim();
2028
+ if (trimmed === "") continue;
2029
+ const keyed = /^([A-Z_]+)\s*=\s*(.*)$/.exec(trimmed);
2030
+ if (keyed?.[1] !== void 0 && keyed[2] !== void 0) {
2031
+ close();
2032
+ const [, key, value] = keyed;
2033
+ const section = Object.keys(SECTION_WIDTHS).find((name) => name === key);
2034
+ if (section !== void 0) {
2035
+ if (sections.has(section) || !WHOLE$3.test(value)) throw new NativeParseError(FORMAT$10, line, sections.has(section) ? `the ${section} section is given twice` : `the ${section} section's count ${JSON.stringify(value)} is not a whole number`);
2036
+ const rows = [];
2037
+ sections.set(section, rows);
2038
+ open = {
2039
+ name: section,
2040
+ rows,
2041
+ count: Number(value),
2042
+ line
2043
+ };
2044
+ continue;
2045
+ }
2046
+ const header = HEADER$3.get(key);
2047
+ if (header === void 0 || meta.has(header[0])) throw new NativeParseError(FORMAT$10, line, header === void 0 ? `the key ${key} is not one this format has` : `the key ${key} is given twice`);
2048
+ const [name, kind] = header;
2049
+ if (kind === "number" && !/^-?\d+(?:\.\d+)?$/.test(value)) throw new NativeParseError(FORMAT$10, line, `${key} must be a number, found ${JSON.stringify(value)}`);
2050
+ meta.set(name, kind === "number" ? Number(value) : value);
2051
+ continue;
2052
+ }
2053
+ const tokens = trimmed.split(/\s+/);
2054
+ if (open === void 0 || tokens.length !== SECTION_WIDTHS[open.name] || !tokens.every((token) => WHOLE$3.test(token))) throw new NativeParseError(FORMAT$10, line, open === void 0 ? `the row ${JSON.stringify(trimmed)} is outside any section` : `a ${open.name} row is ${String(SECTION_WIDTHS[open.name])} whole numbers, found ${JSON.stringify(trimmed)}`);
2055
+ open.rows.push({
2056
+ line,
2057
+ values: tokens.map(Number)
2058
+ });
2059
+ }
2060
+ close();
2061
+ for (const [key, [name]] of HEADER$3) if (!meta.has(name)) throw new NativeParseError(FORMAT$10, 1, `the file has no ${key} line`);
2062
+ const rowsOf = (name) => {
2063
+ const rows = sections.get(name);
2064
+ if (rows === void 0) throw new NativeParseError(FORMAT$10, 1, `the file has no ${name} section`);
2065
+ return rows;
2066
+ };
2067
+ const tools = rowsOf("TOOLS").length;
2068
+ const positions = /* @__PURE__ */ new Map();
2069
+ for (const row of rowsOf("COORDINATES")) {
2070
+ const id = at(row, 0);
2071
+ if (positions.has(id)) throw new NativeParseError(FORMAT$10, row.line, `the coordinate ${String(id)} is given twice`);
2072
+ positions.set(id, [at(row, 1), at(row, 2)]);
2073
+ }
2074
+ const positionOf = (location, line) => {
2075
+ const position = positions.get(location);
2076
+ if (position === void 0) throw new NativeParseError(FORMAT$10, line, `the location ${String(location)} is not among the coordinates`);
2077
+ return position;
2078
+ };
2079
+ const depot = Number(meta.get("depotCoordinate"));
2080
+ const [depotX, depotY] = positionOf(depot, 1);
2081
+ const nodes = [{
2082
+ id: "depot",
2083
+ x: depotX,
2084
+ y: depotY,
2085
+ kind: "depot",
2086
+ location: depot,
2087
+ fromDay: null,
2088
+ toDay: null,
2089
+ stayDays: null,
2090
+ tool: null,
2091
+ amount: null
2092
+ }];
2093
+ const ids = /* @__PURE__ */ new Set();
2094
+ for (const row of rowsOf("REQUESTS")) {
2095
+ const id = String(at(row, REQUEST_AT$1.id));
2096
+ if (ids.has(id)) throw new NativeParseError(FORMAT$10, row.line, `the request ${id} is given twice`);
2097
+ ids.add(id);
2098
+ const location = at(row, REQUEST_AT$1.location);
2099
+ const [x, y] = positionOf(location, row.line);
2100
+ nodes.push({
2101
+ id,
2102
+ x,
2103
+ y,
2104
+ kind: "request",
2105
+ location,
2106
+ fromDay: at(row, REQUEST_AT$1.fromDay),
2107
+ toDay: at(row, REQUEST_AT$1.toDay),
2108
+ stayDays: at(row, REQUEST_AT$1.stayDays),
2109
+ tool: String(at(row, REQUEST_AT$1.tool)),
2110
+ amount: at(row, REQUEST_AT$1.amount)
2111
+ });
2112
+ }
2113
+ return {
2114
+ meta: {
2115
+ ...Object.fromEntries(meta),
2116
+ tools
2117
+ },
2118
+ nodes
2119
+ };
2120
+ }
2121
+ //#endregion
1639
2122
  //#region src/parsers/pajek.ts
1640
2123
  /** Section keywords, lower-cased, mapped to how their lines are read. `*Network` names the network and starts no section. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
1641
2124
  const sections = /* @__PURE__ */ new Map([
@@ -3161,6 +3644,48 @@ const KIBIBYTE = 1024;
3161
3644
  /** Bytes decoded at a time for a format that is read in pieces. The decoder holds only a piece's text at once, so this bounds what a large table costs beyond its bytes; it is large enough that the per-piece work is small beside the piece. */
3162
3645
  const DECODE_PIECE_BYTES = 8 * KIBIBYTE * KIBIBYTE;
3163
3646
  /**
3647
+ * Decodes a file's bytes in the recipe's encoding as they arrive in pieces, as strictly as `decode`: a character whose bytes span two pieces is decoded whole with the later one, and bytes that are not valid in the stated encoding fail.
3648
+ */
3649
+ var PieceDecoder = class {
3650
+ parser;
3651
+ utf8;
3652
+ constructor(parser) {
3653
+ this.parser = parser;
3654
+ this.utf8 = parser.encoding === "latin1" ? void 0 : new TextDecoder("utf-8", { fatal: true });
3655
+ }
3656
+ /**
3657
+ * Decodes the next piece.
3658
+ * @param piece - Bytes that follow the pieces decoded before them.
3659
+ * @returns Their text, without a character the piece ends partway through.
3660
+ * @throws NativeParseError when the bytes are not valid in the stated encoding.
3661
+ */
3662
+ decode(piece) {
3663
+ const { utf8 } = this;
3664
+ return utf8 === void 0 ? decode(piece, this.parser) : this.strictly(() => utf8.decode(piece, { stream: true }));
3665
+ }
3666
+ /**
3667
+ * Ends the file.
3668
+ * @returns The text of any bytes still held.
3669
+ * @throws NativeParseError when the file ends partway through a character.
3670
+ */
3671
+ end() {
3672
+ const { utf8 } = this;
3673
+ return utf8 === void 0 ? "" : this.strictly(() => utf8.decode());
3674
+ }
3675
+ /**
3676
+ * Runs a utf-8 decode, rethrowing its failure in the package's taxonomy: TextDecoder reports invalid bytes only as a TypeError with no position.
3677
+ * @param run - The decode.
3678
+ * @returns Its text.
3679
+ */
3680
+ strictly(run) {
3681
+ try {
3682
+ return run();
3683
+ } catch (error) {
3684
+ throw new NativeParseError(this.parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
3685
+ }
3686
+ }
3687
+ };
3688
+ /**
3164
3689
  * Decodes a file's bytes in the recipe's encoding a piece at a time, as strictly as `decode`, so a file whose text would not fit in one string can still be read.
3165
3690
  * @param bytes - The unpacked file.
3166
3691
  * @param parser - The recipe's parser, which names the encoding.
@@ -3169,20 +3694,10 @@ const DECODE_PIECE_BYTES = 8 * KIBIBYTE * KIBIBYTE;
3169
3694
  * @throws NativeParseError when the bytes are not valid in the stated encoding.
3170
3695
  */
3171
3696
  function* decodePieces(bytes, parser, pieceBytes = DECODE_PIECE_BYTES) {
3172
- const decoder = parser.encoding === "latin1" ? void 0 : new TextDecoder("utf-8", { fatal: true });
3173
- try {
3174
- for (let start = 0; start < bytes.length; start += pieceBytes) {
3175
- const piece = bytes.subarray(start, start + pieceBytes);
3176
- yield decoder === void 0 ? decode(piece, parser) : decoder.decode(piece, { stream: true });
3177
- }
3178
- if (decoder !== void 0) {
3179
- const rest = decoder.decode();
3180
- if (rest !== "") yield rest;
3181
- }
3182
- } catch (error) {
3183
- if (error instanceof NativeParseError) throw error;
3184
- throw new NativeParseError(parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
3185
- }
3697
+ const decoder = new PieceDecoder(parser);
3698
+ for (let start = 0; start < bytes.length; start += pieceBytes) yield decoder.decode(bytes.subarray(start, start + pieceBytes));
3699
+ const rest = decoder.end();
3700
+ if (rest !== "") yield rest;
3186
3701
  }
3187
3702
  function parseNative(bytes, parser) {
3188
3703
  if (parser.format === "csv-table") return parseCsvTable(decodePieces(bytes, parser), parser);
@@ -3193,7 +3708,7 @@ function parseNative(bytes, parser) {
3193
3708
  split: void 0,
3194
3709
  pieces: decodePieces(bytes, parser)
3195
3710
  }], parser);
3196
- if (parser.format === "hgb") throw new NativeParseError(parser.format, 1, "an HGB dataset is two files, nodes and links, which a recipe reads from a parts archive");
3711
+ if (parser.format === "tdc") return parseTdc(decodePieces(bytes, parser));
3197
3712
  if (parser.format === "edge-list") return parseEdgeList(decodePieces(bytes, parser), parser);
3198
3713
  const text = decode(bytes, parser);
3199
3714
  if (parser.format === "geojson") return parseGeojson(text, parser);
@@ -3211,6 +3726,8 @@ function parseNative(bytes, parser) {
3211
3726
  if (parser.format === "vrp-rep") return parseVrpRep(text);
3212
3727
  if (parser.format === "hhcrsp") return parseHhcrsp(text);
3213
3728
  if (parser.format === "ogb") throw new NativeParseError(parser.format, 1, "the format reads the several files a files archive selects, not one file");
3729
+ if (parser.format === "mankowska") return parseMankowska(text);
3730
+ if (parser.format === "verolog-2017") return parseVerolog2017(text);
3214
3731
  return parsePajek(text);
3215
3732
  }
3216
3733
  /**
@@ -5029,18 +5546,20 @@ function unpackShapefile(bytes, member) {
5029
5546
  };
5030
5547
  }
5031
5548
  /**
5032
- * Unpacks downloaded bytes into every file a collection's selector matches, or, with an `xlsx` inner layout, into every sheet of every matched workbook that its `sheets` pattern matches as the sheet's CSV, at the member's path and the sheet's name joined by `/`, members in path order and sheets in the workbook's order.
5549
+ * Unpacks downloaded bytes into every file a collection's selector matches, or, with an `xlsx` inner layout, into every sheet of every matched workbook that its `sheets` pattern matches as the sheet's CSV, at the member's path and the sheet's name joined by `/`, members in path order and sheets in the workbook's order. The files are given one at a time: a zip member is inflated only when it is reached, so a reader that lets each file go before taking the next holds one member's bytes beside the archive, never every member at once.
5033
5550
  * @param bytes - The downloaded bytes.
5034
5551
  * @param archive - The recipe's archive layout, which selects members with `members`.
5035
5552
  * @returns Each file.
5036
- * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind, the selector matches no member, or a member an `xlsx` layout reads is not a workbook or has no sheet the pattern matches.
5553
+ * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind, the selector matches no member (at the first file taken), or a member an `xlsx` layout reads is not a workbook or has no sheet the pattern matches (at that member).
5037
5554
  */
5038
- function unpackMembers(bytes, archive) {
5039
- const members = selectMembers(bytes, archive);
5040
- if (archive.inner === void 0) return members;
5041
- const { sheets } = archive.inner;
5555
+ function* unpackMembers(bytes, archive) {
5556
+ const { sheets } = archive.inner ?? {};
5042
5557
  const wanted = sheets === void 0 ? void 0 : new RegExp(`^(?:${sheets})$`, "u");
5043
- return members.flatMap((member) => {
5558
+ for (const member of selectMembers(bytes, archive)) {
5559
+ if (archive.inner === void 0) {
5560
+ yield member;
5561
+ continue;
5562
+ }
5044
5563
  let names;
5045
5564
  try {
5046
5565
  names = sheetNames(member.bytes).filter((sheet) => wanted === void 0 || wanted.test(sheet));
@@ -5048,24 +5567,24 @@ function unpackMembers(bytes, archive) {
5048
5567
  throw new ArchiveError(`member ${member.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
5049
5568
  }
5050
5569
  if (names.length === 0) throw new ArchiveError(`member ${member.path} has no sheet matching ${sheets ?? ""}`);
5051
- return names.map((sheet) => ({
5570
+ yield* names.map((sheet) => ({
5052
5571
  path: `${member.path}/${sheet}`,
5053
5572
  bytes: sheetToCsv(member.bytes, {
5054
5573
  kind: "xlsx",
5055
5574
  sheet
5056
5575
  })
5057
5576
  }));
5058
- });
5577
+ }
5059
5578
  }
5060
5579
  /**
5061
- * Unpacks every member a collection's selector matches.
5580
+ * Unpacks every member a collection's selector matches, one at a time.
5062
5581
  * @param bytes - The downloaded bytes.
5063
5582
  * @param archive - The recipe's archive layout.
5064
5583
  * @returns Each matching member, in path order.
5065
- * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member.
5584
+ * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member, at the first member taken.
5066
5585
  */
5067
- function selectMembers(bytes, archive) {
5068
- return readMembers(bytes, archive.kind, (paths) => pickAll(paths, archive.members).map((path) => ({ path })));
5586
+ function* selectMembers(bytes, archive) {
5587
+ yield* readMembers(bytes, archive.kind, (paths) => pickAll(paths, archive.members).map((path) => ({ path })));
5069
5588
  }
5070
5589
  /**
5071
5590
  * Unpacks downloaded bytes into the files a parts archive names, each the one member its selector matches.
@@ -5075,57 +5594,45 @@ function selectMembers(bytes, archive) {
5075
5594
  * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or a part's selector does not match exactly one member.
5076
5595
  */
5077
5596
  function unpackParts(bytes, archive) {
5078
- return readMembers(bytes, archive.kind, (paths) => archive.parts.map((part) => ({
5597
+ return [...readMembers(bytes, archive.kind, (paths) => archive.parts.map((part) => ({
5079
5598
  name: part.name,
5080
5599
  path: pick(paths, part.member)
5081
- })));
5600
+ })))];
5082
5601
  }
5083
5602
  /**
5084
- * Reads the members of a file archive that a choice makes from its member paths, listing the archive once and decompressing it once however many members are chosen.
5085
- * @param bytes - The downloaded bytes.
5603
+ * Reads the members of a file archive that a choice makes from its member paths, one at a time: the archive is listed once and a zip member is inflated only when it is reached, so a reader that lets each member go before taking the next holds one member's bytes beside the archive.
5604
+ * @param bytes - The archive's bytes.
5086
5605
  * @param kind - The archive kind.
5087
5606
  * @param choose - Given every member path, the members to read, in the order wanted, each naming its path.
5088
5607
  * @returns Each choice with the bytes of the member it names, in the order chosen.
5089
5608
  * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or a chosen member cannot be read; whatever `choose` throws.
5090
5609
  */
5091
- function readMembers(bytes, kind, choose) {
5610
+ function* readMembers(bytes, kind, choose) {
5092
5611
  if (kind === "zip") {
5093
5612
  const names = [];
5094
5613
  unzip(bytes, (file) => {
5095
5614
  names.push(file.name);
5096
5615
  return false;
5097
5616
  });
5098
- const choices = choose(names);
5099
- const chosen = new Set(choices.map((choice) => choice.path));
5100
- const files = unzip(bytes, (candidate) => chosen.has(candidate.name));
5101
- return choices.map((choice) => {
5102
- const file = getOwn(files, choice.path);
5617
+ for (const choice of choose(names)) {
5618
+ const file = getOwn(unzip(bytes, (candidate) => candidate.name === choice.path), choice.path);
5103
5619
  if (file === void 0) throw new ArchiveError(`zip member ${choice.path} could not be read`);
5104
- return {
5620
+ yield {
5105
5621
  ...choice,
5106
5622
  bytes: file
5107
5623
  };
5108
- });
5624
+ }
5625
+ return;
5109
5626
  }
5110
5627
  const members = readTar(untarred(bytes, kind));
5111
- return choose(members.map((member) => member.path)).map((choice) => {
5628
+ for (const choice of choose(members.map((member) => member.path))) {
5112
5629
  const found = members.find((member) => member.path === choice.path);
5113
5630
  if (found === void 0) throw new ArchiveError(`tar member ${choice.path} could not be read`);
5114
- return {
5631
+ yield {
5115
5632
  ...choice,
5116
5633
  bytes: found.data
5117
5634
  };
5118
- });
5119
- }
5120
- /**
5121
- * Removes the compression a tar archive kind states, leaving the tar bytes.
5122
- * @param bytes - The downloaded bytes.
5123
- * @param kind - The tar archive kind: `tar` is uncompressed, `tar.gz` is gzip and `tar.bz2` is bzip2.
5124
- * @returns The tar archive.
5125
- */
5126
- function untarred(bytes, kind) {
5127
- if (kind === "tar") return bytes;
5128
- return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
5635
+ }
5129
5636
  }
5130
5637
  /**
5131
5638
  * Reads one sheet of a spreadsheet as CSV text, by the spreadsheet's format.
@@ -5150,10 +5657,10 @@ function sheetToCsv(bytes, archive) {
5150
5657
  * @throws ArchiveError when the bytes, or a file `inner` gunzips, are not well-formed, or the selector matches no member.
5151
5658
  */
5152
5659
  function unpackFiles(bytes, archive) {
5153
- const files = unpackMembers(bytes, {
5660
+ const files = [...unpackMembers(bytes, {
5154
5661
  kind: archive.kind,
5155
5662
  members: archive.files
5156
- });
5663
+ })];
5157
5664
  if (archive.inner === void 0) return files;
5158
5665
  return files.map((file) => {
5159
5666
  try {
@@ -5167,6 +5674,17 @@ function unpackFiles(bytes, archive) {
5167
5674
  });
5168
5675
  }
5169
5676
  /**
5677
+ * Unpacks a tar archive's bytes, which may be compressed: `tar.gz` is gunzipped and `tar.bz2` bunzipped first, `tar` read as it is.
5678
+ * @param bytes - The archive's bytes.
5679
+ * @param kind - The archive kind.
5680
+ * @returns The tar's bytes.
5681
+ * @throws ArchiveError when the bytes are not valid gzip or bzip2 data, as `gunzip` and `bunzip2` report.
5682
+ */
5683
+ function untarred(bytes, kind) {
5684
+ if (kind === "tar") return bytes;
5685
+ return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
5686
+ }
5687
+ /**
5170
5688
  * Gunzips bytes, rethrowing zlib's failure in the package's taxonomy.
5171
5689
  * @param bytes - Gzip data.
5172
5690
  * @returns The decompressed bytes.
@@ -5432,15 +5950,25 @@ async function getBytes(port, url, signal) {
5432
5950
  return new Uint8Array(await response.arrayBuffer());
5433
5951
  }
5434
5952
  /**
5435
- * Reads a source's pages, each starting where the rows held so far end, and checks each holds exactly the rows it should, so a response cut short is an error and not a table with rows missing.
5953
+ * Reads a source's pages in order, each starting where the rows read so far end, and checks each holds exactly the rows it should, so a response cut short is an error and not a table with rows missing. Only the page being read and the first page's header are held, so a caller that hands each page on holds no more of the source than that.
5436
5954
  * @param read - The source read.
5437
- * @param held - The pages read so far, each after the first without its header record.
5438
- * @param offset - How many rows the pages hold.
5439
- * @returns Every page.
5440
- * @throws KeepsetError when a page holds a different number of rows than the count says it should, or its header differs from the first page's.
5955
+ * @returns Each page, the first with its header record and every later one without it.
5956
+ * @throws KeepsetError when the source has no rows, a page holds a different number of rows than the count says it should, or its header differs from the first page's.
5441
5957
  */
5442
- async function readResourcePages(read, held, offset) {
5443
- if (offset >= read.count) return held;
5958
+ async function* resourcePages(read) {
5959
+ if (read.count === 0) throw new KeepsetError(`${read.resource} has no rows`);
5960
+ yield* pagesFrom(read, 0, void 0);
5961
+ }
5962
+ /**
5963
+ * Reads one page and then, by recursion, the pages after it.
5964
+ * @param read - The source read.
5965
+ * @param offset - The row the page starts at.
5966
+ * @param header - The first page's header record, or undefined when this is the first page.
5967
+ * @returns This page and every page after it.
5968
+ * @throws KeepsetError as `resourcePages` does.
5969
+ */
5970
+ async function* pagesFrom(read, offset, header) {
5971
+ if (offset >= read.count) return;
5444
5972
  const expected = Math.min(read.pageSize, read.count - offset);
5445
5973
  const page = await withAttempts(async () => {
5446
5974
  const bytes = await getBytes(read.port, read.pageUrl(offset), read.signal);
@@ -5448,10 +5976,15 @@ async function readResourcePages(read, held, offset) {
5448
5976
  if (rows !== expected) throw new KeepsetError(`${read.resource} gave ${String(rows)} rows at offset ${String(offset)} where ${String(expected)} of its ${String(read.count)} were due, so a response was cut short or the table changed during the download`);
5449
5977
  return bytes;
5450
5978
  }, read.signal);
5451
- const first = held[0];
5452
5979
  const headerEnd = page.indexOf(LINE_FEED) + 1;
5453
- if (first !== void 0 && !page.subarray(0, headerEnd).every((byte, index) => byte === first[index])) throw new KeepsetError(`${read.resource} gave a different header at offset ${String(offset)} than at the start`);
5454
- return readResourcePages(read, [...held, first === void 0 ? page : page.subarray(headerEnd)], offset + expected);
5980
+ if (header === void 0) {
5981
+ yield page;
5982
+ yield* pagesFrom(read, offset + expected, page.slice(0, headerEnd));
5983
+ return;
5984
+ }
5985
+ if (headerEnd !== header.length || !page.subarray(0, headerEnd).every((byte, index) => byte === header[index])) throw new KeepsetError(`${read.resource} gave a different header at offset ${String(offset)} than at the start`);
5986
+ yield page.subarray(headerEnd);
5987
+ yield* pagesFrom(read, offset + expected, header);
5455
5988
  }
5456
5989
  /**
5457
5990
  * Downloads a CSV source a page at a time and joins the pages under the first header.
@@ -5460,42 +5993,44 @@ async function readResourcePages(read, held, offset) {
5460
5993
  * @throws KeepsetError when the source has no rows or a page is not what it should be.
5461
5994
  */
5462
5995
  async function downloadPages(read) {
5463
- const pages = await readResourcePages(read, [], 0);
5464
- if (pages.length === 0) throw new KeepsetError(`${read.resource} has no rows`);
5996
+ const pages = [];
5997
+ for await (const page of resourcePages(read)) pages.push(page);
5465
5998
  return Buffer.concat(pages);
5466
5999
  }
5467
6000
  /**
5468
- * Downloads a whole Socrata resource as one CSV: the row count is read first, then pages of `pageSize` rows in the order of the rows' ids, each checked to hold the rows it should, and joined under the first page's header.
6001
+ * How a Socrata resource is read: its row count is read first, then pages of `pageSize` rows in the order of the rows' ids, with only the `select` columns when the recipe lists them, so the columns a parser drops are never transferred.
5469
6002
  * @param resource - The resource's URL, such as `https://data.example.org/resource/abcd-1234.csv`, without a query.
5470
- * @param pageSize - How many rows a page holds.
6003
+ * @param paging - The recipe's paging.
5471
6004
  * @param port - The network port.
5472
6005
  * @param signal - Aborts the download.
5473
- * @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
5474
- * @throws KeepsetError when the resource does not report a count, or a page is not what it should be.
6006
+ * @returns The source read.
6007
+ * @throws KeepsetError when the resource does not report a count.
5475
6008
  */
5476
- async function downloadSocrataResource(resource, pageSize, port, signal) {
6009
+ async function socrataRead(resource, paging, port, signal) {
5477
6010
  const reported = /^"count"\r?\n"(\d+)"/.exec(new TextDecoder("utf-8").decode(await getBytes(port, `${resource}?%24select=count(*)`, signal)));
5478
6011
  if (reported?.[1] === void 0) throw new KeepsetError(`${resource} did not report a row count`);
5479
- return downloadPages({
6012
+ const { pageSize, select } = paging;
6013
+ const columns = select === void 0 ? "" : `%24select=${encodeURIComponent(select.join(","))}&`;
6014
+ return {
5480
6015
  resource,
5481
- pageUrl: (offset) => `${resource}?%24order=%3Aid&%24limit=${String(pageSize)}&%24offset=${String(offset)}`,
6016
+ pageUrl: (offset) => `${resource}?${columns}%24order=%3Aid&%24limit=${String(pageSize)}&%24offset=${String(offset)}`,
5482
6017
  pageSize,
5483
6018
  count: Number(reported[1]),
5484
6019
  port,
5485
6020
  signal
5486
- });
6021
+ };
5487
6022
  }
5488
6023
  const CountResultSchema = z.object({ result: z.object({ total: z.int().min(0) }) });
5489
6024
  /**
5490
- * Downloads a whole CKAN datastore table as one CSV: the row count is read from the portal's datastore API, then pages of `pageSize` rows of the table's CSV dump, each checked to hold the rows it should, and joined under the first page's header.
6025
+ * How a CKAN datastore table is read: the row count is read from the portal's datastore API, then pages of `pageSize` rows of the table's CSV dump.
5491
6026
  * @param dump - The dump's URL, such as `https://data.example.org/datastore/dump/<resource id>`, without a query; a portal that serves CKAN under a path prefix keeps it before `/datastore/dump/`.
5492
6027
  * @param pageSize - How many rows a page holds.
5493
6028
  * @param port - The network port.
5494
6029
  * @param signal - Aborts the download.
5495
- * @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
5496
- * @throws KeepsetError when the URL is not a datastore dump, the table does not report a count, or a page is not what it should be.
6030
+ * @returns The source read.
6031
+ * @throws KeepsetError when the URL is not a datastore dump or the table does not report a count.
5497
6032
  */
5498
- async function downloadCkanDatastore(dump, pageSize, port, signal) {
6033
+ async function ckanRead(dump, pageSize, port, signal) {
5499
6034
  const parsed = /^(.*)\/datastore\/dump\/([^/?#]+)$/.exec(dump);
5500
6035
  if (parsed?.[1] === void 0 || parsed[2] === void 0) throw new KeepsetError(`${dump} is not a CKAN datastore dump URL`);
5501
6036
  const url = `${parsed[1]}/api/3/action/datastore_search?resource_id=${parsed[2]}&limit=0`;
@@ -5507,26 +6042,26 @@ async function downloadCkanDatastore(dump, pageSize, port, signal) {
5507
6042
  }
5508
6043
  const count = CountResultSchema.safeParse(body);
5509
6044
  if (!count.success) throw new KeepsetError(`${dump} did not report a row count`);
5510
- return downloadPages({
6045
+ return {
5511
6046
  resource: dump,
5512
6047
  pageUrl: (offset) => `${dump}?format=csv&limit=${String(pageSize)}&offset=${String(offset)}`,
5513
6048
  pageSize,
5514
6049
  count: count.data.result.total,
5515
6050
  port,
5516
6051
  signal
5517
- });
6052
+ };
5518
6053
  }
5519
6054
  const OpenFemaCountSchema = z.object({ metadata: z.object({ count: z.int().min(0) }) });
5520
6055
  /**
5521
- * Downloads a whole OpenFEMA entity, or the rows of it the source URL selects, as one CSV: the row count is read from the API's own metadata, then pages of `pageSize` rows in the order of the entity's id, each checked to hold the rows it should, and joined under the first page's header.
6056
+ * How an OpenFEMA entity, or the rows of it the source URL selects, is read: the row count is read from the API's own metadata, then pages of `pageSize` rows in the order of the entity's id.
5522
6057
  * @param entity - The entity's URL with any `$select` or `$filter` but no `$top`, `$skip`, `$orderby` or `$format`, such as `https://www.fema.gov/api/open/v3/NfipClaims?$select=id,state`.
5523
6058
  * @param pageSize - How many rows a page holds.
5524
6059
  * @param port - The network port.
5525
6060
  * @param signal - Aborts the download.
5526
- * @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
5527
- * @throws KeepsetError when the source URL sets a paging parameter, the entity does not report a count, or a page is not what it should be.
6061
+ * @returns The source read.
6062
+ * @throws KeepsetError when the source URL sets a paging parameter or the entity does not report a count.
5528
6063
  */
5529
- async function downloadOpenFema(entity, pageSize, port, signal) {
6064
+ async function openFemaRead(entity, pageSize, port, signal) {
5530
6065
  const base = new URL(entity);
5531
6066
  for (const name of [
5532
6067
  "$top",
@@ -5552,7 +6087,7 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
5552
6087
  }
5553
6088
  const count = OpenFemaCountSchema.safeParse(body);
5554
6089
  if (!count.success) throw new KeepsetError(`${entity} did not report a row count`);
5555
- return downloadPages({
6090
+ return {
5556
6091
  resource: entity,
5557
6092
  pageUrl: (offset) => withParameters({
5558
6093
  $format: "csv",
@@ -5564,10 +6099,34 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
5564
6099
  count: count.data.metadata.count,
5565
6100
  port,
5566
6101
  signal
5567
- });
6102
+ };
5568
6103
  }
5569
6104
  /**
5570
- * Downloads a paged source as the one file its pages make, in the way its recipe's paging says.
6105
+ * How a CSV paged source is read, in the way its recipe's paging says.
6106
+ * @param url - The source's URL.
6107
+ * @param paging - How the source is paged.
6108
+ * @param port - The network port.
6109
+ * @param signal - Aborts the download.
6110
+ * @returns The source read.
6111
+ */
6112
+ async function csvRead(url, paging, port, signal) {
6113
+ if (paging.kind === "socrata-resource") return socrataRead(url, paging, port, signal);
6114
+ return paging.kind === "ckan-datastore" ? ckanRead(url, paging.pageSize, port, signal) : openFemaRead(url, paging.pageSize, port, signal);
6115
+ }
6116
+ /**
6117
+ * Reads a CSV paged source a page at a time: the first page with its header and every later page without it, so the pages in order are the one CSV `downloadPaged` joins them into. Only the page being read is held.
6118
+ * @param url - The source's URL.
6119
+ * @param paging - How the source is paged.
6120
+ * @param port - The network port.
6121
+ * @param signal - Aborts the download.
6122
+ * @returns Each page's bytes, in order.
6123
+ * @throws KeepsetError when the source does not report a count, has no rows, or a page is not what it should be.
6124
+ */
6125
+ async function* csvPages(url, paging, port, signal) {
6126
+ yield* resourcePages(await csvRead(url, paging, port, signal));
6127
+ }
6128
+ /**
6129
+ * Downloads a paged source as the one file its pages make, in the way its recipe's paging says: an ArcGIS layer as one GeoJSON feature collection, any other source as one CSV of its pages joined under the first page's header.
5571
6130
  * @param url - The source's URL.
5572
6131
  * @param paging - How the source is paged.
5573
6132
  * @param port - The network port.
@@ -5577,17 +6136,16 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
5577
6136
  */
5578
6137
  async function downloadPaged(url, paging, port, signal) {
5579
6138
  if (paging.kind === "arcgis-query") return downloadArcgisLayer(url, paging.orderBy, port, signal);
5580
- if (paging.kind === "socrata-resource") return downloadSocrataResource(url, paging.pageSize, port, signal);
5581
- return paging.kind === "ckan-datastore" ? downloadCkanDatastore(url, paging.pageSize, port, signal) : downloadOpenFema(url, paging.pageSize, port, signal);
6139
+ return downloadPages(await csvRead(url, paging, port, signal));
5582
6140
  }
5583
6141
  //#endregion
5584
- //#region src/node/fetch-dataset.ts
6142
+ //#region src/node/recipe-run.ts
5585
6143
  /**
5586
- * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
6144
+ * The URL a recipe downloads a source from, as `downloadUrl` gives it. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
5587
6145
  * @param source - The source.
5588
6146
  * @returns The URL to request.
5589
6147
  */
5590
- function downloadUrl(source) {
6148
+ function sourceUrl(source) {
5591
6149
  if (source.status === "dead") {
5592
6150
  if (source.snapshot === void 0) throw new KeepsetError(`dead source ${source.url} has no snapshot`);
5593
6151
  return `https://web.archive.org/web/${source.snapshot.timestamp}id_/${source.url}`;
@@ -5595,6 +6153,18 @@ function downloadUrl(source) {
5595
6153
  return source.url;
5596
6154
  }
5597
6155
  /**
6156
+ * The source record a provenance carries: the entry's source as recorded.
6157
+ * @param source - The source.
6158
+ * @returns Its URL, status and any snapshot.
6159
+ */
6160
+ function sourceRecord(source) {
6161
+ return {
6162
+ url: source.url,
6163
+ status: source.status,
6164
+ ...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
6165
+ };
6166
+ }
6167
+ /**
5598
6168
  * Runs one step, rethrowing any failure as a RecipeError naming the entry and the step, with the original error as its cause.
5599
6169
  * @param entryId - The entry.
5600
6170
  * @param name - The step.
@@ -5621,6 +6191,373 @@ function metaOf(entry) {
5621
6191
  ...entry.properties
5622
6192
  };
5623
6193
  }
6194
+ //#endregion
6195
+ //#region src/node/id-set.ts
6196
+ /** The first table size, in slots: 2 to the 16th, a power of two, so a hash is reduced to a slot by masking (and every doubling keeps it one). */
6197
+ const INITIAL_SLOTS = 65536;
6198
+ /** The first arena size, in bytes: a mebibyte, which a stream of a few tens of thousands of short ids fills before the arena first doubles. */
6199
+ const INITIAL_ARENA_BYTES = 1048576;
6200
+ /** The table grows when more than this share of its slots hold an id, which keeps linear probing short. */
6201
+ const MAX_LOAD = .7;
6202
+ /** The bytes before each id in the arena: its length, as an unsigned 32-bit integer. */
6203
+ const LENGTH_BYTES = 4;
6204
+ /** The largest arena, 2 to the 31st less one: slots hold an arena offset plus one in a signed 32-bit integer. */
6205
+ const MAX_ARENA_BYTES = 2147483647;
6206
+ /** The FNV-1a 32-bit offset basis and prime (http://www.isthe.com/chongo/tech/comp/fnv/). */
6207
+ const FNV_OFFSET = 2166136261;
6208
+ const FNV_PRIME = 16777619;
6209
+ const encoder = new TextEncoder();
6210
+ /**
6211
+ * The FNV-1a hash of bytes.
6212
+ * @param bytes - The bytes.
6213
+ * @returns The 32-bit hash.
6214
+ */
6215
+ function fnv1a(bytes) {
6216
+ let hash = FNV_OFFSET;
6217
+ for (const byte of bytes) hash = Math.imul(hash ^ byte, FNV_PRIME);
6218
+ return hash >>> 0;
6219
+ }
6220
+ /**
6221
+ * Reads a slot of a table.
6222
+ * @param slots - The table.
6223
+ * @param slot - A slot index, always masked into the table's range.
6224
+ * @returns What the slot holds.
6225
+ * @throws RangeError when the index is outside the table, which masking never gives.
6226
+ */
6227
+ function slotOf(slots, slot) {
6228
+ const held = slots[slot];
6229
+ if (held === void 0) throw new RangeError(`slot ${String(slot)} is outside a table of ${String(slots.length)}`);
6230
+ return held;
6231
+ }
6232
+ /**
6233
+ * A set of ids that holds each id as its UTF-8 bytes in one growing arena, indexed by an open-addressed table of arena offsets, so a stream of millions of ids is checked for repeats exactly while holding a few bytes beyond each id's own and no JavaScript string or object per id. Membership compares the bytes themselves, so two different ids are never taken for one.
6234
+ */
6235
+ var IdSet = class {
6236
+ arena = new Uint8Array(INITIAL_ARENA_BYTES);
6237
+ view = new DataView(this.arena.buffer);
6238
+ used = 0;
6239
+ /** Each slot holds an id's arena offset plus one, or zero when empty. */
6240
+ slots = new Int32Array(INITIAL_SLOTS);
6241
+ count = 0;
6242
+ /** @returns How many ids the set holds. */
6243
+ get size() {
6244
+ return this.count;
6245
+ }
6246
+ /**
6247
+ * Adds an id.
6248
+ * @param id - The id.
6249
+ * @returns True when the id was new, false when the set already held it.
6250
+ * @throws KeepsetError when the ids held would pass the largest arena.
6251
+ */
6252
+ add(id) {
6253
+ const bytes = encoder.encode(id);
6254
+ const hash = fnv1a(bytes);
6255
+ const mask = this.slots.length - 1;
6256
+ for (let slot = hash & mask;; slot = slot + 1 & mask) {
6257
+ const held = slotOf(this.slots, slot);
6258
+ if (held === 0) {
6259
+ this.slots[slot] = this.store(bytes) + 1;
6260
+ this.count += 1;
6261
+ if (this.count > this.slots.length * MAX_LOAD) this.grow();
6262
+ return true;
6263
+ }
6264
+ if (this.equals(held - 1, bytes)) return false;
6265
+ }
6266
+ }
6267
+ /**
6268
+ * Whether the id stored at an offset is these bytes.
6269
+ * @param offset - The id's arena offset.
6270
+ * @param bytes - The bytes.
6271
+ * @returns True when they are the same.
6272
+ */
6273
+ equals(offset, bytes) {
6274
+ if (this.lengthAt(offset) !== bytes.length) return false;
6275
+ const start = offset + LENGTH_BYTES;
6276
+ return bytes.every((byte, index) => this.arena[start + index] === byte);
6277
+ }
6278
+ /**
6279
+ * The length of the id stored at an offset.
6280
+ * @param offset - The id's arena offset.
6281
+ * @returns Its byte length.
6282
+ */
6283
+ lengthAt(offset) {
6284
+ return this.view.getUint32(offset, true);
6285
+ }
6286
+ /**
6287
+ * Appends an id's bytes to the arena.
6288
+ * @param bytes - The bytes.
6289
+ * @returns Their offset.
6290
+ */
6291
+ store(bytes) {
6292
+ const needed = this.used + LENGTH_BYTES + bytes.length;
6293
+ if (needed > MAX_ARENA_BYTES) throw new KeepsetError(`the ids of the stream pass ${String(MAX_ARENA_BYTES)} bytes, more than one id set holds`);
6294
+ if (needed > this.arena.length) {
6295
+ const grown = new Uint8Array(Math.min(MAX_ARENA_BYTES, Math.max(needed, this.arena.length * 2)));
6296
+ grown.set(this.arena.subarray(0, this.used));
6297
+ this.arena = grown;
6298
+ this.view = new DataView(grown.buffer);
6299
+ }
6300
+ const offset = this.used;
6301
+ this.view.setUint32(offset, bytes.length, true);
6302
+ this.arena.set(bytes, offset + LENGTH_BYTES);
6303
+ this.used = needed;
6304
+ return offset;
6305
+ }
6306
+ /** Doubles the table, placing every id again by its hash, which is recomputed from the arena so the table holds nothing but offsets. */
6307
+ grow() {
6308
+ const slots = new Int32Array(this.slots.length * 2);
6309
+ const mask = slots.length - 1;
6310
+ for (const held of this.slots) if (held !== 0) {
6311
+ const start = held - 1 + LENGTH_BYTES;
6312
+ let slot = fnv1a(this.arena.subarray(start, start + this.lengthAt(held - 1))) & mask;
6313
+ while (slotOf(slots, slot) !== 0) slot = slot + 1 & mask;
6314
+ slots[slot] = held;
6315
+ }
6316
+ this.slots = slots;
6317
+ }
6318
+ };
6319
+ //#endregion
6320
+ //#region src/node/stream-dataset.ts
6321
+ /**
6322
+ * The parts of a recipe a stream reads, checked to be a streamed recipe: the entry schema refuses a `stream` recipe that is not a CSV paged source read by the csv-table parser with no archive, so this only fails for an entry whose recipe does not stream.
6323
+ * @param entry - The entry.
6324
+ * @returns The recipe's source, paging and parser, and the category.
6325
+ * @throws RecipeError when the recipe is not fetchable, does not stream, or its category has no streamed form.
6326
+ */
6327
+ function streamedRecipe(entry) {
6328
+ const { recipe, category } = entry;
6329
+ if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
6330
+ if ("files" in recipe || recipe.stream !== true) throw new RecipeError(entry.id, "recipe", "the recipe is not streamed, so fetchDataset reads it as one document");
6331
+ const { paging, parser } = recipe;
6332
+ if (paging === void 0 || paging.kind === "arcgis-query" || parser.format !== "csv-table" || !isStreamable(category)) throw new RecipeError(entry.id, "recipe", "a streamed recipe reads a CSV paged source through the csv-table parser, for a category with a streamed form");
6333
+ return {
6334
+ source: recipe.source,
6335
+ paging,
6336
+ parser,
6337
+ category
6338
+ };
6339
+ }
6340
+ /**
6341
+ * Whether a recipe is streamed, so `fetchDataset` refuses it and `streamDataset` reads it.
6342
+ * @param recipe - A recipe.
6343
+ * @returns True for a streamed recipe.
6344
+ */
6345
+ function isStreamed(recipe) {
6346
+ return recipe.status === "fetchable" && !("files" in recipe) && recipe.stream === true;
6347
+ }
6348
+ /**
6349
+ * What a crosswalk path addresses, for the separability check: the document's meta, a field of every node, or anything else.
6350
+ * @param path - A crosswalk path.
6351
+ * @returns Its class.
6352
+ */
6353
+ function pathClass(path) {
6354
+ if (path === "meta" || path.startsWith("meta.")) return "meta";
6355
+ return path === "nodes[]" || path.startsWith("nodes[].") ? "node" : "other";
6356
+ }
6357
+ /**
6358
+ * Why a crosswalk cannot be applied a page of nodes at a time, if it cannot: every rule must read and write the meta alone or one node's fields alone, with no `when`, so encoding a page's nodes under the document's meta gives exactly that page's part of encoding the whole document, and a round trip that holds on every page holds on the whole.
6359
+ * @param crosswalk - The crosswalk.
6360
+ * @returns The first rule that breaks this, described, or undefined when every rule keeps to it.
6361
+ */
6362
+ function inseparableRule(crosswalk) {
6363
+ for (const [index, rule] of [...crosswalk.rules, ...crosswalk.oneWay].entries()) {
6364
+ if ("when" in rule && rule.when !== void 0) return `rule ${String(index)} (${rule.op}) has a when, which may read the whole document`;
6365
+ const paths = ["from" in rule ? rule.from : void 0, "to" in rule ? rule.to : void 0].filter((path) => path !== void 0);
6366
+ const classes = new Set(paths.map(pathClass));
6367
+ const [only] = classes;
6368
+ if (classes.size !== 1 || only === "other") return `rule ${String(index)} (${rule.op}) reads or writes ${paths.join(" and ")}, not the meta alone or one node's fields alone`;
6369
+ }
6370
+ }
6371
+ /**
6372
+ * Passes on what an async iterable yields, rethrowing its failure as a RecipeError naming the entry and the step, as `step` does for one call.
6373
+ * @param entryId - The entry.
6374
+ * @param name - The step.
6375
+ * @param source - What the step yields.
6376
+ * @returns The same items, in order.
6377
+ */
6378
+ async function* inStep(entryId, name, source) {
6379
+ try {
6380
+ yield* source;
6381
+ } catch (error) {
6382
+ if (error instanceof RecipeError) throw error;
6383
+ throw new RecipeError(entryId, name, error instanceof Error ? error.message : String(error), { cause: error });
6384
+ }
6385
+ }
6386
+ /**
6387
+ * Runs a streamed recipe's download and parse steps a page at a time: each page's bytes are hashed, decoded and read into rows, and then let go.
6388
+ * @param dataset - The dataset.
6389
+ * @param recipe - Its streamed recipe.
6390
+ * @param options - The network port, abort signal and clock.
6391
+ * @returns The stream.
6392
+ */
6393
+ function openNative(dataset, recipe, options) {
6394
+ const { entry } = dataset;
6395
+ const { fetch: port = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
6396
+ const source = entry.sources[recipe.source];
6397
+ if (source === void 0) throw new RecipeError(entry.id, "recipe", `the recipe names source ${String(recipe.source)}, which the entry does not have`);
6398
+ const fetchedUrl = sourceUrl(source);
6399
+ const retrievedAt = now().toISOString();
6400
+ const hash = createHash("sha256");
6401
+ let byteLength = 0;
6402
+ async function* pages() {
6403
+ const decoder = new PieceDecoder(recipe.parser);
6404
+ const reader = new TableReader(recipe.parser);
6405
+ for await (const page of inStep(entry.id, "download", csvPages(fetchedUrl, recipe.paging, port, signal))) {
6406
+ hash.update(page);
6407
+ byteLength += page.byteLength;
6408
+ const rows = await step(entry.id, "parse", () => reader.push(decoder.decode(page)));
6409
+ if (rows.length > 0) yield rows;
6410
+ }
6411
+ const rows = await step(entry.id, "parse", () => [...reader.push(decoder.end()), ...reader.end()]);
6412
+ if (rows.length > 0) yield rows;
6413
+ }
6414
+ return {
6415
+ meta: metaOf(entry),
6416
+ pages: pages(),
6417
+ provenance: () => ({
6418
+ entryId: entry.id,
6419
+ catalogue: dataset.catalogue,
6420
+ category: entry.category,
6421
+ source: sourceRecord(source),
6422
+ fetchedUrl,
6423
+ retrievedAt,
6424
+ sha256: hash.copy().digest("hex"),
6425
+ byteLength
6426
+ })
6427
+ };
6428
+ }
6429
+ /**
6430
+ * The nodes of an encoded page: the crosswalk writes only the meta and the nodes, so the result is an object with a `nodes` array.
6431
+ * @param encoded - The crosswalk's output for a page.
6432
+ * @returns Its nodes.
6433
+ * @throws KeepsetError when the output is not an object with a nodes array and nothing beyond meta.
6434
+ */
6435
+ function nodesOf(encoded) {
6436
+ if (typeof encoded !== "object" || encoded === null || Array.isArray(encoded) || !("nodes" in encoded) || !Array.isArray(encoded.nodes)) throw new KeepsetError("the crosswalk's output is not an object with a nodes array");
6437
+ const extra = Object.keys(encoded).filter((key) => key !== "meta" && key !== "nodes");
6438
+ if (extra.length > 0) throw new KeepsetError(`the crosswalk's output has ${extra.join(", ")} beside meta and nodes`);
6439
+ return encoded.nodes;
6440
+ }
6441
+ /**
6442
+ * Checks a crosswalk can run a page at a time, or fails the crosswalk step.
6443
+ * @param entry - The entry.
6444
+ * @param crosswalk - Its crosswalk.
6445
+ * @throws RecipeError when a rule reaches beyond the meta or one node's fields.
6446
+ */
6447
+ function assertSeparable(entry, crosswalk) {
6448
+ const reason = inseparableRule(crosswalk);
6449
+ if (reason !== void 0) throw new RecipeError(entry.id, "crosswalk", `a streamed crosswalk must apply to a page of nodes on its own: ${reason}`);
6450
+ }
6451
+ /**
6452
+ * Streams a table, as `streamEntry` describes.
6453
+ * @param dataset - A dataset whose recipe streams.
6454
+ * @param sink - Where the meta and the nodes go.
6455
+ * @param options - The network port, abort signal and clock.
6456
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
6457
+ * @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
6458
+ */
6459
+ async function streamTable(dataset, sink, options) {
6460
+ const { entry, crosswalk } = dataset;
6461
+ const recipe = streamedRecipe(entry);
6462
+ assertSeparable(entry, crosswalk);
6463
+ const shape = getStreamedShape(recipe.category);
6464
+ const native = openNative(dataset, recipe, options);
6465
+ const meta = await step(entry.id, "validate", async () => {
6466
+ const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, {
6467
+ meta: native.meta,
6468
+ nodes: []
6469
+ }));
6470
+ const parsed = shape.meta.safeParse(typeof encoded === "object" && encoded !== null && !Array.isArray(encoded) && "meta" in encoded ? encoded.meta : void 0);
6471
+ if (!parsed.success) throw new KeepsetError(`the result's meta is not ${entry.category} meta: ${parsed.error.issues.map((issue) => `${["meta", ...issue.path.map(String)].join(".")}: ${issue.message}`).join("; ")}`);
6472
+ return parsed.data;
6473
+ });
6474
+ await sink.meta(meta);
6475
+ const ids = new IdSet();
6476
+ for await (const rows of native.pages) {
6477
+ const offset = ids.size;
6478
+ const encoded = await step(entry.id, "crosswalk", () => nodesOf(encode(crosswalk, {
6479
+ meta: native.meta,
6480
+ nodes: rows
6481
+ })));
6482
+ const nodes = await step(entry.id, "validate", () => encoded.map((candidate, index) => {
6483
+ const parsed = shape.node.safeParse(candidate);
6484
+ if (!parsed.success) throw new KeepsetError(`the result is not ${entry.category}: ${parsed.error.issues.map((issue) => `${[
6485
+ "nodes",
6486
+ String(offset + index),
6487
+ ...issue.path.map(String)
6488
+ ].join(".")}: ${issue.message}`).join("; ")}`);
6489
+ if (!ids.add(parsed.data.id)) throw new KeepsetError(`the result is not ${entry.category}: nodes.${String(offset + index)}.id: duplicate node id ${JSON.stringify(parsed.data.id)}`);
6490
+ return parsed.data;
6491
+ }));
6492
+ await sink.nodes(nodes);
6493
+ }
6494
+ const counts = /* @__PURE__ */ new Map([["nodes", ids.size]]);
6495
+ const warnings = Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`);
6496
+ return {
6497
+ dataset,
6498
+ meta,
6499
+ provenance: native.provenance(),
6500
+ counts,
6501
+ warnings
6502
+ };
6503
+ }
6504
+ /**
6505
+ * Checks a streamed dataset's crosswalk on its real data a page at a time: crossmap's `assertRoundTrip` on the meta alone and then on each page's rows under the meta. The crosswalk applies to a page on its own, so the round trip holds on the whole table exactly when it holds on every page. It stops at the first page that differs and reports the path in the whole document.
6506
+ * @param dataset - A dataset whose recipe streams.
6507
+ * @param options - The network port, abort signal and clock.
6508
+ * @returns Whether the round trip holds, and where it first differs when it does not, with the provenance of the bytes it was checked on: every byte when it holds, those read up to the page that differs when it does not.
6509
+ * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison or the crosswalk cannot run.
6510
+ */
6511
+ async function roundTripTable(dataset, options) {
6512
+ const { entry, crosswalk } = dataset;
6513
+ const recipe = streamedRecipe(entry);
6514
+ assertSeparable(entry, crosswalk);
6515
+ const native = openNative(dataset, recipe, options);
6516
+ const check = (nodes, offset) => {
6517
+ try {
6518
+ assertRoundTrip(crosswalk, [{
6519
+ meta: native.meta,
6520
+ nodes
6521
+ }]);
6522
+ } catch (error) {
6523
+ if (error instanceof RoundTripMismatchError) {
6524
+ const path = error.path.replace(/^nodes\[(\d+)\]/, (_, index) => `nodes[${String(offset + Number(index))}]`);
6525
+ return {
6526
+ holds: false,
6527
+ path,
6528
+ message: error.message.replaceAll(error.path, path),
6529
+ provenance: native.provenance()
6530
+ };
6531
+ }
6532
+ throw new RecipeError(entry.id, "crosswalk", error instanceof Error ? error.message : String(error), { cause: error });
6533
+ }
6534
+ };
6535
+ const metaMismatch = check([], 0);
6536
+ if (metaMismatch !== void 0) return metaMismatch;
6537
+ let offset = 0;
6538
+ for await (const rows of native.pages) {
6539
+ const mismatch = check(rows, offset);
6540
+ if (mismatch !== void 0) {
6541
+ await native.pages.return();
6542
+ return mismatch;
6543
+ }
6544
+ offset += rows.length;
6545
+ }
6546
+ return {
6547
+ holds: true,
6548
+ provenance: native.provenance()
6549
+ };
6550
+ }
6551
+ //#endregion
6552
+ //#region src/node/fetch-dataset.ts
6553
+ /**
6554
+ * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
6555
+ * @param source - The source.
6556
+ * @returns The URL to request.
6557
+ */
6558
+ function downloadUrl(source) {
6559
+ return sourceUrl(source);
6560
+ }
5624
6561
  /**
5625
6562
  * Puts an instance's id beside the fields of its encoded document. A document that is not an object is left as it is, for the collection schema to refuse.
5626
6563
  * @param document - The crosswalk's output for one member.
@@ -5634,6 +6571,139 @@ function withId(document, id) {
5634
6571
  } : document;
5635
6572
  }
5636
6573
  /**
6574
+ * Downloads one URL's bytes through the network port.
6575
+ * @param port - The network port.
6576
+ * @param url - The URL.
6577
+ * @param signal - Aborts the download.
6578
+ * @returns The bytes.
6579
+ * @throws KeepsetError when the server does not answer successfully.
6580
+ */
6581
+ async function downloadBytes(port, url, signal) {
6582
+ const response = await port(url, signal === void 0 ? {} : { signal });
6583
+ if (!response.ok) throw new KeepsetError(`${url} answered ${String(response.status)} ${response.statusText}`);
6584
+ return new Uint8Array(await response.arrayBuffer());
6585
+ }
6586
+ /**
6587
+ * The record of one download: the source, the URL requested and a hash of exactly the bytes that arrived.
6588
+ * @param source - The entry's source.
6589
+ * @param fetchedUrl - The URL requested.
6590
+ * @param bytes - The bytes.
6591
+ * @returns The download record.
6592
+ */
6593
+ function downloadOf(source, fetchedUrl, bytes) {
6594
+ return {
6595
+ source: sourceRecord(source),
6596
+ fetchedUrl,
6597
+ sha256: createHash("sha256").update(bytes).digest("hex"),
6598
+ byteLength: bytes.byteLength
6599
+ };
6600
+ }
6601
+ /**
6602
+ * Parses one file of a collection into its native document, naming the file when it fails.
6603
+ * @param id - The instance's id.
6604
+ * @param bytes - The file's bytes.
6605
+ * @param parser - The recipe's parser.
6606
+ * @returns The instance's native document, with only its own header values as `meta`.
6607
+ * @throws KeepsetError naming the instance when the file does not parse.
6608
+ */
6609
+ function parseInstance(id, bytes, parser) {
6610
+ try {
6611
+ const parsed = parseNative(bytes, parser);
6612
+ return {
6613
+ id,
6614
+ native: {
6615
+ ...parsed,
6616
+ meta: "meta" in parsed ? parsed.meta : {}
6617
+ }
6618
+ };
6619
+ } catch (error) {
6620
+ throw new KeepsetError(`member ${id}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
6621
+ }
6622
+ }
6623
+ /**
6624
+ * Unpacks and parses the bytes of one archive or file into instances, each member let go once it is parsed.
6625
+ * @param bytes - The bytes.
6626
+ * @param archive - Their archive layout.
6627
+ * @param parser - The recipe's parser.
6628
+ * @param prefix - The path the instance ids are filed under, joined to each member's path, or undefined to use the member's path alone.
6629
+ * @returns The instances, in member order.
6630
+ */
6631
+ function instancesOf(bytes, archive, parser, prefix) {
6632
+ const named = (member) => {
6633
+ if (prefix === void 0) {
6634
+ if (member === void 0) throw new KeepsetError("an instance of a collection has no path");
6635
+ return member;
6636
+ }
6637
+ return member === void 0 ? prefix : `${prefix}/${member}`;
6638
+ };
6639
+ if ("members" in archive) return Array.from(unpackMembers(bytes, archive), (member) => parseInstance(named(member.path), member.bytes, parser));
6640
+ if ("parts" in archive) throw new ArchiveError(`an instance is read from one file or from matching members, not from the named parts ${archive.parts.map((part) => part.name).join(", ")}`);
6641
+ if ("files" in archive) throw new ArchiveError("an instance is read from one file or from matching members, not from every file a multi-file format's files selector matches");
6642
+ const unpacked = unpack(bytes, archive);
6643
+ return [parseInstance(named(unpacked.member), unpacked.bytes, parser)];
6644
+ }
6645
+ /**
6646
+ * Unpacks and parses instances as `instancesOf` does, failing at the extract step when the archive cannot be read or its selector matches nothing, and at the parse step when a member does not parse: the members are unpacked one at a time as they are parsed, so the two steps interleave.
6647
+ * @param entryId - The entry.
6648
+ * @param run - The unpacking and parsing.
6649
+ * @returns The instances.
6650
+ * @throws RecipeError naming the entry and the step that failed.
6651
+ */
6652
+ function extractAndParse(entryId, run) {
6653
+ try {
6654
+ return run();
6655
+ } catch (error) {
6656
+ throw new RecipeError(entryId, error instanceof ArchiveError ? "extract" : "parse", error instanceof Error ? error.message : String(error), { cause: error });
6657
+ }
6658
+ }
6659
+ /**
6660
+ * Runs a files recipe: downloads each file in turn (a dead file from its own pinned snapshot), unpacks it and parses it into its instances, so only one file's bytes are held at a time.
6661
+ * @param dataset - The dataset.
6662
+ * @param recipe - Its files recipe.
6663
+ * @param options - The network port, abort signal and clock.
6664
+ * @returns The instances and the provenance of every file.
6665
+ */
6666
+ async function runFiles(dataset, recipe, options) {
6667
+ const { entry } = dataset;
6668
+ const { fetch: port = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
6669
+ const retrievedAt = now().toISOString();
6670
+ const read = async (index) => {
6671
+ const file = recipe.files[index];
6672
+ if (file === void 0) return [];
6673
+ const source = entry.sources[file.source];
6674
+ if (source === void 0) throw new RecipeError(entry.id, "recipe", `file ${file.path} names source ${String(file.source)}, which the entry does not have`);
6675
+ const fetchedUrl = sourceUrl(source);
6676
+ const bytes = await step(entry.id, "download", async () => downloadBytes(port, fetchedUrl, signal));
6677
+ const documents = extractAndParse(entry.id, () => instancesOf(bytes, file.archive, recipe.parser, file.path));
6678
+ return [{
6679
+ documents,
6680
+ download: {
6681
+ ...downloadOf(source, fetchedUrl, bytes),
6682
+ path: file.path,
6683
+ instances: documents.map((document) => document.id)
6684
+ }
6685
+ }, ...await read(index + 1)];
6686
+ };
6687
+ const files = await read(0);
6688
+ const documents = files.flatMap((file) => file.documents);
6689
+ const ids = /* @__PURE__ */ new Set();
6690
+ for (const { id } of documents) {
6691
+ if (ids.has(id)) throw new RecipeError(entry.id, "extract", `two files give the instance ${id}`);
6692
+ ids.add(id);
6693
+ }
6694
+ return {
6695
+ collection: true,
6696
+ documents,
6697
+ provenance: {
6698
+ entryId: entry.id,
6699
+ catalogue: dataset.catalogue,
6700
+ category: entry.category,
6701
+ retrievedAt,
6702
+ files: files.map((file) => file.download)
6703
+ }
6704
+ };
6705
+ }
6706
+ /**
5637
6707
  * Runs a recipe's download, extract and parse steps, and builds the native documents the crosswalk receives.
5638
6708
  * @param dataset - The dataset.
5639
6709
  * @param options - The network port, abort signal and clock.
@@ -5644,51 +6714,28 @@ async function runToNative(dataset, options) {
5644
6714
  const { fetch: fetchPort = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
5645
6715
  const { recipe } = entry;
5646
6716
  if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
6717
+ if ("files" in recipe) return runFiles(dataset, recipe, options);
6718
+ if (recipe.stream === true) throw new RecipeError(entry.id, "recipe", "the table is too large to hold as one document, so its recipe streams: read it with streamDataset, which hands its nodes on a page at a time");
5647
6719
  const source = entry.sources[recipe.source];
5648
6720
  if (source === void 0) throw new RecipeError(entry.id, "recipe", `the recipe names source ${String(recipe.source)}, which the entry does not have`);
5649
- const fetchedUrl = downloadUrl(source);
6721
+ const fetchedUrl = sourceUrl(source);
5650
6722
  const retrievedAt = now().toISOString();
5651
- const bytes = await step(entry.id, "download", async () => {
5652
- if (recipe.paging !== void 0) return downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal);
5653
- const response = await fetchPort(fetchedUrl, signal === void 0 ? {} : { signal });
5654
- if (!response.ok) throw new KeepsetError(`${fetchedUrl} answered ${String(response.status)} ${response.statusText}`);
5655
- return new Uint8Array(await response.arrayBuffer());
5656
- });
6723
+ const bytes = await step(entry.id, "download", async () => recipe.paging === void 0 ? downloadBytes(fetchPort, fetchedUrl, signal) : downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal));
5657
6724
  const provenanceOf = (extra) => ({
5658
6725
  entryId: entry.id,
5659
6726
  catalogue: dataset.catalogue,
5660
6727
  category: entry.category,
5661
- source: {
5662
- url: source.url,
5663
- status: source.status,
5664
- ...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
5665
- },
5666
- fetchedUrl,
5667
6728
  retrievedAt,
5668
- sha256: createHash("sha256").update(bytes).digest("hex"),
5669
- byteLength: bytes.byteLength,
6729
+ ...downloadOf(source, fetchedUrl, bytes),
5670
6730
  ...extra
5671
6731
  });
5672
6732
  const { archive } = recipe;
5673
6733
  if ("members" in archive) {
5674
- const files = await step(entry.id, "extract", () => unpackMembers(bytes, archive));
6734
+ const documents = extractAndParse(entry.id, () => instancesOf(bytes, archive, recipe.parser, void 0));
5675
6735
  return {
5676
6736
  collection: true,
5677
- documents: await step(entry.id, "parse", () => files.map((file) => {
5678
- try {
5679
- const parsed = parseNative(file.bytes, recipe.parser);
5680
- return {
5681
- id: file.path,
5682
- native: {
5683
- ...parsed,
5684
- meta: "meta" in parsed ? parsed.meta : {}
5685
- }
5686
- };
5687
- } catch (error) {
5688
- throw new KeepsetError(`member ${file.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
5689
- }
5690
- })),
5691
- provenance: provenanceOf({ members: files.map((file) => file.path) })
6737
+ documents,
6738
+ provenance: provenanceOf({ members: documents.map((document) => document.id) })
5692
6739
  };
5693
6740
  }
5694
6741
  if ("parts" in archive) {
@@ -5741,7 +6788,7 @@ async function runToNative(dataset, options) {
5741
6788
  * @param dataset - A dataset from any catalogue.
5742
6789
  * @param options - The network port, abort signal and clock.
5743
6790
  * @returns The resolved target with its provenance and any count warnings.
5744
- * @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
6791
+ * @throws RecipeError naming the entry and the failing step, including for a recipe that streams, which `streamEntry` reads; nothing partial or default-shaped is ever returned.
5745
6792
  */
5746
6793
  async function fetchEntry(dataset, options = {}) {
5747
6794
  const { entry, crosswalk } = dataset;
@@ -5782,13 +6829,14 @@ async function fetchDataset(id, options = {}) {
5782
6829
  return fetchEntry(getDataset(id), options);
5783
6830
  }
5784
6831
  /**
5785
- * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
6832
+ * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve. A streamed recipe is checked a page at a time, with the round trip holding on the whole table exactly when it holds on every page, so the table is never held.
5786
6833
  * @param dataset - A dataset from any catalogue.
5787
6834
  * @param options - The network port, abort signal and clock.
5788
6835
  * @returns Whether the round trip holds, and where it first differs when it does not.
5789
6836
  * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
5790
6837
  */
5791
6838
  async function roundTripEntry(dataset, options = {}) {
6839
+ if (isStreamed(dataset.entry.recipe)) return roundTripTable(dataset, options);
5792
6840
  const { documents, provenance } = await runToNative(dataset, options);
5793
6841
  try {
5794
6842
  assertRoundTrip(dataset.crosswalk, documents.map(({ native }) => native));
@@ -5807,6 +6855,28 @@ async function roundTripEntry(dataset, options = {}) {
5807
6855
  };
5808
6856
  }
5809
6857
  /**
6858
+ * Streams a dataset of the bundled catalogue by id, as `streamEntry` does: for a table too large to hold as one document, whose recipe streams.
6859
+ * @param id - The entry id.
6860
+ * @param sink - Where the meta and the nodes go.
6861
+ * @param options - The network port, abort signal and clock.
6862
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
6863
+ * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `streamEntry` does.
6864
+ */
6865
+ async function streamDataset(id, sink, options = {}) {
6866
+ return streamEntry(getDataset(id), sink, options);
6867
+ }
6868
+ /**
6869
+ * Streams a dataset whose table is too large to hold as one document: reads its paged source a page at a time, applies the crosswalk to each page's rows under the document's meta, checks the meta once and each node against the category's node shape and for an id no earlier node had, and hands the nodes on to the sink before reading the next page. What it holds at once is one page's bytes and rows and the ids seen so far (as bytes, not strings), never the table, so the result is the category target `{ meta, nodes }` delivered in order rather than held: the same nodes `fetchEntry` would give, checked by the same rules.
6870
+ * @param dataset - A dataset whose recipe streams.
6871
+ * @param sink - Where the meta and the nodes go.
6872
+ * @param options - The network port, abort signal and clock.
6873
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
6874
+ * @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
6875
+ */
6876
+ async function streamEntry(dataset, sink, options = {}) {
6877
+ return streamTable(dataset, sink, options);
6878
+ }
6879
+ /**
5810
6880
  * Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
5811
6881
  * @param id - The entry id.
5812
6882
  * @param options - The network port, abort signal and clock.
@@ -5817,4 +6887,4 @@ async function roundTripDataset(id, options = {}) {
5817
6887
  return roundTripEntry(getDataset(id), options);
5818
6888
  }
5819
6889
  //#endregion
5820
- export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry };
6890
+ export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry, streamDataset, streamEntry };