keepset 0.38.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { $ as ArchiveError, H as getCollectionTarget, L as getOwn, U as getTarget, at as RecipeError, it as NativeParseError, rt as KeepsetError, t as getDataset, z as countTarget } from "../bundled-U2l4WT5T.js";
1
+ import { J as getTarget, K as getCollectionTarget, U as countTarget, V as getOwn, Y as isStreamable, dt as NativeParseError, ft as RecipeError, ot as ArchiveError, q as getStreamedShape, t as getDataset, ut as KeepsetError } from "../bundled-CL1eRNRR.js";
2
2
  import { RoundTripMismatchError, assertRoundTrip, encode } from "crossmap";
3
3
  import { z } from "zod";
4
4
  import { createHash } from "node:crypto";
@@ -127,6 +127,15 @@ function endpoint$1(fields, column, lineNumber) {
127
127
  return value.trim();
128
128
  }
129
129
  /**
130
+ * Whether a line meets one of the recipe's `where` conditions.
131
+ * @param fields - The line's fields, at least as many as the highest column the recipe names.
132
+ * @param condition - The condition.
133
+ * @returns True when the condition's column, trimmed, equals its value.
134
+ */
135
+ function meets(fields, condition) {
136
+ return fields[condition.column]?.trim() === condition.equals;
137
+ }
138
+ /**
130
139
  * Resolves `detect` to a fixed delimiter from the first data line: tab when it holds a tab, comma when it holds a comma, whitespace otherwise. The rule is a function of the file's bytes, so a recipe that detects still reads the same way every time.
131
140
  * @param dataLine - The first line that is neither skipped, blank nor a comment.
132
141
  * @returns The fixed delimiter.
@@ -149,31 +158,33 @@ function multiplicity(fields, column, lineNumber) {
149
158
  return Number(value);
150
159
  }
151
160
  /**
152
- * Parses a delimited edge list: one edge per line, source and target in the configured columns, after `skipLines` header lines, with blank lines and lines whose first non-blank characters are a comment prefix skipped. A `detect` delimiter is fixed from the first data line. A recipe that names a `keep` column reads only the lines whose field there is one of its values; one that names a `multiplicityColumn` emits as many edges for a line as the field says, none for a zero; one that says `mirrored` emits each pair of mirror-image lines once. Nodes are the endpoints of the lines read, in order of first appearance.
153
- * @param text - The file text.
161
+ * Parses a delimited edge list: one edge per line, source and target in the configured columns, after `skipLines` header lines, with blank lines and lines whose first non-blank characters are a comment prefix skipped. A `detect` delimiter is fixed from the first data line. A recipe that names a `keep` column reads only the lines whose field there is one of its values, and one that states `where` reads only the lines whose listed columns, trimmed, equal their values, so a file that records several kinds of row (an interaction file listing genetic beside physical interactions) is read for the rows the dataset is made of; a line that is not read is still checked for the fields it needs. A recipe that names a `multiplicityColumn` emits as many edges for a line as the field says, none for a zero; one that says `mirrored` emits each pair of mirror-image lines once. Nodes are the endpoints of the lines read, in order of first appearance. The text is read line by line, so a file longer than the engine's longest string is read too.
162
+ * @param text - The file text, whole or in pieces.
154
163
  * @param options - The recipe's edge-list parser.
155
164
  * @returns The native graph.
156
- * @throws NativeParseError naming the line when a data line has too few fields, an empty endpoint or a multiplicity that is not a whole number, when a mirrored table has a line without its mirror, or when the file holds no edges.
165
+ * @throws NativeParseError naming the line when a data line has too few fields, an empty endpoint or a multiplicity that is not a whole number, when a mirrored table has a line without its mirror, or when no line is an edge.
157
166
  */
158
167
  function parseEdgeList(text, options) {
159
168
  let split = options.delimiter === "detect" ? void 0 : splitters[options.delimiter];
160
- const needed = Math.max(options.sourceColumn, options.targetColumn, options.keep?.column ?? 0, options.multiplicityColumn ?? 0) + 1;
169
+ const { where } = options;
170
+ const needed = Math.max(options.sourceColumn, options.targetColumn, options.keep?.column ?? 0, options.multiplicityColumn ?? 0, ...where === void 0 ? [] : where.map((condition) => condition.column)) + 1;
161
171
  const order = new NodeOrder();
162
172
  const edges = [];
163
173
  const awaiting = /* @__PURE__ */ new Map();
164
- const all = lines(text);
165
- for (const [index, line] of all.entries()) {
166
- const lineNumber = index + 1;
174
+ let lastLine = 0;
175
+ forEachLine(typeof text === "string" ? [text] : text, (line, lineNumber) => {
176
+ lastLine = lineNumber;
167
177
  const content = line.trimStart();
168
- if (index < options.skipLines || content === "" || options.commentPrefixes.some((prefix) => content.startsWith(prefix))) continue;
178
+ if (lineNumber <= options.skipLines || content === "" || options.commentPrefixes.some((prefix) => content.startsWith(prefix))) return;
169
179
  split ??= splitters[detect(line)];
170
180
  const fields = split(line);
171
181
  if (fields.length < needed) throw new NativeParseError("edge-list", lineNumber, `expected at least ${String(needed)} fields, found ${String(fields.length)}`);
172
- if (options.keep !== void 0 && !options.keep.values.includes(fields[options.keep.column]?.trim() ?? "")) continue;
182
+ if (options.keep !== void 0 && !options.keep.values.includes(fields[options.keep.column]?.trim() ?? "")) return;
183
+ if (where !== void 0 && !where.every((condition) => meets(fields, condition))) return;
173
184
  const source = endpoint$1(fields, options.sourceColumn, lineNumber);
174
185
  const target = endpoint$1(fields, options.targetColumn, lineNumber);
175
186
  const count = options.multiplicityColumn === void 0 ? 1 : multiplicity(fields, options.multiplicityColumn, lineNumber);
176
- if (count === 0) continue;
187
+ if (count === 0) return;
177
188
  order.add(source);
178
189
  order.add(target);
179
190
  if (options.mirrored === true && source !== target) {
@@ -185,7 +196,7 @@ function parseEdgeList(text, options) {
185
196
  const waiting = awaiting.get(key);
186
197
  if (waiting !== void 0 && waiting.length > 0) {
187
198
  waiting.shift();
188
- continue;
199
+ return;
189
200
  }
190
201
  const mirror = JSON.stringify([
191
202
  target,
@@ -198,12 +209,12 @@ function parseEdgeList(text, options) {
198
209
  source,
199
210
  target
200
211
  });
201
- }
212
+ });
202
213
  for (const [key, waiting] of awaiting) {
203
214
  const [first] = waiting;
204
215
  if (first !== void 0) throw new NativeParseError("edge-list", first, `the table is mirrored but this line has no mirror image (${key} is never listed)`);
205
216
  }
206
- if (edges.length === 0) throw new NativeParseError("edge-list", all.length, "the file holds no edges");
217
+ if (edges.length === 0) throw new NativeParseError("edge-list", lastLine, where === void 0 ? "the file holds no edges" : "no line of the file meets the recipe's where conditions");
207
218
  return {
208
219
  nodes: order.nodes(),
209
220
  edges
@@ -211,8 +222,8 @@ function parseEdgeList(text, options) {
211
222
  }
212
223
  //#endregion
213
224
  //#region src/parsers/gd-contest.ts
214
- const FORMAT$14 = "gd-contest";
215
- const NUMBER$9 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
225
+ const FORMAT$18 = "gd-contest";
226
+ const NUMBER$10 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
216
227
  /**
217
228
  * Reads a vertex number, which the format numbers from 0 to N - 1.
218
229
  * @param token - The token.
@@ -220,7 +231,7 @@ const NUMBER$9 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
220
231
  * @returns The vertex id as text.
221
232
  */
222
233
  function vertex$1(token, count) {
223
- if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$14, token.line, `expected a vertex number from 0 to ${String(count - 1)}, found ${JSON.stringify(token.text)}`);
234
+ if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$18, token.line, `expected a vertex number from 0 to ${String(count - 1)}, found ${JSON.stringify(token.text)}`);
224
235
  return token.text;
225
236
  }
226
237
  /**
@@ -240,17 +251,17 @@ function parseGdContest(text) {
240
251
  });
241
252
  }
242
253
  const [first] = tokens;
243
- if (first === void 0) throw new NativeParseError(FORMAT$14, 1, "the file holds no vertex count");
244
- if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$14, first.line, `the vertex count must be a non-negative integer, found ${JSON.stringify(first.text)}`);
254
+ if (first === void 0) throw new NativeParseError(FORMAT$18, 1, "the file holds no vertex count");
255
+ if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$18, first.line, `the vertex count must be a non-negative integer, found ${JSON.stringify(first.text)}`);
245
256
  const count = Number(first.text);
246
257
  const coordinates = tokens.slice(1, 1 + 2 * count);
247
- if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$14, lines(text).length, `expected ${String(2 * count)} coordinates for ${String(count)} vertices, found ${String(coordinates.length)}`);
248
- for (const coordinate of coordinates) if (!NUMBER$9.test(coordinate.text)) throw new NativeParseError(FORMAT$14, coordinate.line, `coordinate ${JSON.stringify(coordinate.text)} is not a number`);
258
+ if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$18, lines(text).length, `expected ${String(2 * count)} coordinates for ${String(count)} vertices, found ${String(coordinates.length)}`);
259
+ for (const coordinate of coordinates) if (!NUMBER$10.test(coordinate.text)) throw new NativeParseError(FORMAT$18, coordinate.line, `coordinate ${JSON.stringify(coordinate.text)} is not a number`);
249
260
  const ends = tokens.slice(1 + 2 * count);
250
261
  const edges = [];
251
262
  for (let source = ends.shift(); source !== void 0; source = ends.shift()) {
252
263
  const target = ends.shift();
253
- if (target === void 0) throw new NativeParseError(FORMAT$14, source.line, `edge from vertex ${source.text} has no second vertex`);
264
+ if (target === void 0) throw new NativeParseError(FORMAT$18, source.line, `edge from vertex ${source.text} has no second vertex`);
254
265
  edges.push({
255
266
  source: vertex$1(source, count),
256
267
  target: vertex$1(target, count)
@@ -264,7 +275,7 @@ function parseGdContest(text) {
264
275
  //#endregion
265
276
  //#region src/parsers/gml.ts
266
277
  const KEY = /[A-Za-z_][A-Za-z0-9_]*/y;
267
- const NUMBER$8 = /[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?/y;
278
+ const NUMBER$9 = /[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?/y;
268
279
  /**
269
280
  * Matches a sticky pattern at a position.
270
281
  * @param pattern - A sticky regular expression.
@@ -315,7 +326,7 @@ function tokenise(text) {
315
326
  position = end + 1;
316
327
  } else {
317
328
  const key = matchAt(KEY, text, position);
318
- const number = key === void 0 ? matchAt(NUMBER$8, text, position) : void 0;
329
+ const number = key === void 0 ? matchAt(NUMBER$9, text, position) : void 0;
319
330
  const match = key ?? number;
320
331
  if (match === void 0) throw new NativeParseError("gml", line, `unexpected character ${JSON.stringify(char)}`);
321
332
  tokens.push({
@@ -420,7 +431,7 @@ function parseGml(text) {
420
431
  }
421
432
  //#endregion
422
433
  //#region src/parsers/hgb.ts
423
- const FORMAT$13 = "hgb";
434
+ const FORMAT$17 = "hgb";
424
435
  /** A type code or node id as the HGB files write them: a non-negative decimal integer with no sign or leading zero. */
425
436
  const CODE = /^(?:0|[1-9]\d*)$/;
426
437
  /** A number as a decimal literal with an optional sign and exponent, so `Number` cannot read hex, an empty string or infinity. */
@@ -440,7 +451,7 @@ const LINK_FIELDS = 4;
440
451
  */
441
452
  function resolve(field, types, where) {
442
453
  const type = types[CODE.test(field) ? Number(field) : -1];
443
- if (type === void 0) throw new NativeParseError(FORMAT$13, where.line, `${where.file}: the ${where.kind} type ${JSON.stringify(field)} is not one of the ${String(types.length)} the recipe names`);
454
+ if (type === void 0) throw new NativeParseError(FORMAT$17, where.line, `${where.file}: the ${where.kind} type ${JSON.stringify(field)} is not one of the ${String(types.length)} the recipe names`);
444
455
  return type;
445
456
  }
446
457
  /**
@@ -458,9 +469,9 @@ function parseHgb(parts, options) {
458
469
  forEachLine(parts.nodes, (line, number) => {
459
470
  const fields = line.split(" ");
460
471
  const [id, , type] = fields;
461
- if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$13, number, `node.dat: expected ${String(NODE_FIELDS)} or ${String(NODE_FIELDS_WITH_ATTRIBUTES)} fields separated by tabs, found ${String(fields.length)}`);
462
- if (!CODE.test(id)) throw new NativeParseError(FORMAT$13, number, `node.dat: the node id ${JSON.stringify(id)} is not a non-negative integer`);
463
- if (typeOf.has(id)) throw new NativeParseError(FORMAT$13, number, `node.dat: the node id ${id} is repeated`);
472
+ if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$17, number, `node.dat: expected ${String(NODE_FIELDS)} or ${String(NODE_FIELDS_WITH_ATTRIBUTES)} fields separated by tabs, found ${String(fields.length)}`);
473
+ if (!CODE.test(id)) throw new NativeParseError(FORMAT$17, number, `node.dat: the node id ${JSON.stringify(id)} is not a non-negative integer`);
474
+ if (typeOf.has(id)) throw new NativeParseError(FORMAT$17, number, `node.dat: the node id ${id} is repeated`);
464
475
  const nodeType = resolve(type, options.nodeTypes, {
465
476
  file: "node.dat",
466
477
  line: number,
@@ -472,12 +483,12 @@ function parseHgb(parts, options) {
472
483
  type: nodeType
473
484
  });
474
485
  });
475
- if (nodes.length === 0) throw new NativeParseError(FORMAT$13, 1, "node.dat: the file holds no nodes");
486
+ if (nodes.length === 0) throw new NativeParseError(FORMAT$17, 1, "node.dat: the file holds no nodes");
476
487
  const edges = [];
477
488
  forEachLine(parts.links, (line, number) => {
478
489
  const fields = line.split(" ");
479
490
  const [source, target, type, weight] = fields;
480
- if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$13, number, `link.dat: expected ${String(LINK_FIELDS)} fields separated by tabs, found ${String(fields.length)}`);
491
+ if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$17, number, `link.dat: expected ${String(LINK_FIELDS)} fields separated by tabs, found ${String(fields.length)}`);
481
492
  const linkType = resolve(type, options.linkTypes, {
482
493
  file: "link.dat",
483
494
  line: number,
@@ -493,17 +504,17 @@ function parseHgb(parts, options) {
493
504
  linkType.target
494
505
  ]]) {
495
506
  const actual = typeOf.get(id);
496
- if (actual === void 0) throw new NativeParseError(FORMAT$13, number, `link.dat: the ${end} ${JSON.stringify(id)} is not a node of node.dat`);
497
- if (actual !== expected) throw new NativeParseError(FORMAT$13, number, `link.dat: a ${linkType.name} link's ${end} must be a ${expected} node, and ${id} is a ${actual} node`);
507
+ if (actual === void 0) throw new NativeParseError(FORMAT$17, number, `link.dat: the ${end} ${JSON.stringify(id)} is not a node of node.dat`);
508
+ if (actual !== expected) throw new NativeParseError(FORMAT$17, number, `link.dat: a ${linkType.name} link's ${end} must be a ${expected} node, and ${id} is a ${actual} node`);
498
509
  }
499
- if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$13, number, `link.dat: the weight ${JSON.stringify(weight)} is not 1; a weighted HGB file would lose its weights`);
510
+ if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$17, number, `link.dat: the weight ${JSON.stringify(weight)} is not 1; a weighted HGB file would lose its weights`);
500
511
  edges.push({
501
512
  source,
502
513
  target,
503
514
  type: linkType.name
504
515
  });
505
516
  });
506
- if (edges.length === 0) throw new NativeParseError(FORMAT$13, 1, "link.dat: the file holds no links");
517
+ if (edges.length === 0) throw new NativeParseError(FORMAT$17, 1, "link.dat: the file holds no links");
507
518
  return {
508
519
  nodes,
509
520
  edges
@@ -511,7 +522,7 @@ function parseHgb(parts, options) {
511
522
  }
512
523
  //#endregion
513
524
  //#region src/parsers/kg-triples.ts
514
- const FORMAT$12 = "kg-triples";
525
+ const FORMAT$16 = "kg-triples";
515
526
  /** The character each delimiter option splits a line on. A line is split on exactly that character, so a field cannot hide a second delimiter. */
516
527
  const DELIMITERS$1 = {
517
528
  tab: " ",
@@ -533,8 +544,8 @@ const QUOTE$2 = "\"";
533
544
  */
534
545
  function value(field, quoted, where) {
535
546
  const inner = quoted ? field.slice(1, -1) : field;
536
- if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$12, where.line, `${where.part}the field ${JSON.stringify(field)} is not one value wrapped in double quotes`);
537
- if (inner === "") throw new NativeParseError(FORMAT$12, where.line, `${where.part}a field is empty`);
547
+ if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$16, where.line, `${where.part}the field ${JSON.stringify(field)} is not one value wrapped in double quotes`);
548
+ if (inner === "") throw new NativeParseError(FORMAT$16, where.line, `${where.part}a field is empty`);
538
549
  return inner;
539
550
  }
540
551
  /**
@@ -558,16 +569,16 @@ function parseKgTriples(parts, options) {
558
569
  let lastLine = 0;
559
570
  forEachLine(part.pieces, (line, number) => {
560
571
  lastLine = number;
561
- if (line.trim() === "") throw new NativeParseError(FORMAT$12, number, `${label}the line is blank`);
572
+ if (line.trim() === "") throw new NativeParseError(FORMAT$16, number, `${label}the line is blank`);
562
573
  if (number === 1 && options.header === "names") return;
563
574
  if (number === 1 && options.header === "count") {
564
- if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$12, number, `${label}expected the number of triples on the first line, found ${JSON.stringify(line)}`);
575
+ if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$16, number, `${label}expected the number of triples on the first line, found ${JSON.stringify(line)}`);
565
576
  declared = Number(line);
566
577
  return;
567
578
  }
568
579
  const fields = line.split(delimiter);
569
580
  const [head, middle, last] = fields;
570
- if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$12, number, `${label}expected ${String(TRIPLE_FIELDS)} fields separated by ${options.delimiter}, found ${String(fields.length)}`);
581
+ if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$16, number, `${label}expected ${String(TRIPLE_FIELDS)} fields separated by ${options.delimiter}, found ${String(fields.length)}`);
571
582
  const where = {
572
583
  line: number,
573
584
  part: label
@@ -593,8 +604,8 @@ function parseKgTriples(parts, options) {
593
604
  });
594
605
  triples += 1;
595
606
  });
596
- if (triples === 0) throw new NativeParseError(FORMAT$12, lastLine, `${label}the file holds no triples`);
597
- if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$12, lastLine, `${label}the first line declares ${String(declared)} triples, the file holds ${String(triples)}`);
607
+ if (triples === 0) throw new NativeParseError(FORMAT$16, lastLine, `${label}the file holds no triples`);
608
+ if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$16, lastLine, `${label}the first line declares ${String(declared)} triples, the file holds ${String(triples)}`);
598
609
  }
599
610
  return {
600
611
  nodes: order.nodes(),
@@ -603,7 +614,7 @@ function parseKgTriples(parts, options) {
603
614
  }
604
615
  //#endregion
605
616
  //#region src/parsers/konect.ts
606
- const FORMAT$11 = "konect";
617
+ const FORMAT$15 = "konect";
607
618
  /** The network formats of KONECT's first line (Kunegis, "The KONECT Handbook", table 1): undirected, directed and bipartite. */
608
619
  const NETWORK_FORMATS = /* @__PURE__ */ new Set([
609
620
  "sym",
@@ -678,9 +689,9 @@ function parseKonect(pieces, options) {
678
689
  }));
679
690
  });
680
691
  const { header } = state;
681
- if (header === void 0) throw new NativeParseError(FORMAT$11, lastLine, "the file has no first line naming its format and weights");
682
- if (edges.length === 0) throw new NativeParseError(FORMAT$11, lastLine, "the file holds no edges");
683
- if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$11, lastLine, `the second line declares ${String(header.counts.edges)} edges, the file holds ${String(edges.length)}`);
692
+ if (header === void 0) throw new NativeParseError(FORMAT$15, lastLine, "the file has no first line naming its format and weights");
693
+ if (edges.length === 0) throw new NativeParseError(FORMAT$15, lastLine, "the file holds no edges");
694
+ if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$15, lastLine, `the second line declares ${String(header.counts.edges)} edges, the file holds ${String(edges.length)}`);
684
695
  return {
685
696
  nodes: order.nodes(),
686
697
  edges
@@ -698,9 +709,9 @@ function readHeader(content, number, options) {
698
709
  const match = FORMAT_LINE.exec(content);
699
710
  const network = match?.[1];
700
711
  const weights = match?.[2];
701
- if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$11, number, `expected a first line "% FORMAT WEIGHTS" with FORMAT one of ${[...NETWORK_FORMATS].join(", ")}, found ${JSON.stringify(content)}`);
702
- if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$11, number, `the weight type ${JSON.stringify(weights)} is not one this parser reads (its third column is not an edge weight or a repetition count)`);
703
- if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$11, number, `the file's weight type is ${weights}, so a recipe with weighted ${String(WEIGHTED_TYPES.has(weights))} reads it, not one with weighted ${String(options.weighted)}`);
712
+ if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$15, number, `expected a first line "% FORMAT WEIGHTS" with FORMAT one of ${[...NETWORK_FORMATS].join(", ")}, found ${JSON.stringify(content)}`);
713
+ if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$15, number, `the weight type ${JSON.stringify(weights)} is not one this parser reads (its third column is not an edge weight or a repetition count)`);
714
+ if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$15, number, `the file's weight type is ${weights}, so a recipe with weighted ${String(WEIGHTED_TYPES.has(weights))} reads it, not one with weighted ${String(options.weighted)}`);
704
715
  return {
705
716
  line: number,
706
717
  network
@@ -720,15 +731,15 @@ function readEdge(content, number, context) {
720
731
  const [from, to, third, time] = fields;
721
732
  const expected = options.timestamped ? TIMESTAMPED_COLUMNS : options.weighted ? WEIGHTED_COLUMNS : void 0;
722
733
  const fits = expected === void 0 ? fields.length === ENDPOINT_COLUMNS || fields.length === WEIGHTED_COLUMNS : fields.length === expected;
723
- if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$11, number, `expected ${expected === void 0 ? `${String(ENDPOINT_COLUMNS)} or ${String(WEIGHTED_COLUMNS)}` : String(expected)} columns for this recipe, found ${String(fields.length)}`);
734
+ if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$15, number, `expected ${expected === void 0 ? `${String(ENDPOINT_COLUMNS)} or ${String(WEIGHTED_COLUMNS)}` : String(expected)} columns for this recipe, found ${String(fields.length)}`);
724
735
  const bipartite = header.network === "bip";
725
736
  const source = endpoint(from, bipartite ? LEFT_PREFIX : "", header.counts?.left, number);
726
737
  const target = endpoint(to, bipartite ? RIGHT_PREFIX : "", header.counts?.right, number);
727
738
  order.add(source);
728
739
  order.add(target);
729
- if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$11, number, `the third column ${JSON.stringify(third)} is not 1; an edge that stands for several would be read as one`);
730
- if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$11, number, `the weight ${JSON.stringify(third)} is not a number`);
731
- if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$11, number, `the time ${JSON.stringify(time)} is not a number`);
740
+ if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$15, number, `the third column ${JSON.stringify(third)} is not 1; an edge that stands for several would be read as one`);
741
+ if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$15, number, `the weight ${JSON.stringify(third)} is not a number`);
742
+ if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$15, number, `the time ${JSON.stringify(time)} is not a number`);
732
743
  const edge = {
733
744
  source,
734
745
  target
@@ -747,10 +758,215 @@ function readEdge(content, number, context) {
747
758
  * @throws NativeParseError when the token is not a positive integer or exceeds the set's size.
748
759
  */
749
760
  function endpoint(token, prefix, size, number) {
750
- if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$11, number, `${JSON.stringify(token)} is not a node id from 1${size === void 0 ? "" : ` to ${String(size)}`}`);
761
+ if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$15, number, `${JSON.stringify(token)} is not a node id from 1${size === void 0 ? "" : ` to ${String(size)}`}`);
751
762
  return `${prefix}${token}`;
752
763
  }
753
764
  //#endregion
765
+ //#region src/parsers/ogb.ts
766
+ const FORMAT$14 = "ogb";
767
+ /** The files of an OGB raw directory this reader takes, by their path under the dataset's root directory. A relation's files sit in `raw/relations/<head>___<relation>___<tail>/`. */
768
+ const RAW = /^(?<root>(?:[^/]+\/)*?)raw\/(?<name>num-node-list|num-node-dict|edge|num-edge-list|triplet-type-list|relations\/[^/]+\/(?:edge|num-edge-list))\.csv\.gz$/u;
769
+ const INDEX = /^(?:0|[1-9]\d*)$/;
770
+ /** The fields of a line of `triplet-type-list.csv.gz`: a head type, a relation and a tail type. */
771
+ const TRIPLET_PARTS = 3;
772
+ /**
773
+ * Orders two names by code point, the order Python's `sorted` gives the ASCII type and relation names OGB uses.
774
+ * @param left - A name.
775
+ * @param right - Another.
776
+ * @returns Negative, zero or positive.
777
+ */
778
+ function compare(left, right) {
779
+ return left < right ? -1 : left > right ? 1 : 0;
780
+ }
781
+ /**
782
+ * Visits the non-blank lines of a file, read a piece at a time.
783
+ * @param file - The file.
784
+ * @param visit - Called with each non-blank line and its number.
785
+ */
786
+ function forEachRow(file, visit) {
787
+ forEachLine(file.text(), (text, line) => {
788
+ if (text.trim() !== "") visit({
789
+ text,
790
+ line
791
+ });
792
+ });
793
+ }
794
+ /**
795
+ * The non-blank lines of a small file, such as a count or the triplet list.
796
+ * @param file - The file.
797
+ * @returns Each non-blank line and its number.
798
+ */
799
+ function rows(file) {
800
+ const all = [];
801
+ forEachRow(file, (row) => {
802
+ all.push(row);
803
+ });
804
+ return all;
805
+ }
806
+ /**
807
+ * Reads a file that holds one line, such as a count.
808
+ * @param file - The file.
809
+ * @returns The line's text.
810
+ * @throws NativeParseError when the file holds no line or more than one.
811
+ */
812
+ function onlyLine(file) {
813
+ const all = rows(file);
814
+ const [only] = all;
815
+ if (all.length !== 1 || only === void 0) throw new NativeParseError(FORMAT$14, all[1]?.line ?? 1, `${file.path} must hold one line, found ${String(all.length)}`);
816
+ return only.text.trim();
817
+ }
818
+ /**
819
+ * Reads a non-negative integer.
820
+ * @param text - The field.
821
+ * @param file - The file, for the error.
822
+ * @param line - The 1-based line, for the error.
823
+ * @returns The integer.
824
+ * @throws NativeParseError when the field is not a non-negative integer.
825
+ */
826
+ function index(text, file, line) {
827
+ const trimmed = text.trim();
828
+ if (!INDEX.test(trimmed)) throw new NativeParseError(FORMAT$14, line, `${file.path}: expected a non-negative integer, found ${JSON.stringify(text)}`);
829
+ return Number(trimmed);
830
+ }
831
+ /**
832
+ * Reads an `edge.csv.gz` file, one `<source>,<target>` pair of node indices per line, checking each index against its node type's count and the line count against the relation's `num-edge-list.csv.gz`.
833
+ * @param edges - The edge file.
834
+ * @param count - The relation's `num-edge-list.csv.gz`.
835
+ * @param ends - The source and target node types' counts and how each index is named.
836
+ * @param emit - Called with each edge, in file order, so a large file is never held twice.
837
+ * @throws NativeParseError naming the file and line for a line that is not a pair of indices, an index outside its type's nodes, or a line count other than the recorded one.
838
+ */
839
+ function readEdges(edges, count, ends, emit) {
840
+ const recorded = index(onlyLine(count), count, 1);
841
+ let read = 0;
842
+ forEachRow(edges, ({ text, line }) => {
843
+ const fields = text.split(",");
844
+ const [first, second] = fields;
845
+ if (fields.length !== 2 || first === void 0 || second === void 0) throw new NativeParseError(FORMAT$14, line, `${edges.path}: expected a source and a target index, found ${JSON.stringify(text)}`);
846
+ const source = index(first, edges, line);
847
+ const target = index(second, edges, line);
848
+ if (source >= ends.sources || target >= ends.targets) throw new NativeParseError(FORMAT$14, line, `${edges.path}: edge ${text} names a node outside the ${String(ends.sources)} sources and ${String(ends.targets)} targets the dataset records`);
849
+ emit({
850
+ source: ends.source(source),
851
+ target: ends.target(target)
852
+ });
853
+ read += 1;
854
+ });
855
+ if (read !== recorded) throw new NativeParseError(FORMAT$14, read, `${edges.path} holds ${String(read)} edges, but ${count.path} records ${String(recorded)}`);
856
+ }
857
+ /**
858
+ * Takes one file of the dataset directory by its name under `raw/`.
859
+ * @param files - The files by name.
860
+ * @param name - The name, such as `edge` or `relations/paper___cites___paper/edge`.
861
+ * @param root - The dataset's root directory, for the error.
862
+ * @returns The file.
863
+ * @throws NativeParseError when the archive's selection holds no such file.
864
+ */
865
+ function required$1(files, name, root) {
866
+ const file = files.get(name);
867
+ if (file === void 0) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has no raw/${name}.csv.gz among the files the recipe selects`);
868
+ return file;
869
+ }
870
+ /**
871
+ * Parses an Open Graph Benchmark dataset from the files of its raw directory, as OGB's own `read_csv_graph_raw` and `read_csv_heterograph_raw` read them (ogb/io/read_graph_raw.py in snap-stanford/ogb). A homogeneous dataset has `raw/num-node-list.csv.gz` (its one graph's node count), `raw/edge.csv.gz` (one `<source>,<target>` pair of node indices per line, no header) and `raw/num-edge-list.csv.gz`: its nodes are `0` to `N - 1`, isolated ones included, and its edges the pairs in file order. A heterogeneous dataset has `raw/num-node-dict.csv.gz` (a header of node types over one row of their counts), `raw/triplet-type-list.csv.gz` (one `<head>,<relation>,<tail>` type triplet per line) and, per triplet, `raw/relations/<head>___<relation>___<tail>/edge.csv.gz` and `num-edge-list.csv.gz`, whose indices count within the head and tail types: its nodes are `<type>:<index>` for each type in sorted order and each index from 0, and its edges are each relation's pairs in file order, relations in the sorted order of their triplets, which is the order OGB reads them in. Edges keep the file's direction. Relation names, node features, labels and splits are not carried, as no graph parser in this version carries more than topology.
872
+ * @param files - The files the recipe's archive selects, gunzipped and decoded: every one must be a file of one dataset's raw directory named above.
873
+ * @returns The native graph.
874
+ * @throws NativeParseError naming the file and line for a file outside the raw layout, files of two datasets, a dataset with neither or both node-count files, a missing file, a graph count other than one, a malformed count or triplet, a relation directory the triplet list does not name, an edge that is not a pair of indices of the dataset's nodes, or an edge file whose line count differs from its recorded count.
875
+ */
876
+ function parseOgb(files) {
877
+ const byName = /* @__PURE__ */ new Map();
878
+ const roots = /* @__PURE__ */ new Set();
879
+ for (const file of files) {
880
+ const match = RAW.exec(file.path);
881
+ const name = match?.groups?.["name"];
882
+ const root = match?.groups?.["root"];
883
+ if (name === void 0 || root === void 0) throw new NativeParseError(FORMAT$14, 1, `${file.path} is not a file of an OGB raw directory this reader takes`);
884
+ roots.add(root);
885
+ byName.set(name, file);
886
+ }
887
+ const [root] = roots;
888
+ if (roots.size !== 1 || root === void 0) throw new NativeParseError(FORMAT$14, 1, `the recipe must select the raw files of one dataset, found ${roots.size === 0 ? "none" : `the directories ${[...roots].join(", ")}`}`);
889
+ const homogeneous = byName.get("num-node-list");
890
+ const heterogeneous = byName.get("num-node-dict");
891
+ const neitherOrBoth = `the dataset directory ${root} must have exactly one of raw/num-node-list.csv.gz (one node type) and raw/num-node-dict.csv.gz (several)`;
892
+ const edges = [];
893
+ if (homogeneous !== void 0) {
894
+ if (heterogeneous !== void 0) throw new NativeParseError(FORMAT$14, 1, neitherOrBoth);
895
+ const [relation] = [...byName.keys()].filter((name) => name.startsWith("relations/"));
896
+ if (relation !== void 0 || byName.has("triplet-type-list")) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has raw/num-node-list.csv.gz, so it has one node type, but the recipe selects ${relation === void 0 ? "raw/triplet-type-list.csv.gz" : `raw/${relation}.csv.gz`} as well`);
897
+ const count = index(onlyLine(homogeneous), homogeneous, 1);
898
+ const name = (node) => String(node);
899
+ readEdges(required$1(byName, "edge", root), required$1(byName, "num-edge-list", root), {
900
+ sources: count,
901
+ targets: count,
902
+ source: name,
903
+ target: name
904
+ }, (edge) => {
905
+ edges.push(edge);
906
+ });
907
+ return {
908
+ nodes: Array.from({ length: count }, (_, node) => ({ id: String(node) })),
909
+ edges
910
+ };
911
+ }
912
+ if (heterogeneous === void 0) throw new NativeParseError(FORMAT$14, 1, neitherOrBoth);
913
+ for (const stray of ["edge", "num-edge-list"]) if (byName.has(stray)) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has raw/num-node-dict.csv.gz, so its edges are in raw/relations/, but the recipe selects raw/${stray}.csv.gz as well`);
914
+ const header = rows(heterogeneous);
915
+ const [types, counts] = header;
916
+ if (header.length !== 2 || types === void 0 || counts === void 0) throw new NativeParseError(FORMAT$14, header[2]?.line ?? 1, `${heterogeneous.path} must hold a header of node types and one row of their counts, found ${String(header.length)} lines`);
917
+ const typeNames = types.text.split(",").map((type) => type.trim());
918
+ const typeCounts = counts.text.split(",");
919
+ if (typeNames.length !== typeCounts.length || typeNames.some((type) => type === "") || new Set(typeNames).size !== typeNames.length) throw new NativeParseError(FORMAT$14, counts.line, `${heterogeneous.path} must name each node type once, over one count per type`);
920
+ const nodeTypes = typeNames.map((type, position) => {
921
+ const count = typeCounts[position];
922
+ if (count === void 0) throw new NativeParseError(FORMAT$14, counts.line, `${heterogeneous.path} gives node type ${type} no count`);
923
+ return {
924
+ type,
925
+ count: index(count, heterogeneous, counts.line)
926
+ };
927
+ });
928
+ nodeTypes.sort((left, right) => compare(left.type, right.type));
929
+ const nodeCounts = new Map(nodeTypes.map(({ type, count }) => [type, count]));
930
+ const triplets = required$1(byName, "triplet-type-list", root);
931
+ const relations = [];
932
+ for (const { text, line } of rows(triplets)) {
933
+ const parts = text.split(",").map((part) => part.trim());
934
+ const [head, relation, tail] = parts;
935
+ if (parts.length !== TRIPLET_PARTS || head === void 0 || relation === void 0 || tail === void 0 || relation === "") throw new NativeParseError(FORMAT$14, line, `${triplets.path}: expected a head type, a relation and a tail type, found ${JSON.stringify(text)}`);
936
+ const sources = nodeCounts.get(head);
937
+ const targets = nodeCounts.get(tail);
938
+ if (sources === void 0 || targets === void 0) throw new NativeParseError(FORMAT$14, line, `${triplets.path}: ${text} names a node type that ${heterogeneous.path} does not count`);
939
+ relations.push({
940
+ head,
941
+ relation,
942
+ tail,
943
+ sources,
944
+ targets
945
+ });
946
+ }
947
+ relations.sort((left, right) => compare(left.head, right.head) || compare(left.relation, right.relation) || compare(left.tail, right.tail));
948
+ const listed = new Set(relations.map(({ head, relation, tail }) => `relations/${head}___${relation}___${tail}`));
949
+ for (const name of byName.keys()) {
950
+ const directory = name.slice(0, name.lastIndexOf("/"));
951
+ if (name.startsWith("relations/") && !listed.has(directory)) throw new NativeParseError(FORMAT$14, 1, `raw/${directory}/ is not one of the relations ${triplets.path} lists`);
952
+ }
953
+ for (const { head, relation, tail, sources, targets } of relations) {
954
+ const directory = `relations/${head}___${relation}___${tail}`;
955
+ readEdges(required$1(byName, `${directory}/edge`, root), required$1(byName, `${directory}/num-edge-list`, root), {
956
+ sources,
957
+ targets,
958
+ source: (node) => `${head}:${String(node)}`,
959
+ target: (node) => `${tail}:${String(node)}`
960
+ }, (edge) => {
961
+ edges.push(edge);
962
+ });
963
+ }
964
+ return {
965
+ nodes: nodeTypes.flatMap(({ type, count }) => Array.from({ length: count }, (_, node) => ({ id: `${type}:${String(node)}` }))),
966
+ edges
967
+ };
968
+ }
969
+ //#endregion
754
970
  //#region src/parsers/matrix-market.ts
755
971
  /** How many tokens an entry line of each field type holds: the coordinates, then the value's parts (none for a pattern, a real and an imaginary part for a complex value). */
756
972
  const tokensPerEntry = /* @__PURE__ */ new Map([
@@ -985,69 +1201,71 @@ var RecordScanner = class {
985
1201
  }
986
1202
  };
987
1203
  /**
988
- * Splits text into records and fields by RFC 4180 as the text arrives in pieces, one piece's records at a time, so neither the whole file nor every cell is held at once: a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR. A piece may end anywhere, even between the two quotes of a doubled quote or between a CR and its LF.
989
- * @param pieces - The file text in order, after the lines the recipe skips.
990
- * @param delimiter - The field delimiter.
991
- * @param firstLine - The 1-based line of the file the text starts on.
992
- * @returns The records in order, blank lines included as a record of one empty field.
993
- * @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
994
- */
995
- function* records$1(pieces, delimiter, firstLine) {
996
- const scanner = new RecordScanner(delimiter.charCodeAt(0), firstLine);
997
- let carry = "";
998
- for (const piece of pieces) {
999
- const scanned = scanner.scan(carry + piece, false);
1000
- carry = scanned.carry;
1001
- yield* scanned.completed;
1204
+ * Reads a table file's records as its text arrives in pieces, so neither the whole file nor every cell is held at once: a leading byte order mark is removed, the physical lines the recipe skips are dropped, and the rest is split into records and fields by RFC 4180 (a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR). Blank lines are not records. A piece may end anywhere, even between the two quotes of a doubled quote or between a CR and its LF.
1205
+ */
1206
+ var RecordReader = class {
1207
+ options;
1208
+ scanner;
1209
+ carry = "";
1210
+ first = true;
1211
+ toSkip;
1212
+ constructor(options) {
1213
+ this.options = options;
1214
+ this.scanner = new RecordScanner(DELIMITERS[options.delimiter].charCodeAt(0), options.skipLines + 1);
1215
+ this.toSkip = options.skipLines;
1002
1216
  }
1003
- yield* scanner.scan(carry, true).completed;
1004
- const last = scanner.finish();
1005
- if (last !== void 0) yield last;
1006
- }
1217
+ /**
1218
+ * Reads the next piece of the file's text.
1219
+ * @param piece - The text, which follows the pieces read before it.
1220
+ * @returns The records the piece completes, in order.
1221
+ * @throws NativeParseError when a quote opens inside an unquoted field.
1222
+ */
1223
+ push(piece) {
1224
+ let rest = this.first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
1225
+ this.first = false;
1226
+ while (this.toSkip > 0) {
1227
+ const next = rest.indexOf("\n");
1228
+ if (next === -1) return [];
1229
+ rest = rest.slice(next + 1);
1230
+ this.toSkip -= 1;
1231
+ }
1232
+ if (rest === "") return [];
1233
+ const scanned = this.scanner.scan(this.carry + rest, false);
1234
+ this.carry = scanned.carry;
1235
+ return scanned.completed.filter(isNotBlank);
1236
+ }
1237
+ /**
1238
+ * Ends the file.
1239
+ * @returns The records left when the text ends, in order.
1240
+ * @throws NativeParseError when a quoted field is never closed, a quote opens inside an unquoted field, or the file has fewer lines than the recipe skips.
1241
+ */
1242
+ end() {
1243
+ if (this.toSkip > 0) throw new NativeParseError("csv-table", this.options.skipLines - this.toSkip + 1, `the file has fewer than ${String(this.options.skipLines)} lines to skip before the header`);
1244
+ const completed = this.scanner.scan(this.carry, true).completed;
1245
+ this.carry = "";
1246
+ const last = this.scanner.finish();
1247
+ return [...completed, ...last === void 0 ? [] : [last]].filter(isNotBlank);
1248
+ }
1249
+ };
1007
1250
  /**
1008
- * Splits comma-separated text into its records' fields by RFC 4180, as `records` does, keeping every record (blank lines included) at its position: the grid a workbook sheet's CSV makes, for a parser that reads a layout of blocks and not one table.
1251
+ * Splits comma-separated text into its records' fields by RFC 4180, as the record reader does, keeping every record (blank lines included) at its position: the grid a workbook sheet's CSV makes, for a parser that reads a layout of blocks and not one table.
1009
1252
  * @param text - The text.
1010
1253
  * @returns Each record's fields, in order.
1011
1254
  * @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
1012
1255
  */
1013
1256
  function csvGrid(text) {
1014
- return [...records$1([text], ",", 1)].map((record) => record.fields);
1015
- }
1016
- /**
1017
- * Drops the first physical lines of a file given in pieces.
1018
- * @param pieces - The file text in order.
1019
- * @param count - How many physical lines to drop.
1020
- * @returns The rest of the text, in pieces.
1021
- * @throws NativeParseError when the file has fewer lines than that.
1022
- */
1023
- function* afterLines(pieces, count) {
1024
- let remaining = count;
1025
- for (const piece of pieces) {
1026
- let rest = piece;
1027
- while (remaining > 0) {
1028
- const next = rest.indexOf("\n");
1029
- if (next === -1) {
1030
- rest = "";
1031
- break;
1032
- }
1033
- rest = rest.slice(next + 1);
1034
- remaining -= 1;
1035
- }
1036
- if (remaining === 0 && rest !== "") yield rest;
1037
- }
1038
- if (remaining > 0) throw new NativeParseError("csv-table", count - remaining + 1, `the file has fewer than ${String(count)} lines to skip before the header`);
1257
+ const scanner = new RecordScanner(DELIMITERS.comma.charCodeAt(0), 1);
1258
+ const { completed } = scanner.scan(text, true);
1259
+ const last = scanner.finish();
1260
+ return [...completed, ...last === void 0 ? [] : [last]].map((record) => record.fields);
1039
1261
  }
1040
1262
  /**
1041
- * The text of a file as its pieces, with a leading byte order mark removed.
1042
- * @param text - The file text, or its pieces in order.
1043
- * @returns The pieces.
1263
+ * Whether a record holds anything: a blank line is a record of one empty field.
1264
+ * @param record - The record.
1265
+ * @returns False for a blank line.
1044
1266
  */
1045
- function* withoutByteOrderMark(text) {
1046
- let first = true;
1047
- for (const piece of typeof text === "string" ? [text] : text) {
1048
- yield first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
1049
- first = false;
1050
- }
1267
+ function isNotBlank(record) {
1268
+ return record.fields.length > 1 || record.fields[0]?.trim() !== "";
1051
1269
  }
1052
1270
  /**
1053
1271
  * Reads a numeric cell.
@@ -1064,27 +1282,81 @@ function numeric$1(cell, factor, column, line) {
1064
1282
  return Number(cell) * factor;
1065
1283
  }
1066
1284
  /**
1067
- * The records of a table file after the lines the recipe skips and without its blank lines: the header first, then the rows, one at a time.
1068
- * @param text - The file text, or its pieces in order, which may begin with a byte order mark.
1069
- * @param options - The recipe's table parser, of which the delimiter and the lines to skip are read.
1070
- * @returns Each record.
1071
- * @throws NativeParseError when a quote is never closed or opens inside an unquoted field, or the file has fewer lines than the recipe skips.
1285
+ * Reads a delimited table into rows as its text arrives in pieces, holding only the record in progress between pieces, so a caller that hands each row on as it is returned holds no more of the table than that. The rows are exactly those `parseCsvTable` gives, in order, with the same errors on the same lines however the text is cut.
1072
1286
  */
1073
- function* tableRecords(text, options) {
1074
- const body = afterLines(withoutByteOrderMark(text), options.skipLines);
1075
- for (const record of records$1(body, DELIMITERS[options.delimiter], options.skipLines + 1)) if (record.fields.length > 1 || record.fields[0]?.trim() !== "") yield record;
1076
- }
1287
+ var TableReader = class {
1288
+ options;
1289
+ records;
1290
+ layout;
1291
+ rows = 0;
1292
+ missing;
1293
+ constructor(options) {
1294
+ this.options = options;
1295
+ this.records = new RecordReader(options);
1296
+ this.missing = new Set(options.nullValues ?? []);
1297
+ }
1298
+ /**
1299
+ * Reads the next piece of the file's text.
1300
+ * @param piece - The text, which follows the pieces read before it.
1301
+ * @returns The rows the piece completes, in file order.
1302
+ * @throws NativeParseError as `parseCsvTable` does, naming the line of the problem.
1303
+ */
1304
+ push(piece) {
1305
+ return this.rowsOf(this.records.push(piece));
1306
+ }
1307
+ /**
1308
+ * Ends the file.
1309
+ * @returns The rows left when the text ends, in file order.
1310
+ * @throws NativeParseError as `parseCsvTable` does, including for a file with no header row.
1311
+ */
1312
+ end() {
1313
+ const rows = this.rowsOf(this.records.end());
1314
+ if (this.layout === void 0) throw new NativeParseError("csv-table", this.options.skipLines + 1, "the file has no header row");
1315
+ return rows;
1316
+ }
1317
+ /**
1318
+ * Builds the rows of records, the first record of the file being its header.
1319
+ * @param records - Records in file order.
1320
+ * @returns The rows among them.
1321
+ */
1322
+ rowsOf(records) {
1323
+ const rows = [];
1324
+ for (const record of records) if (this.layout === void 0) this.layout = layoutOf(record, this.options);
1325
+ else rows.push(this.rowOf(record, this.layout));
1326
+ return rows;
1327
+ }
1328
+ /**
1329
+ * Builds one row.
1330
+ * @param record - A record after the header.
1331
+ * @param layout - The header's layout.
1332
+ * @returns The row.
1333
+ * @throws NativeParseError when the record's width differs from the header's, its id is empty or a numeric cell is not a number.
1334
+ */
1335
+ rowOf(record, layout) {
1336
+ const { idColumn } = this.options;
1337
+ if (record.fields.length !== layout.width) throw new NativeParseError("csv-table", record.line, `the row has ${String(record.fields.length)} fields, the header has ${String(layout.width)}`);
1338
+ this.rows += 1;
1339
+ const id = idColumn === null ? String(this.rows) : (record.fields[layout.idPosition] ?? "").trim();
1340
+ if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
1341
+ const cells = layout.kept.map(({ name, position, factor }) => {
1342
+ const trimmed = (record.fields[position] ?? "").trim();
1343
+ const cell = this.missing.has(trimmed) ? "" : trimmed;
1344
+ return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
1345
+ });
1346
+ return {
1347
+ ...Object.fromEntries(cells),
1348
+ id
1349
+ };
1350
+ }
1351
+ };
1077
1352
  /**
1078
- * Parses a delimited table into rows (a cell whose text is one of the recipe's `nullValues` is empty, so null): the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
1079
- * @param text - The file text, or its pieces in order.
1353
+ * Reads a table's header into the layout its rows are built by, checking the recipe against it.
1354
+ * @param header - The header record.
1080
1355
  * @param options - The recipe's table parser.
1081
- * @returns The rows, in file order.
1082
- * @throws NativeParseError naming the line of the first problem, including a listed or numeric column the header lacks or a numeric column that is not listed.
1356
+ * @returns The layout.
1357
+ * @throws NativeParseError when two header cells get the same name, the recipe names a column the header lacks, lists the id column among the columns, or declares numeric a column it does not keep.
1083
1358
  */
1084
- function parseCsvTable(text, options) {
1085
- const table = tableRecords(text, options);
1086
- const { value: header } = table.next();
1087
- if (header === void 0) throw new NativeParseError("csv-table", options.skipLines + 1, "the file has no header row");
1359
+ function layoutOf(header, options) {
1088
1360
  const names = columnNames(header.fields, options.idColumn, header.line);
1089
1361
  const { idColumn, numericColumns, columns } = options;
1090
1362
  const wanted = new Set(columns ?? names.filter((name) => name !== idColumn));
@@ -1096,28 +1368,28 @@ function parseCsvTable(text, options) {
1096
1368
  if (idColumn !== null && wanted.has(idColumn)) throw new NativeParseError("csv-table", header.line, `the id column ${idColumn} is the row's id, so it is not listed among the columns`);
1097
1369
  for (const { column } of numericColumns) if (!wanted.has(column)) throw new NativeParseError("csv-table", header.line, `the column ${column} is declared numeric but is not among the columns kept`);
1098
1370
  const factors = new Map(numericColumns.map((entry) => [entry.column, entry.factor]));
1099
- const missing = new Set(options.nullValues ?? []);
1100
- const idPosition = idColumn === null ? -1 : names.indexOf(idColumn);
1101
- const kept = names.flatMap((name, position) => name !== idColumn && wanted.has(name) ? [{
1102
- name,
1103
- position,
1104
- factor: factors.get(name)
1105
- }] : []);
1371
+ return {
1372
+ width: names.length,
1373
+ idPosition: idColumn === null ? -1 : names.indexOf(idColumn),
1374
+ kept: names.flatMap((name, position) => name !== idColumn && wanted.has(name) ? [{
1375
+ name,
1376
+ position,
1377
+ factor: factors.get(name)
1378
+ }] : [])
1379
+ };
1380
+ }
1381
+ /**
1382
+ * Parses a delimited table into rows (a cell whose text is one of the recipe's `nullValues` is empty, so null): the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
1383
+ * @param text - The file text, or its pieces in order.
1384
+ * @param options - The recipe's table parser.
1385
+ * @returns The rows, in file order.
1386
+ * @throws NativeParseError naming the line of the first problem, including a listed or numeric column the header lacks or a numeric column that is not listed.
1387
+ */
1388
+ function parseCsvTable(text, options) {
1389
+ const reader = new TableReader(options);
1106
1390
  const nodes = [];
1107
- for (const record of table) {
1108
- if (record.fields.length !== names.length) throw new NativeParseError("csv-table", record.line, `the row has ${String(record.fields.length)} fields, the header has ${String(names.length)}`);
1109
- const id = idColumn === null ? String(nodes.length + 1) : (record.fields[idPosition] ?? "").trim();
1110
- if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
1111
- const cells = kept.map(({ name, position, factor }) => {
1112
- const trimmed = (record.fields[position] ?? "").trim();
1113
- const cell = missing.has(trimmed) ? "" : trimmed;
1114
- return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
1115
- });
1116
- nodes.push({
1117
- ...Object.fromEntries(cells),
1118
- id
1119
- });
1120
- }
1391
+ for (const piece of typeof text === "string" ? [text] : text) for (const row of reader.push(piece)) nodes.push(row);
1392
+ for (const row of reader.end()) nodes.push(row);
1121
1393
  return { nodes };
1122
1394
  }
1123
1395
  //#endregion
@@ -1222,7 +1494,7 @@ function parseGeojson(text, options) {
1222
1494
  }
1223
1495
  //#endregion
1224
1496
  //#region src/parsers/hhcrsp.ts
1225
- const FORMAT$10 = "hhcrsp";
1497
+ const FORMAT$13 = "hhcrsp";
1226
1498
  /** The labels of the sheet's blocks, each the top-left cell of a block whose header runs to its right and whose rows run below it. */
1227
1499
  const LABEL = {
1228
1500
  patients: "Patients",
@@ -1234,9 +1506,9 @@ const LABEL = {
1234
1506
  };
1235
1507
  /** A service's header, `Service <k>`, a patient's row label and a caregiver's. */
1236
1508
  const SERVICE = /^Service (\d+)$/;
1237
- const NUMBER$7 = /^-?\d+(?:\.\d+)?$/;
1509
+ const NUMBER$8 = /^-?\d+(?:\.\d+)?$/;
1238
1510
  /** How many required services a patient may have: the instances ask for one, or two that must be synchronised, and the parser has a duration column for each. */
1239
- const MOST_SERVICES = 2;
1511
+ const MOST_SERVICES$1 = 2;
1240
1512
  /**
1241
1513
  * Finds a block by its label: the one cell of the sheet that holds it, its header the cells to its right up to the first empty one, and its rows those below it up to the first whose label and cells are all empty. Past its header a block has an empty column, so the cell there must be empty in every row and a value the header does not label is refused rather than left unread; `Pat / Ser` instead keeps its count there, with the next block starting right after it, and its count is checked on its own.
1242
1514
  * @param grid - The sheet's cells by row.
@@ -1251,7 +1523,7 @@ function blockOf(grid, label, pastHeader = "empty") {
1251
1523
  column
1252
1524
  }] : []));
1253
1525
  const [only] = found;
1254
- if (only === void 0 || found.length !== 1) throw new NativeParseError(FORMAT$10, 1, `expected one block labelled ${JSON.stringify(label)}, found ${String(found.length)}`);
1526
+ if (only === void 0 || found.length !== 1) throw new NativeParseError(FORMAT$13, 1, `expected one block labelled ${JSON.stringify(label)}, found ${String(found.length)}`);
1255
1527
  const headerRow = grid[only.line] ?? [];
1256
1528
  const end = headerRow.findIndex((cell, column) => column > only.column && cell.trim() === "");
1257
1529
  const header = headerRow.slice(only.column + 1, end === -1 ? headerRow.length : end).map((cell) => cell.trim());
@@ -1260,7 +1532,7 @@ function blockOf(grid, label, pastHeader = "empty") {
1260
1532
  const cells = (grid[line] ?? []).slice(only.column, only.column + 1 + header.length).map((cell) => cell.trim());
1261
1533
  if (cells.every((cell) => cell === "")) break;
1262
1534
  const past = (grid[line] ?? [])[only.column + 1 + header.length]?.trim();
1263
- if (pastHeader === "empty" && past !== void 0 && past !== "") throw new NativeParseError(FORMAT$10, line + 1, `the ${label} block's row holds ${JSON.stringify(past)} past its header ${JSON.stringify(header)}, which does not label it`);
1535
+ if (pastHeader === "empty" && past !== void 0 && past !== "") throw new NativeParseError(FORMAT$13, line + 1, `the ${label} block's row holds ${JSON.stringify(past)} past its header ${JSON.stringify(header)}, which does not label it`);
1264
1536
  rows.push({
1265
1537
  line: line + 1,
1266
1538
  label: cells[0] ?? "",
@@ -1283,7 +1555,7 @@ function blockOf(grid, label, pastHeader = "empty") {
1283
1555
  * @throws NativeParseError when the cell is not a decimal number.
1284
1556
  */
1285
1557
  function numberOf$4(cell, line, what) {
1286
- if (cell === void 0 || !NUMBER$7.test(cell)) throw new NativeParseError(FORMAT$10, line, `expected ${what}, found ${JSON.stringify(cell)}`);
1558
+ if (cell === void 0 || !NUMBER$8.test(cell)) throw new NativeParseError(FORMAT$13, line, `expected ${what}, found ${JSON.stringify(cell)}`);
1287
1559
  return Number(cell);
1288
1560
  }
1289
1561
  /**
@@ -1295,7 +1567,7 @@ function numberOf$4(cell, line, what) {
1295
1567
  */
1296
1568
  function checkNumbered(block, noun, label) {
1297
1569
  block.rows.forEach((row, index) => {
1298
- if (row.label !== `${noun} ${String(index + 1)}`) throw new NativeParseError(FORMAT$10, row.line, `the ${label} block's row ${String(index + 1)} is labelled ${JSON.stringify(row.label)}, not ${noun} ${String(index + 1)}`);
1570
+ if (row.label !== `${noun} ${String(index + 1)}`) throw new NativeParseError(FORMAT$13, row.line, `the ${label} block's row ${String(index + 1)} is labelled ${JSON.stringify(row.label)}, not ${noun} ${String(index + 1)}`);
1299
1571
  });
1300
1572
  }
1301
1573
  /**
@@ -1307,7 +1579,7 @@ function checkNumbered(block, noun, label) {
1307
1579
  */
1308
1580
  function serviceCount(block, label) {
1309
1581
  block.header.forEach((cell, index) => {
1310
- if (SERVICE.exec(cell)?.[1] !== String(index + 1)) throw new NativeParseError(FORMAT$10, block.line, `the ${label} block's header cell ${String(index + 1)} is ${JSON.stringify(cell)}, not Service ${String(index + 1)}`);
1582
+ if (SERVICE.exec(cell)?.[1] !== String(index + 1)) throw new NativeParseError(FORMAT$13, block.line, `the ${label} block's header cell ${String(index + 1)} is ${JSON.stringify(cell)}, not Service ${String(index + 1)}`);
1311
1583
  });
1312
1584
  return block.header.length;
1313
1585
  }
@@ -1326,7 +1598,7 @@ function parseHhcrsp(text) {
1326
1598
  const caregiverWindows = blockOf(grid, LABEL.caregiverWindows);
1327
1599
  const windows = blockOf(grid, LABEL.patientWindows);
1328
1600
  const serviceKinds = serviceCount(requests, LABEL.requests);
1329
- if (serviceCount(times, LABEL.times) !== serviceKinds || serviceCount(skills, LABEL.skills) !== serviceKinds) throw new NativeParseError(FORMAT$10, times.line, `the ${LABEL.requests}, ${LABEL.times} and ${LABEL.skills} blocks name different numbers of services`);
1601
+ if (serviceCount(times, LABEL.times) !== serviceKinds || serviceCount(skills, LABEL.skills) !== serviceKinds) throw new NativeParseError(FORMAT$13, times.line, `the ${LABEL.requests}, ${LABEL.times} and ${LABEL.skills} blocks name different numbers of services`);
1330
1602
  checkNumbered(requests, "Patient", LABEL.requests);
1331
1603
  checkNumbered(windows, "Patient", LABEL.patientWindows);
1332
1604
  checkNumbered(skills, "Caregiver", LABEL.skills);
@@ -1341,11 +1613,11 @@ function parseHhcrsp(text) {
1341
1613
  "",
1342
1614
  "center",
1343
1615
  "centre"
1344
- ].includes(centreTimes.label.toLowerCase()) || windows.rows.length !== patients || patientTimes.length !== patients || skills.rows.length !== caregiverWindows.rows.length) throw new NativeParseError(FORMAT$10, requests.line, `the blocks disagree: ${String(patients)} patients requesting services, ${String(patientTimes.length)} with durations after the centre's row, ${String(windows.rows.length)} with time windows, and ${String(skills.rows.length)} caregivers with skills against ${String(caregiverWindows.rows.length)} with windows`);
1345
- if (positions.header.join(",") !== "X,Y" || positions.rows.length !== patients + 2 || positions.rows.some((row, index) => row.label !== String(index))) throw new NativeParseError(FORMAT$10, positions.line, `the ${LABEL.patients} block must give X and Y for nodes 0 to ${String(patients + 1)}, the centre, ${String(patients)} patients and the centre again`);
1616
+ ].includes(centreTimes.label.toLowerCase()) || windows.rows.length !== patients || patientTimes.length !== patients || skills.rows.length !== caregiverWindows.rows.length) throw new NativeParseError(FORMAT$13, requests.line, `the blocks disagree: ${String(patients)} patients requesting services, ${String(patientTimes.length)} with durations after the centre's row, ${String(windows.rows.length)} with time windows, and ${String(skills.rows.length)} caregivers with skills against ${String(caregiverWindows.rows.length)} with windows`);
1617
+ if (positions.header.join(",") !== "X,Y" || positions.rows.length !== patients + 2 || positions.rows.some((row, index) => row.label !== String(index))) throw new NativeParseError(FORMAT$13, positions.line, `the ${LABEL.patients} block must give X and Y for nodes 0 to ${String(patients + 1)}, the centre, ${String(patients)} patients and the centre again`);
1346
1618
  const position = (index) => {
1347
1619
  const row = positions.rows[index];
1348
- if (row === void 0) throw new NativeParseError(FORMAT$10, positions.line, `the ${LABEL.patients} block has no node ${String(index)}`);
1620
+ if (row === void 0) throw new NativeParseError(FORMAT$13, positions.line, `the ${LABEL.patients} block has no node ${String(index)}`);
1349
1621
  return {
1350
1622
  x: numberOf$4(row.cells[0], row.line, "an X"),
1351
1623
  y: numberOf$4(row.cells[1], row.line, "a Y"),
@@ -1354,7 +1626,7 @@ function parseHhcrsp(text) {
1354
1626
  };
1355
1627
  const { x, y } = position(0);
1356
1628
  const closing = position(patients + 1);
1357
- if (closing.x !== x || closing.y !== y) throw new NativeParseError(FORMAT$10, closing.line, `the last node of the ${LABEL.patients} block is not at the centre's position, so it is not the route's end`);
1629
+ if (closing.x !== x || closing.y !== y) throw new NativeParseError(FORMAT$13, closing.line, `the last node of the ${LABEL.patients} block is not at the centre's position, so it is not the route's end`);
1358
1630
  const windowColumn = (name) => {
1359
1631
  const index = windows.header.indexOf(name);
1360
1632
  return index === -1 ? void 0 : index;
@@ -1365,7 +1637,7 @@ function parseHhcrsp(text) {
1365
1637
  earliest2: windowColumn("ETW 2"),
1366
1638
  latest2: windowColumn("LTW 2")
1367
1639
  };
1368
- if (columns.earliest1 === void 0 || columns.latest1 === void 0 || columns.earliest2 === void 0 !== (columns.latest2 === void 0) || windows.header.length !== (columns.earliest2 === void 0 ? 2 : 4)) throw new NativeParseError(FORMAT$10, windows.line, `the ${LABEL.patientWindows} header ${JSON.stringify(windows.header)} is not ETW 1 and LTW 1, with or without ETW 2 and LTW 2`);
1640
+ if (columns.earliest1 === void 0 || columns.latest1 === void 0 || columns.earliest2 === void 0 !== (columns.latest2 === void 0) || windows.header.length !== (columns.earliest2 === void 0 ? 2 : 4)) throw new NativeParseError(FORMAT$13, windows.line, `the ${LABEL.patientWindows} header ${JSON.stringify(windows.header)} is not ETW 1 and LTW 1, with or without ETW 2 and LTW 2`);
1369
1641
  const nodes = [{
1370
1642
  id: "0",
1371
1643
  role: "centre",
@@ -1381,15 +1653,15 @@ function parseHhcrsp(text) {
1381
1653
  }];
1382
1654
  requests.rows.forEach((request, index) => {
1383
1655
  const required = request.cells.flatMap((cell, service) => {
1384
- if (cell !== "0" && cell !== "1" && cell !== "") throw new NativeParseError(FORMAT$10, request.line, `${request.label} has ${JSON.stringify(cell)} for Service ${String(service + 1)}, not 0, 1 or empty`);
1656
+ if (cell !== "0" && cell !== "1" && cell !== "") throw new NativeParseError(FORMAT$13, request.line, `${request.label} has ${JSON.stringify(cell)} for Service ${String(service + 1)}, not 0, 1 or empty`);
1385
1657
  return cell === "1" ? [service] : [];
1386
1658
  });
1387
1659
  const stated = (grid[request.line - 1] ?? [])[requests.column + serviceKinds + 1]?.trim();
1388
- if (stated !== String(required.length)) throw new NativeParseError(FORMAT$10, request.line, `${request.label} requires ${String(required.length)} services by its marks, and the count beside them is ${JSON.stringify(stated)}`);
1389
- if (required.length === 0 || required.length > MOST_SERVICES) throw new NativeParseError(FORMAT$10, request.line, `${request.label} requires ${String(required.length)} services, and a patient requires one or two`);
1660
+ if (stated !== String(required.length)) throw new NativeParseError(FORMAT$13, request.line, `${request.label} requires ${String(required.length)} services by its marks, and the count beside them is ${JSON.stringify(stated)}`);
1661
+ if (required.length === 0 || required.length > MOST_SERVICES$1) throw new NativeParseError(FORMAT$13, request.line, `${request.label} requires ${String(required.length)} services, and a patient requires one or two`);
1390
1662
  const timeRow = patientTimes[index];
1391
1663
  const windowRow = windows.rows[index];
1392
- if (timeRow === void 0 || windowRow === void 0) throw new NativeParseError(FORMAT$10, request.line, `${request.label} has no durations or time windows`);
1664
+ if (timeRow === void 0 || windowRow === void 0) throw new NativeParseError(FORMAT$13, request.line, `${request.label} has no durations or time windows`);
1393
1665
  const duration = (which) => {
1394
1666
  const service = required[which];
1395
1667
  return service === void 0 ? null : numberOf$4(timeRow.cells[service], timeRow.line, `the duration of Service ${String(service + 1)}`);
@@ -1420,6 +1692,433 @@ function parseHhcrsp(text) {
1420
1692
  };
1421
1693
  }
1422
1694
  //#endregion
1695
+ //#region src/parsers/mankowska.ts
1696
+ const FORMAT$12 = "mankowska";
1697
+ /** The sections of the file, each a keyword on a line of its own followed by its numbers. */
1698
+ const SECTIONS$1 = [
1699
+ "nbNodes",
1700
+ "nbVehi",
1701
+ "nbServi",
1702
+ "r",
1703
+ "DS",
1704
+ "a",
1705
+ "x",
1706
+ "y",
1707
+ "d",
1708
+ "p",
1709
+ "mind",
1710
+ "maxd",
1711
+ "e",
1712
+ "l"
1713
+ ];
1714
+ /** How many patients a double service has, and so the most required services a patient can have. */
1715
+ const MOST_SERVICES = 2;
1716
+ /** The depot copies: the first and the last node. */
1717
+ const DEPOTS = 2;
1718
+ const NUMBER$7 = /^-?\d+(?:\.\d+)?$/;
1719
+ const KEYWORD = /^[A-Za-z]+$/;
1720
+ /**
1721
+ * Splits the file into its sections: a line holding one word starts a section, and every number on the lines after it, up to the next such line, is the section's.
1722
+ * @param text - The file text.
1723
+ * @returns Each section by its keyword.
1724
+ * @throws NativeParseError for a keyword the format does not have, one given twice, a value that is not a number, or a number before the first keyword.
1725
+ */
1726
+ function sectionsOf$1(text) {
1727
+ const sections = /* @__PURE__ */ new Map();
1728
+ let current;
1729
+ for (const [index, raw] of lines(text).entries()) {
1730
+ const tokens = raw.trim().split(/\s+/).filter((token) => token !== "");
1731
+ const [only] = tokens;
1732
+ if (tokens.length === 1 && only !== void 0 && KEYWORD.test(only)) {
1733
+ const name = SECTIONS$1.find((section) => section === only);
1734
+ if (name === void 0 || sections.has(name)) throw new NativeParseError(FORMAT$12, index + 1, name === void 0 ? `the section ${JSON.stringify(only)} is not one of ${SECTIONS$1.join(", ")}` : `the section ${name} is given twice`);
1735
+ current = [];
1736
+ sections.set(name, {
1737
+ line: index + 1,
1738
+ values: current
1739
+ });
1740
+ continue;
1741
+ }
1742
+ for (const token of tokens) {
1743
+ if (current === void 0 || !NUMBER$7.test(token)) throw new NativeParseError(FORMAT$12, index + 1, current === void 0 ? "a number comes before the first section" : `expected a number, found ${JSON.stringify(token)}`);
1744
+ current.push(Number(token));
1745
+ }
1746
+ }
1747
+ return sections;
1748
+ }
1749
+ /**
1750
+ * Reads a section that must hold a given count of numbers.
1751
+ * @param sections - The sections.
1752
+ * @param name - The section.
1753
+ * @param count - How many numbers it must hold.
1754
+ * @returns Its numbers.
1755
+ * @throws NativeParseError when the section is missing or holds another count.
1756
+ */
1757
+ function sized(sections, name, count) {
1758
+ const section = sections.get(name);
1759
+ if (section === void 0) throw new NativeParseError(FORMAT$12, 1, `the file has no ${name} section`);
1760
+ if (section.values.length !== count) throw new NativeParseError(FORMAT$12, section.line, `the ${name} section holds ${String(section.values.length)} numbers where ${String(count)} are due`);
1761
+ return section.values;
1762
+ }
1763
+ /**
1764
+ * Reads a count section (`nbNodes`, `nbVehi`, `nbServi`): one whole number of at least a least value.
1765
+ * @param sections - The sections.
1766
+ * @param name - The section.
1767
+ * @param least - The smallest count that makes an instance.
1768
+ * @returns The count.
1769
+ * @throws NativeParseError when the section is missing, holds more than one number, or the count is not a whole number of at least `least`.
1770
+ */
1771
+ function countOf(sections, name, least) {
1772
+ const [count] = sized(sections, name, 1);
1773
+ if (count === void 0 || !Number.isInteger(count) || count < least) throw new NativeParseError(FORMAT$12, sections.get(name)?.line ?? 1, `${name} must be a whole number of at least ${String(least)}, found ${String(count)}`);
1774
+ return count;
1775
+ }
1776
+ /**
1777
+ * Reads one cell of a row-major matrix section.
1778
+ * @param values - The section's numbers.
1779
+ * @param columns - The matrix's width.
1780
+ * @param row - The row.
1781
+ * @param column - The column.
1782
+ * @returns The cell.
1783
+ * @throws NativeParseError when the cell is outside the section, which `sized` has already ruled out.
1784
+ */
1785
+ function cell(values, columns, row, column) {
1786
+ const value = values[row * columns + column];
1787
+ if (value === void 0) throw new NativeParseError(FORMAT$12, 1, `cell ${String(row)}, ${String(column)} is outside its section`);
1788
+ return value;
1789
+ }
1790
+ /**
1791
+ * Parses a home health care routing and scheduling instance in the keyword text format of Mankowska, Meisel and Bierwirth (2014), as the OVIG generator writes it: the counts of nodes (`nbNodes`, the patients and two copies of the depot, the first and the last), vehicles (`nbVehi`, the caregivers) and services (`nbServi`); the services each node requires (`r`, a node by service matrix of zeros and ones), the double service nodes (`DS`, node numbers from one), the services each caregiver can give (`a`), the positions (`x`, `y`), the travel matrix (`d`), the processing time of each service at each node by each caregiver (`p`, a row per node and caregiver), the least and most time between a double service's two services (`mind`, `maxd`), and the time windows (`e`, `l`). A node's id is its number from zero. The depot copies are marked as requiring every service, which is the format's convention and not a visit, so they carry no services or processing times. A patient's required services are their numbers from one, joined by spaces, and its first and second processing time are those of its required services in that order. The travel matrix and the caregivers' qualifications are not node fields and are not kept; the processing times must not differ between caregivers, since a node column holds one value. The gaps are kept for a double service node only. The units of time are the generator's own.
1792
+ * @param text - The file text.
1793
+ * @returns The instance: its patient, caregiver and service kind counts as `meta`, and its nodes in node order.
1794
+ * @throws NativeParseError naming the line for a section that is missing, given twice or of the wrong size, a value that is not a number, a requirement that is not zero or one, a node with more than two required services, a double service node with fewer than two or out of range, or processing times that differ between caregivers.
1795
+ */
1796
+ function parseMankowska(text) {
1797
+ const sections = sectionsOf$1(text);
1798
+ const nodeCount = countOf(sections, "nbNodes", DEPOTS);
1799
+ const caregivers = countOf(sections, "nbVehi", 1);
1800
+ const serviceKinds = countOf(sections, "nbServi", 1);
1801
+ const required = sized(sections, "r", nodeCount * serviceKinds);
1802
+ sized(sections, "a", caregivers * serviceKinds);
1803
+ const xs = sized(sections, "x", nodeCount);
1804
+ const ys = sized(sections, "y", nodeCount);
1805
+ sized(sections, "d", nodeCount * nodeCount);
1806
+ const times = sized(sections, "p", nodeCount * caregivers * serviceKinds);
1807
+ const least = sized(sections, "mind", nodeCount);
1808
+ const most = sized(sections, "maxd", nodeCount);
1809
+ const earliest = sized(sections, "e", nodeCount);
1810
+ const latest = sized(sections, "l", nodeCount);
1811
+ const line = (name) => sections.get(name)?.line ?? 1;
1812
+ const doubles = new Set((sections.get("DS")?.values ?? []).map((number) => {
1813
+ if (!Number.isInteger(number) || number < 1 || number > nodeCount) throw new NativeParseError(FORMAT$12, line("DS"), `the double service node ${String(number)} is not a node number from 1 to ${String(nodeCount)}`);
1814
+ return number - 1;
1815
+ }));
1816
+ const nodes = Array.from({ length: nodeCount }, (_, node) => {
1817
+ const at = (values) => cell(values, 1, node, 0);
1818
+ if (node === 0 || node === nodeCount - 1) return {
1819
+ id: String(node),
1820
+ role: "depot",
1821
+ x: at(xs),
1822
+ y: at(ys),
1823
+ services: null,
1824
+ duration1: null,
1825
+ duration2: null,
1826
+ earliest: at(earliest),
1827
+ latest: at(latest),
1828
+ minGap: null,
1829
+ maxGap: null
1830
+ };
1831
+ const services = Array.from({ length: serviceKinds }, (__, service) => service).filter((service) => {
1832
+ const flag = cell(required, serviceKinds, node, service);
1833
+ if (flag !== 0 && flag !== 1) throw new NativeParseError(FORMAT$12, line("r"), `node ${String(node)} has the requirement ${String(flag)} for service ${String(service + 1)}, not 0 or 1`);
1834
+ return flag === 1;
1835
+ });
1836
+ if (services.length > MOST_SERVICES || doubles.has(node) && services.length !== MOST_SERVICES) throw new NativeParseError(FORMAT$12, line("r"), `node ${String(node)} requires ${String(services.length)} services${doubles.has(node) ? " but is a double service node" : ""}`);
1837
+ const [first, second] = services.map((service) => {
1838
+ const byCaregiver = Array.from({ length: caregivers }, (__, caregiver) => cell(times, serviceKinds, node * caregivers + caregiver, service));
1839
+ const [time] = byCaregiver;
1840
+ if (time === void 0 || byCaregiver.some((other) => other !== time)) throw new NativeParseError(FORMAT$12, line("p"), `node ${String(node)} has processing times for service ${String(service + 1)} that differ between caregivers (${byCaregiver.join(", ")})`);
1841
+ return time;
1842
+ });
1843
+ return {
1844
+ id: String(node),
1845
+ role: "patient",
1846
+ x: at(xs),
1847
+ y: at(ys),
1848
+ services: services.length === 0 ? null : services.map((service) => String(service + 1)).join(" "),
1849
+ duration1: first ?? null,
1850
+ duration2: second ?? null,
1851
+ earliest: at(earliest),
1852
+ latest: at(latest),
1853
+ minGap: doubles.has(node) ? at(least) : null,
1854
+ maxGap: doubles.has(node) ? at(most) : null
1855
+ };
1856
+ });
1857
+ return {
1858
+ meta: {
1859
+ patients: nodeCount - DEPOTS,
1860
+ caregivers,
1861
+ serviceKinds
1862
+ },
1863
+ nodes
1864
+ };
1865
+ }
1866
+ //#endregion
1867
+ //#region src/parsers/tdc.ts
1868
+ const FORMAT$11 = "tdc";
1869
+ /** A technician line: its id, skills, time window and whether it is an investment option. */
1870
+ const TECHNICIAN = /^Technician id \d+, skills \[[^\]]*\], time window \[[^\]]*\], investment option (?:true|false)$/;
1871
+ /** A task line, with the fields the parser keeps captured: id, skills, duration, window opening and closing, priority, penalty, creation, modification (or `nothing`) and cancellation; its relevant technicians follow and are not kept. */
1872
+ const TASK = /^Task id (\d+), skills \[([^\]]*)\], duration (\d+), time window \[([^,\]]+), ([^\]]+)\], priority (\d+), penalty (-?\d+(?:\.\d+)?), created ([^,]+), modified ([^,]+), cancelled (true|false), relevant technician ids (?:Int64)?\[[\d, ]*\]$/;
1873
+ /** A travel time line, between two tasks or between a technician and a task, in either direction. */
1874
+ const TRAVEL = /^(?:task|technician)\d+(?:task|technician)\d+: -?\d+(?:\.\d+)?$/;
1875
+ /** The section headings, in the order the file gives them. */
1876
+ const HEADINGS = [
1877
+ "Technicians:",
1878
+ "Tasks:",
1879
+ "Travel time:"
1880
+ ];
1881
+ /** What a line before any heading and a line of each section is, for errors: indexed by the section's position plus one, so the lines before the first heading are first. */
1882
+ const SECTION_NOUNS = [
1883
+ "heading",
1884
+ "technician",
1885
+ "task",
1886
+ "travel time"
1887
+ ];
1888
+ /**
1889
+ * Names what a position's lines are, for an error.
1890
+ * @param position - A section's position plus one.
1891
+ * @returns The noun.
1892
+ */
1893
+ function nounOf(position) {
1894
+ return SECTION_NOUNS[position] ?? `section ${String(position)}`;
1895
+ }
1896
+ /** Where the task pattern captures each field it keeps. */
1897
+ const TASK_AT = {
1898
+ id: 1,
1899
+ skills: 2,
1900
+ duration: 3,
1901
+ earliest: 4,
1902
+ latest: 5,
1903
+ priority: 6,
1904
+ penalty: 7,
1905
+ created: 8,
1906
+ modified: 9,
1907
+ cancelled: 10
1908
+ };
1909
+ /** How much of a refused line an error shows. */
1910
+ const SHOWN = 120;
1911
+ /**
1912
+ * Parses a technician routing and scheduling instance in the sectioned text format of Gamst and Pisinger (2024): a `Technicians:` heading and a line per technician, a `Tasks:` heading and a line per task (`Task id <n>, skills ["<skill>", ...], duration <minutes>, time window [<opening>, <closing>], priority <n>, penalty <n>, created <time>, modified <time or nothing>, cancelled <true or false>, relevant technician ids [<ids>]`), then a `Travel time:` heading and a line per pair, the travel minutes between two tasks or between a technician and a task. The file is read a line at a time and the travel time lines, almost all of a file, are checked and let go, as the tasks/v1 target has no arcs; a technician's line is counted. A task's id is its id, and a modification of `nothing` is null. Times are as the file writes them.
1913
+ * @param pieces - The file's text, in pieces.
1914
+ * @returns The instance: its technician count as `meta`, and its tasks in file order.
1915
+ * @throws NativeParseError naming the line for a heading out of order or missing, a line that is not of its section's shape, or a task id given twice.
1916
+ */
1917
+ function parseTdc(pieces) {
1918
+ let section = -1;
1919
+ let technicians = 0;
1920
+ const nodes = [];
1921
+ const ids = /* @__PURE__ */ new Set();
1922
+ forEachLine(pieces, (raw, line) => {
1923
+ const text = raw.trim();
1924
+ if (text === "") return;
1925
+ const heading = HEADINGS.findIndex((candidate) => candidate === text);
1926
+ if (heading !== -1) {
1927
+ if (heading !== section + 1) throw new NativeParseError(FORMAT$11, line, `the heading ${text} is out of order`);
1928
+ section = heading;
1929
+ return;
1930
+ }
1931
+ if (section === 0 && TECHNICIAN.test(text)) {
1932
+ technicians += 1;
1933
+ return;
1934
+ }
1935
+ if (section === 2 && TRAVEL.test(text)) return;
1936
+ const task = section === 1 ? TASK.exec(text) : null;
1937
+ if (task === null) throw new NativeParseError(FORMAT$11, line, `the line is not a ${nounOf(section + 1)} line: ${JSON.stringify(text.slice(0, SHOWN))}`);
1938
+ const field = (index) => {
1939
+ const value = task[index];
1940
+ if (value === void 0) throw new NativeParseError(FORMAT$11, line, `the task line has no field ${String(index)}`);
1941
+ return value;
1942
+ };
1943
+ const id = field(TASK_AT.id);
1944
+ if (ids.has(id)) throw new NativeParseError(FORMAT$11, line, `the task ${id} is given twice`);
1945
+ ids.add(id);
1946
+ const skills = field(TASK_AT.skills).split(",").map((skill) => skill.trim().replace(/^"|"$/g, "")).filter((skill) => skill !== "");
1947
+ const modified = field(TASK_AT.modified);
1948
+ nodes.push({
1949
+ id,
1950
+ skills: skills.length === 0 ? null : skills.join(" "),
1951
+ durationMinutes: Number(field(TASK_AT.duration)),
1952
+ earliest: field(TASK_AT.earliest).trim(),
1953
+ latest: field(TASK_AT.latest).trim(),
1954
+ priority: Number(field(TASK_AT.priority)),
1955
+ penalty: Number(field(TASK_AT.penalty)),
1956
+ created: field(TASK_AT.created),
1957
+ modified: modified === "nothing" ? null : modified,
1958
+ cancelled: field(TASK_AT.cancelled)
1959
+ });
1960
+ });
1961
+ if (section !== HEADINGS.length - 1) throw new NativeParseError(FORMAT$11, 1, `the file has no ${nounOf(section + 2)} heading`);
1962
+ return {
1963
+ meta: { technicians },
1964
+ nodes
1965
+ };
1966
+ }
1967
+ //#endregion
1968
+ //#region src/parsers/verolog-2017.ts
1969
+ const FORMAT$10 = "verolog-2017";
1970
+ /** The header lines, with the key of `meta` each fills and whether its value is text. */
1971
+ const HEADER$3 = /* @__PURE__ */ new Map([
1972
+ ["DATASET", ["dataset", "text"]],
1973
+ ["NAME", ["name", "text"]],
1974
+ ["DAYS", ["days", "number"]],
1975
+ ["CAPACITY", ["capacity", "number"]],
1976
+ ["MAX_TRIP_DISTANCE", ["maxTripDistance", "number"]],
1977
+ ["DEPOT_COORDINATE", ["depotCoordinate", "number"]],
1978
+ ["VEHICLE_COST", ["vehicleCost", "number"]],
1979
+ ["VEHICLE_DAY_COST", ["vehicleDayCost", "number"]],
1980
+ ["DISTANCE_COST", ["distanceCost", "number"]]
1981
+ ]);
1982
+ /** The sections that hold rows, and how many numbers each row has. */
1983
+ const SECTION_WIDTHS = {
1984
+ TOOLS: 4,
1985
+ COORDINATES: 3,
1986
+ REQUESTS: 7
1987
+ };
1988
+ /** Where a request row keeps its fields. */
1989
+ const REQUEST_AT$1 = {
1990
+ id: 0,
1991
+ location: 1,
1992
+ fromDay: 2,
1993
+ toDay: 3,
1994
+ stayDays: 4,
1995
+ tool: 5,
1996
+ amount: 6
1997
+ };
1998
+ const WHOLE$3 = /^-?\d+$/;
1999
+ /**
2000
+ * Reads a field of a row.
2001
+ * @param row - The row.
2002
+ * @param index - The field's position.
2003
+ * @returns The number.
2004
+ * @throws NativeParseError when the row has no such field.
2005
+ */
2006
+ function at(row, index) {
2007
+ const value = row.values[index];
2008
+ if (value === void 0) throw new NativeParseError(FORMAT$10, row.line, `the row has no field ${String(index + 1)}`);
2009
+ return value;
2010
+ }
2011
+ /**
2012
+ * Parses an instance of the ORTEC VeRoLog Solver Challenge 2016 to 2017 (Kheiri et al. 2019): `KEY = value` header lines (the dataset and instance names, the planning days, the vehicle capacity, the longest trip, the depot's coordinate id and the three costs), then three sections each opened by `NAME = <count>` and followed by that many rows of whole numbers: `TOOLS` (id, size, how many are available, cost a day), `COORDINATES` (id, x, y) and `REQUESTS` (id, location, first day, last day, days the tools stay, tool kind, how many). Fields are separated by runs of spaces or tabs, and blank lines are skipped. The nodes are the depot, at its coordinate, with the id `depot`, then one node per request under the request's id at the position of its location. The tool table's sizes, availability and costs are not node fields and are not kept; its row count is.
2013
+ * @param text - The file text.
2014
+ * @returns The instance: its header values and tool count as `meta`, and its nodes, the depot first and the requests in file order.
2015
+ * @throws NativeParseError naming the line for an unknown or repeated key, a header value that is not a number where one is due, a missing header line or section, a section with a different number of rows than it says, a row that is not its section's count of whole numbers, a coordinate given twice, a request at a location the file does not give, or a request id given twice.
2016
+ */
2017
+ function parseVerolog2017(text) {
2018
+ const meta = /* @__PURE__ */ new Map();
2019
+ const sections = /* @__PURE__ */ new Map();
2020
+ let open;
2021
+ const close = () => {
2022
+ if (open !== void 0 && open.rows.length !== open.count) throw new NativeParseError(FORMAT$10, open.line, `the ${open.name} section says ${String(open.count)} rows and has ${String(open.rows.length)}`);
2023
+ open = void 0;
2024
+ };
2025
+ for (const [index, raw] of lines(text).entries()) {
2026
+ const line = index + 1;
2027
+ const trimmed = raw.trim();
2028
+ if (trimmed === "") continue;
2029
+ const keyed = /^([A-Z_]+)\s*=\s*(.*)$/.exec(trimmed);
2030
+ if (keyed?.[1] !== void 0 && keyed[2] !== void 0) {
2031
+ close();
2032
+ const [, key, value] = keyed;
2033
+ const section = Object.keys(SECTION_WIDTHS).find((name) => name === key);
2034
+ if (section !== void 0) {
2035
+ if (sections.has(section) || !WHOLE$3.test(value)) throw new NativeParseError(FORMAT$10, line, sections.has(section) ? `the ${section} section is given twice` : `the ${section} section's count ${JSON.stringify(value)} is not a whole number`);
2036
+ const rows = [];
2037
+ sections.set(section, rows);
2038
+ open = {
2039
+ name: section,
2040
+ rows,
2041
+ count: Number(value),
2042
+ line
2043
+ };
2044
+ continue;
2045
+ }
2046
+ const header = HEADER$3.get(key);
2047
+ if (header === void 0 || meta.has(header[0])) throw new NativeParseError(FORMAT$10, line, header === void 0 ? `the key ${key} is not one this format has` : `the key ${key} is given twice`);
2048
+ const [name, kind] = header;
2049
+ if (kind === "number" && !/^-?\d+(?:\.\d+)?$/.test(value)) throw new NativeParseError(FORMAT$10, line, `${key} must be a number, found ${JSON.stringify(value)}`);
2050
+ meta.set(name, kind === "number" ? Number(value) : value);
2051
+ continue;
2052
+ }
2053
+ const tokens = trimmed.split(/\s+/);
2054
+ if (open === void 0 || tokens.length !== SECTION_WIDTHS[open.name] || !tokens.every((token) => WHOLE$3.test(token))) throw new NativeParseError(FORMAT$10, line, open === void 0 ? `the row ${JSON.stringify(trimmed)} is outside any section` : `a ${open.name} row is ${String(SECTION_WIDTHS[open.name])} whole numbers, found ${JSON.stringify(trimmed)}`);
2055
+ open.rows.push({
2056
+ line,
2057
+ values: tokens.map(Number)
2058
+ });
2059
+ }
2060
+ close();
2061
+ for (const [key, [name]] of HEADER$3) if (!meta.has(name)) throw new NativeParseError(FORMAT$10, 1, `the file has no ${key} line`);
2062
+ const rowsOf = (name) => {
2063
+ const rows = sections.get(name);
2064
+ if (rows === void 0) throw new NativeParseError(FORMAT$10, 1, `the file has no ${name} section`);
2065
+ return rows;
2066
+ };
2067
+ const tools = rowsOf("TOOLS").length;
2068
+ const positions = /* @__PURE__ */ new Map();
2069
+ for (const row of rowsOf("COORDINATES")) {
2070
+ const id = at(row, 0);
2071
+ if (positions.has(id)) throw new NativeParseError(FORMAT$10, row.line, `the coordinate ${String(id)} is given twice`);
2072
+ positions.set(id, [at(row, 1), at(row, 2)]);
2073
+ }
2074
+ const positionOf = (location, line) => {
2075
+ const position = positions.get(location);
2076
+ if (position === void 0) throw new NativeParseError(FORMAT$10, line, `the location ${String(location)} is not among the coordinates`);
2077
+ return position;
2078
+ };
2079
+ const depot = Number(meta.get("depotCoordinate"));
2080
+ const [depotX, depotY] = positionOf(depot, 1);
2081
+ const nodes = [{
2082
+ id: "depot",
2083
+ x: depotX,
2084
+ y: depotY,
2085
+ kind: "depot",
2086
+ location: depot,
2087
+ fromDay: null,
2088
+ toDay: null,
2089
+ stayDays: null,
2090
+ tool: null,
2091
+ amount: null
2092
+ }];
2093
+ const ids = /* @__PURE__ */ new Set();
2094
+ for (const row of rowsOf("REQUESTS")) {
2095
+ const id = String(at(row, REQUEST_AT$1.id));
2096
+ if (ids.has(id)) throw new NativeParseError(FORMAT$10, row.line, `the request ${id} is given twice`);
2097
+ ids.add(id);
2098
+ const location = at(row, REQUEST_AT$1.location);
2099
+ const [x, y] = positionOf(location, row.line);
2100
+ nodes.push({
2101
+ id,
2102
+ x,
2103
+ y,
2104
+ kind: "request",
2105
+ location,
2106
+ fromDay: at(row, REQUEST_AT$1.fromDay),
2107
+ toDay: at(row, REQUEST_AT$1.toDay),
2108
+ stayDays: at(row, REQUEST_AT$1.stayDays),
2109
+ tool: String(at(row, REQUEST_AT$1.tool)),
2110
+ amount: at(row, REQUEST_AT$1.amount)
2111
+ });
2112
+ }
2113
+ return {
2114
+ meta: {
2115
+ ...Object.fromEntries(meta),
2116
+ tools
2117
+ },
2118
+ nodes
2119
+ };
2120
+ }
2121
+ //#endregion
1423
2122
  //#region src/parsers/pajek.ts
1424
2123
  /** Section keywords, lower-cased, mapped to how their lines are read. `*Network` names the network and starts no section. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
1425
2124
  const sections = /* @__PURE__ */ new Map([
@@ -2945,28 +3644,60 @@ const KIBIBYTE = 1024;
2945
3644
  /** Bytes decoded at a time for a format that is read in pieces. The decoder holds only a piece's text at once, so this bounds what a large table costs beyond its bytes; it is large enough that the per-piece work is small beside the piece. */
2946
3645
  const DECODE_PIECE_BYTES = 8 * KIBIBYTE * KIBIBYTE;
2947
3646
  /**
2948
- * Decodes a file's bytes in the recipe's encoding a piece at a time, as strictly as `decode`, so a file whose text would not fit in one string can still be read.
2949
- * @param bytes - The unpacked file.
2950
- * @param parser - The recipe's parser, which names the encoding.
2951
- * @param pieceBytes - How many bytes to decode at a time.
2952
- * @returns The text of each piece, in order; a character that spans two pieces is yielded whole with the later one.
2953
- * @throws NativeParseError when the bytes are not valid in the stated encoding.
3647
+ * Decodes a file's bytes in the recipe's encoding as they arrive in pieces, as strictly as `decode`: a character whose bytes span two pieces is decoded whole with the later one, and bytes that are not valid in the stated encoding fail.
2954
3648
  */
2955
- function* decodePieces(bytes, parser, pieceBytes = DECODE_PIECE_BYTES) {
2956
- const decoder = parser.encoding === "latin1" ? void 0 : new TextDecoder("utf-8", { fatal: true });
2957
- try {
2958
- for (let start = 0; start < bytes.length; start += pieceBytes) {
2959
- const piece = bytes.subarray(start, start + pieceBytes);
2960
- yield decoder === void 0 ? decode(piece, parser) : decoder.decode(piece, { stream: true });
2961
- }
2962
- if (decoder !== void 0) {
2963
- const rest = decoder.decode();
2964
- if (rest !== "") yield rest;
3649
+ var PieceDecoder = class {
3650
+ parser;
3651
+ utf8;
3652
+ constructor(parser) {
3653
+ this.parser = parser;
3654
+ this.utf8 = parser.encoding === "latin1" ? void 0 : new TextDecoder("utf-8", { fatal: true });
3655
+ }
3656
+ /**
3657
+ * Decodes the next piece.
3658
+ * @param piece - Bytes that follow the pieces decoded before them.
3659
+ * @returns Their text, without a character the piece ends partway through.
3660
+ * @throws NativeParseError when the bytes are not valid in the stated encoding.
3661
+ */
3662
+ decode(piece) {
3663
+ const { utf8 } = this;
3664
+ return utf8 === void 0 ? decode(piece, this.parser) : this.strictly(() => utf8.decode(piece, { stream: true }));
3665
+ }
3666
+ /**
3667
+ * Ends the file.
3668
+ * @returns The text of any bytes still held.
3669
+ * @throws NativeParseError when the file ends partway through a character.
3670
+ */
3671
+ end() {
3672
+ const { utf8 } = this;
3673
+ return utf8 === void 0 ? "" : this.strictly(() => utf8.decode());
3674
+ }
3675
+ /**
3676
+ * Runs a utf-8 decode, rethrowing its failure in the package's taxonomy: TextDecoder reports invalid bytes only as a TypeError with no position.
3677
+ * @param run - The decode.
3678
+ * @returns Its text.
3679
+ */
3680
+ strictly(run) {
3681
+ try {
3682
+ return run();
3683
+ } catch (error) {
3684
+ throw new NativeParseError(this.parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
2965
3685
  }
2966
- } catch (error) {
2967
- if (error instanceof NativeParseError) throw error;
2968
- throw new NativeParseError(parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
2969
3686
  }
3687
+ };
3688
+ /**
3689
+ * Decodes a file's bytes in the recipe's encoding a piece at a time, as strictly as `decode`, so a file whose text would not fit in one string can still be read.
3690
+ * @param bytes - The unpacked file.
3691
+ * @param parser - The recipe's parser, which names the encoding.
3692
+ * @param pieceBytes - How many bytes to decode at a time.
3693
+ * @returns The text of each piece, in order; a character that spans two pieces is yielded whole with the later one.
3694
+ * @throws NativeParseError when the bytes are not valid in the stated encoding.
3695
+ */
3696
+ function* decodePieces(bytes, parser, pieceBytes = DECODE_PIECE_BYTES) {
3697
+ const decoder = new PieceDecoder(parser);
3698
+ for (let start = 0; start < bytes.length; start += pieceBytes) yield decoder.decode(bytes.subarray(start, start + pieceBytes));
3699
+ const rest = decoder.end();
3700
+ if (rest !== "") yield rest;
2970
3701
  }
2971
3702
  function parseNative(bytes, parser) {
2972
3703
  if (parser.format === "csv-table") return parseCsvTable(decodePieces(bytes, parser), parser);
@@ -2977,10 +3708,10 @@ function parseNative(bytes, parser) {
2977
3708
  split: void 0,
2978
3709
  pieces: decodePieces(bytes, parser)
2979
3710
  }], parser);
2980
- if (parser.format === "hgb") throw new NativeParseError(parser.format, 1, "an HGB dataset is two files, nodes and links, which a recipe reads from a parts archive");
3711
+ if (parser.format === "tdc") return parseTdc(decodePieces(bytes, parser));
3712
+ if (parser.format === "edge-list") return parseEdgeList(decodePieces(bytes, parser), parser);
2981
3713
  const text = decode(bytes, parser);
2982
3714
  if (parser.format === "geojson") return parseGeojson(text, parser);
2983
- if (parser.format === "edge-list") return parseEdgeList(text, parser);
2984
3715
  if (parser.format === "matrix-market") return parseMatrixMarket(text);
2985
3716
  if (parser.format === "gml") return parseGml(text);
2986
3717
  if (parser.format === "ucinet-dl") return parseUcinetDl(text, parser);
@@ -2994,6 +3725,9 @@ function parseNative(bytes, parser) {
2994
3725
  if (parser.format === "verolog-2019") return parseVerolog2019(text);
2995
3726
  if (parser.format === "vrp-rep") return parseVrpRep(text);
2996
3727
  if (parser.format === "hhcrsp") return parseHhcrsp(text);
3728
+ if (parser.format === "ogb") throw new NativeParseError(parser.format, 1, "the format reads the several files a files archive selects, not one file");
3729
+ if (parser.format === "mankowska") return parseMankowska(text);
3730
+ if (parser.format === "verolog-2017") return parseVerolog2017(text);
2997
3731
  return parsePajek(text);
2998
3732
  }
2999
3733
  /**
@@ -3019,6 +3753,20 @@ function parseNativeParts(parts, parser) {
3019
3753
  }
3020
3754
  throw new NativeParseError(parser.format, 1, `the ${parser.format} parser reads one file, not named parts`);
3021
3755
  }
3756
+ /**
3757
+ * Reads the several files of a multi-file format into the one native document they make together.
3758
+ * @param files - The files a `files` archive selects, each with its path in the archive, in path order.
3759
+ * @param parser - The recipe's parser, which must be a multi-file format.
3760
+ * @returns The native graph.
3761
+ * @throws NativeParseError when a file is not valid in the stated encoding or the files are not in the stated format, or when the parser reads one file and not several.
3762
+ */
3763
+ function parseNativeFiles(files, parser) {
3764
+ if (parser.format !== "ogb") throw new NativeParseError(parser.format, 1, "the format reads one file, not the several a files archive selects");
3765
+ return parseOgb(files.map((file) => ({
3766
+ path: file.path,
3767
+ text: () => decodePieces(file.bytes, parser)
3768
+ })));
3769
+ }
3022
3770
  //#endregion
3023
3771
  //#region src/node/bzip2.ts
3024
3772
  /** The four bytes every bzip2 stream starts with, as ASCII: `BZh` and the block size digit, `1` to `9`. */
@@ -4798,18 +5546,20 @@ function unpackShapefile(bytes, member) {
4798
5546
  };
4799
5547
  }
4800
5548
  /**
4801
- * Unpacks downloaded bytes into every file a collection's selector matches, or, with an `xlsx` inner layout, into every sheet of every matched workbook that its `sheets` pattern matches as the sheet's CSV, at the member's path and the sheet's name joined by `/`, members in path order and sheets in the workbook's order.
5549
+ * Unpacks downloaded bytes into every file a collection's selector matches, or, with an `xlsx` inner layout, into every sheet of every matched workbook that its `sheets` pattern matches as the sheet's CSV, at the member's path and the sheet's name joined by `/`, members in path order and sheets in the workbook's order. The files are given one at a time: a zip member is inflated only when it is reached, so a reader that lets each file go before taking the next holds one member's bytes beside the archive, never every member at once.
4802
5550
  * @param bytes - The downloaded bytes.
4803
5551
  * @param archive - The recipe's archive layout, which selects members with `members`.
4804
5552
  * @returns Each file.
4805
- * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind, the selector matches no member, or a member an `xlsx` layout reads is not a workbook or has no sheet the pattern matches.
5553
+ * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind, the selector matches no member (at the first file taken), or a member an `xlsx` layout reads is not a workbook or has no sheet the pattern matches (at that member).
4806
5554
  */
4807
- function unpackMembers(bytes, archive) {
4808
- const members = selectMembers(bytes, archive);
4809
- if (archive.inner === void 0) return members;
4810
- const { sheets } = archive.inner;
5555
+ function* unpackMembers(bytes, archive) {
5556
+ const { sheets } = archive.inner ?? {};
4811
5557
  const wanted = sheets === void 0 ? void 0 : new RegExp(`^(?:${sheets})$`, "u");
4812
- return members.flatMap((member) => {
5558
+ for (const member of selectMembers(bytes, archive)) {
5559
+ if (archive.inner === void 0) {
5560
+ yield member;
5561
+ continue;
5562
+ }
4813
5563
  let names;
4814
5564
  try {
4815
5565
  names = sheetNames(member.bytes).filter((sheet) => wanted === void 0 || wanted.test(sheet));
@@ -4817,24 +5567,24 @@ function unpackMembers(bytes, archive) {
4817
5567
  throw new ArchiveError(`member ${member.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
4818
5568
  }
4819
5569
  if (names.length === 0) throw new ArchiveError(`member ${member.path} has no sheet matching ${sheets ?? ""}`);
4820
- return names.map((sheet) => ({
5570
+ yield* names.map((sheet) => ({
4821
5571
  path: `${member.path}/${sheet}`,
4822
5572
  bytes: sheetToCsv(member.bytes, {
4823
5573
  kind: "xlsx",
4824
5574
  sheet
4825
5575
  })
4826
5576
  }));
4827
- });
5577
+ }
4828
5578
  }
4829
5579
  /**
4830
- * Unpacks every member a collection's selector matches.
5580
+ * Unpacks every member a collection's selector matches, one at a time.
4831
5581
  * @param bytes - The downloaded bytes.
4832
5582
  * @param archive - The recipe's archive layout.
4833
5583
  * @returns Each matching member, in path order.
4834
- * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member.
5584
+ * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member, at the first member taken.
4835
5585
  */
4836
- function selectMembers(bytes, archive) {
4837
- return readMembers(bytes, archive.kind, (paths) => pickAll(paths, archive.members).map((path) => ({ path })));
5586
+ function* selectMembers(bytes, archive) {
5587
+ yield* readMembers(bytes, archive.kind, (paths) => pickAll(paths, archive.members).map((path) => ({ path })));
4838
5588
  }
4839
5589
  /**
4840
5590
  * Unpacks downloaded bytes into the files a parts archive names, each the one member its selector matches.
@@ -4844,57 +5594,45 @@ function selectMembers(bytes, archive) {
4844
5594
  * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or a part's selector does not match exactly one member.
4845
5595
  */
4846
5596
  function unpackParts(bytes, archive) {
4847
- return readMembers(bytes, archive.kind, (paths) => archive.parts.map((part) => ({
5597
+ return [...readMembers(bytes, archive.kind, (paths) => archive.parts.map((part) => ({
4848
5598
  name: part.name,
4849
5599
  path: pick(paths, part.member)
4850
- })));
5600
+ })))];
4851
5601
  }
4852
5602
  /**
4853
- * Reads the members of a file archive that a choice makes from its member paths, listing the archive once and decompressing it once however many members are chosen.
4854
- * @param bytes - The downloaded bytes.
5603
+ * Reads the members of a file archive that a choice makes from its member paths, one at a time: the archive is listed once and a zip member is inflated only when it is reached, so a reader that lets each member go before taking the next holds one member's bytes beside the archive.
5604
+ * @param bytes - The archive's bytes.
4855
5605
  * @param kind - The archive kind.
4856
5606
  * @param choose - Given every member path, the members to read, in the order wanted, each naming its path.
4857
5607
  * @returns Each choice with the bytes of the member it names, in the order chosen.
4858
5608
  * @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or a chosen member cannot be read; whatever `choose` throws.
4859
5609
  */
4860
- function readMembers(bytes, kind, choose) {
5610
+ function* readMembers(bytes, kind, choose) {
4861
5611
  if (kind === "zip") {
4862
5612
  const names = [];
4863
5613
  unzip(bytes, (file) => {
4864
5614
  names.push(file.name);
4865
5615
  return false;
4866
5616
  });
4867
- const choices = choose(names);
4868
- const chosen = new Set(choices.map((choice) => choice.path));
4869
- const files = unzip(bytes, (candidate) => chosen.has(candidate.name));
4870
- return choices.map((choice) => {
4871
- const file = getOwn(files, choice.path);
5617
+ for (const choice of choose(names)) {
5618
+ const file = getOwn(unzip(bytes, (candidate) => candidate.name === choice.path), choice.path);
4872
5619
  if (file === void 0) throw new ArchiveError(`zip member ${choice.path} could not be read`);
4873
- return {
5620
+ yield {
4874
5621
  ...choice,
4875
5622
  bytes: file
4876
5623
  };
4877
- });
5624
+ }
5625
+ return;
4878
5626
  }
4879
5627
  const members = readTar(untarred(bytes, kind));
4880
- return choose(members.map((member) => member.path)).map((choice) => {
5628
+ for (const choice of choose(members.map((member) => member.path))) {
4881
5629
  const found = members.find((member) => member.path === choice.path);
4882
5630
  if (found === void 0) throw new ArchiveError(`tar member ${choice.path} could not be read`);
4883
- return {
5631
+ yield {
4884
5632
  ...choice,
4885
5633
  bytes: found.data
4886
5634
  };
4887
- });
4888
- }
4889
- /**
4890
- * Removes the compression a tar archive kind states, leaving the tar bytes.
4891
- * @param bytes - The downloaded bytes.
4892
- * @param kind - The tar archive kind: `tar` is uncompressed, `tar.gz` is gzip and `tar.bz2` is bzip2.
4893
- * @returns The tar archive.
4894
- */
4895
- function untarred(bytes, kind) {
4896
- if (kind === "tar") return bytes;
4897
- return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
5635
+ }
4898
5636
  }
4899
5637
  /**
4900
5638
  * Reads one sheet of a spreadsheet as CSV text, by the spreadsheet's format.
@@ -4912,6 +5650,41 @@ function sheetToCsv(bytes, archive) {
4912
5650
  return gridToCsv(read(bytes, archive.sheet));
4913
5651
  }
4914
5652
  /**
5653
+ * Unpacks downloaded bytes into every file a multi-file format's selector matches, gunzipping each when the layout's `inner` says so. The paths stay the archive's own, so a gunzipped `edge.csv.gz` keeps its name.
5654
+ * @param bytes - The downloaded bytes.
5655
+ * @param archive - The recipe's archive layout, which selects files with `files`.
5656
+ * @returns Each matching file, in path order.
5657
+ * @throws ArchiveError when the bytes, or a file `inner` gunzips, are not well-formed, or the selector matches no member.
5658
+ */
5659
+ function unpackFiles(bytes, archive) {
5660
+ const files = [...unpackMembers(bytes, {
5661
+ kind: archive.kind,
5662
+ members: archive.files
5663
+ })];
5664
+ if (archive.inner === void 0) return files;
5665
+ return files.map((file) => {
5666
+ try {
5667
+ return {
5668
+ path: file.path,
5669
+ bytes: gunzip(file.bytes)
5670
+ };
5671
+ } catch (error) {
5672
+ throw new ArchiveError(`member ${file.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
5673
+ }
5674
+ });
5675
+ }
5676
+ /**
5677
+ * Unpacks a tar archive's bytes, which may be compressed: `tar.gz` is gunzipped and `tar.bz2` bunzipped first, `tar` read as it is.
5678
+ * @param bytes - The archive's bytes.
5679
+ * @param kind - The archive kind.
5680
+ * @returns The tar's bytes.
5681
+ * @throws ArchiveError when the bytes are not valid gzip or bzip2 data, as `gunzip` and `bunzip2` report.
5682
+ */
5683
+ function untarred(bytes, kind) {
5684
+ if (kind === "tar") return bytes;
5685
+ return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
5686
+ }
5687
+ /**
4915
5688
  * Gunzips bytes, rethrowing zlib's failure in the package's taxonomy.
4916
5689
  * @param bytes - Gzip data.
4917
5690
  * @returns The decompressed bytes.
@@ -5177,15 +5950,25 @@ async function getBytes(port, url, signal) {
5177
5950
  return new Uint8Array(await response.arrayBuffer());
5178
5951
  }
5179
5952
  /**
5180
- * Reads a source's pages, each starting where the rows held so far end, and checks each holds exactly the rows it should, so a response cut short is an error and not a table with rows missing.
5953
+ * Reads a source's pages in order, each starting where the rows read so far end, and checks each holds exactly the rows it should, so a response cut short is an error and not a table with rows missing. Only the page being read and the first page's header are held, so a caller that hands each page on holds no more of the source than that.
5181
5954
  * @param read - The source read.
5182
- * @param held - The pages read so far, each after the first without its header record.
5183
- * @param offset - How many rows the pages hold.
5184
- * @returns Every page.
5185
- * @throws KeepsetError when a page holds a different number of rows than the count says it should, or its header differs from the first page's.
5955
+ * @returns Each page, the first with its header record and every later one without it.
5956
+ * @throws KeepsetError when the source has no rows, a page holds a different number of rows than the count says it should, or its header differs from the first page's.
5186
5957
  */
5187
- async function readResourcePages(read, held, offset) {
5188
- if (offset >= read.count) return held;
5958
+ async function* resourcePages(read) {
5959
+ if (read.count === 0) throw new KeepsetError(`${read.resource} has no rows`);
5960
+ yield* pagesFrom(read, 0, void 0);
5961
+ }
5962
+ /**
5963
+ * Reads one page and then, by recursion, the pages after it.
5964
+ * @param read - The source read.
5965
+ * @param offset - The row the page starts at.
5966
+ * @param header - The first page's header record, or undefined when this is the first page.
5967
+ * @returns This page and every page after it.
5968
+ * @throws KeepsetError as `resourcePages` does.
5969
+ */
5970
+ async function* pagesFrom(read, offset, header) {
5971
+ if (offset >= read.count) return;
5189
5972
  const expected = Math.min(read.pageSize, read.count - offset);
5190
5973
  const page = await withAttempts(async () => {
5191
5974
  const bytes = await getBytes(read.port, read.pageUrl(offset), read.signal);
@@ -5193,10 +5976,15 @@ async function readResourcePages(read, held, offset) {
5193
5976
  if (rows !== expected) throw new KeepsetError(`${read.resource} gave ${String(rows)} rows at offset ${String(offset)} where ${String(expected)} of its ${String(read.count)} were due, so a response was cut short or the table changed during the download`);
5194
5977
  return bytes;
5195
5978
  }, read.signal);
5196
- const first = held[0];
5197
5979
  const headerEnd = page.indexOf(LINE_FEED) + 1;
5198
- if (first !== void 0 && !page.subarray(0, headerEnd).every((byte, index) => byte === first[index])) throw new KeepsetError(`${read.resource} gave a different header at offset ${String(offset)} than at the start`);
5199
- return readResourcePages(read, [...held, first === void 0 ? page : page.subarray(headerEnd)], offset + expected);
5980
+ if (header === void 0) {
5981
+ yield page;
5982
+ yield* pagesFrom(read, offset + expected, page.slice(0, headerEnd));
5983
+ return;
5984
+ }
5985
+ if (headerEnd !== header.length || !page.subarray(0, headerEnd).every((byte, index) => byte === header[index])) throw new KeepsetError(`${read.resource} gave a different header at offset ${String(offset)} than at the start`);
5986
+ yield page.subarray(headerEnd);
5987
+ yield* pagesFrom(read, offset + expected, header);
5200
5988
  }
5201
5989
  /**
5202
5990
  * Downloads a CSV source a page at a time and joins the pages under the first header.
@@ -5205,42 +5993,44 @@ async function readResourcePages(read, held, offset) {
5205
5993
  * @throws KeepsetError when the source has no rows or a page is not what it should be.
5206
5994
  */
5207
5995
  async function downloadPages(read) {
5208
- const pages = await readResourcePages(read, [], 0);
5209
- if (pages.length === 0) throw new KeepsetError(`${read.resource} has no rows`);
5996
+ const pages = [];
5997
+ for await (const page of resourcePages(read)) pages.push(page);
5210
5998
  return Buffer.concat(pages);
5211
5999
  }
5212
6000
  /**
5213
- * Downloads a whole Socrata resource as one CSV: the row count is read first, then pages of `pageSize` rows in the order of the rows' ids, each checked to hold the rows it should, and joined under the first page's header.
6001
+ * How a Socrata resource is read: its row count is read first, then pages of `pageSize` rows in the order of the rows' ids, with only the `select` columns when the recipe lists them, so the columns a parser drops are never transferred.
5214
6002
  * @param resource - The resource's URL, such as `https://data.example.org/resource/abcd-1234.csv`, without a query.
5215
- * @param pageSize - How many rows a page holds.
6003
+ * @param paging - The recipe's paging.
5216
6004
  * @param port - The network port.
5217
6005
  * @param signal - Aborts the download.
5218
- * @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
5219
- * @throws KeepsetError when the resource does not report a count, or a page is not what it should be.
6006
+ * @returns The source read.
6007
+ * @throws KeepsetError when the resource does not report a count.
5220
6008
  */
5221
- async function downloadSocrataResource(resource, pageSize, port, signal) {
6009
+ async function socrataRead(resource, paging, port, signal) {
5222
6010
  const reported = /^"count"\r?\n"(\d+)"/.exec(new TextDecoder("utf-8").decode(await getBytes(port, `${resource}?%24select=count(*)`, signal)));
5223
6011
  if (reported?.[1] === void 0) throw new KeepsetError(`${resource} did not report a row count`);
5224
- return downloadPages({
6012
+ const { pageSize, select } = paging;
6013
+ const columns = select === void 0 ? "" : `%24select=${encodeURIComponent(select.join(","))}&`;
6014
+ return {
5225
6015
  resource,
5226
- pageUrl: (offset) => `${resource}?%24order=%3Aid&%24limit=${String(pageSize)}&%24offset=${String(offset)}`,
6016
+ pageUrl: (offset) => `${resource}?${columns}%24order=%3Aid&%24limit=${String(pageSize)}&%24offset=${String(offset)}`,
5227
6017
  pageSize,
5228
6018
  count: Number(reported[1]),
5229
6019
  port,
5230
6020
  signal
5231
- });
6021
+ };
5232
6022
  }
5233
6023
  const CountResultSchema = z.object({ result: z.object({ total: z.int().min(0) }) });
5234
6024
  /**
5235
- * Downloads a whole CKAN datastore table as one CSV: the row count is read from the portal's datastore API, then pages of `pageSize` rows of the table's CSV dump, each checked to hold the rows it should, and joined under the first page's header.
6025
+ * How a CKAN datastore table is read: the row count is read from the portal's datastore API, then pages of `pageSize` rows of the table's CSV dump.
5236
6026
  * @param dump - The dump's URL, such as `https://data.example.org/datastore/dump/<resource id>`, without a query; a portal that serves CKAN under a path prefix keeps it before `/datastore/dump/`.
5237
6027
  * @param pageSize - How many rows a page holds.
5238
6028
  * @param port - The network port.
5239
6029
  * @param signal - Aborts the download.
5240
- * @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
5241
- * @throws KeepsetError when the URL is not a datastore dump, the table does not report a count, or a page is not what it should be.
6030
+ * @returns The source read.
6031
+ * @throws KeepsetError when the URL is not a datastore dump or the table does not report a count.
5242
6032
  */
5243
- async function downloadCkanDatastore(dump, pageSize, port, signal) {
6033
+ async function ckanRead(dump, pageSize, port, signal) {
5244
6034
  const parsed = /^(.*)\/datastore\/dump\/([^/?#]+)$/.exec(dump);
5245
6035
  if (parsed?.[1] === void 0 || parsed[2] === void 0) throw new KeepsetError(`${dump} is not a CKAN datastore dump URL`);
5246
6036
  const url = `${parsed[1]}/api/3/action/datastore_search?resource_id=${parsed[2]}&limit=0`;
@@ -5252,26 +6042,26 @@ async function downloadCkanDatastore(dump, pageSize, port, signal) {
5252
6042
  }
5253
6043
  const count = CountResultSchema.safeParse(body);
5254
6044
  if (!count.success) throw new KeepsetError(`${dump} did not report a row count`);
5255
- return downloadPages({
6045
+ return {
5256
6046
  resource: dump,
5257
6047
  pageUrl: (offset) => `${dump}?format=csv&limit=${String(pageSize)}&offset=${String(offset)}`,
5258
6048
  pageSize,
5259
6049
  count: count.data.result.total,
5260
6050
  port,
5261
6051
  signal
5262
- });
6052
+ };
5263
6053
  }
5264
6054
  const OpenFemaCountSchema = z.object({ metadata: z.object({ count: z.int().min(0) }) });
5265
6055
  /**
5266
- * Downloads a whole OpenFEMA entity, or the rows of it the source URL selects, as one CSV: the row count is read from the API's own metadata, then pages of `pageSize` rows in the order of the entity's id, each checked to hold the rows it should, and joined under the first page's header.
6056
+ * How an OpenFEMA entity, or the rows of it the source URL selects, is read: the row count is read from the API's own metadata, then pages of `pageSize` rows in the order of the entity's id.
5267
6057
  * @param entity - The entity's URL with any `$select` or `$filter` but no `$top`, `$skip`, `$orderby` or `$format`, such as `https://www.fema.gov/api/open/v3/NfipClaims?$select=id,state`.
5268
6058
  * @param pageSize - How many rows a page holds.
5269
6059
  * @param port - The network port.
5270
6060
  * @param signal - Aborts the download.
5271
- * @returns The bytes of one CSV, which is what the parser reads and what the provenance hashes.
5272
- * @throws KeepsetError when the source URL sets a paging parameter, the entity does not report a count, or a page is not what it should be.
6061
+ * @returns The source read.
6062
+ * @throws KeepsetError when the source URL sets a paging parameter or the entity does not report a count.
5273
6063
  */
5274
- async function downloadOpenFema(entity, pageSize, port, signal) {
6064
+ async function openFemaRead(entity, pageSize, port, signal) {
5275
6065
  const base = new URL(entity);
5276
6066
  for (const name of [
5277
6067
  "$top",
@@ -5297,7 +6087,7 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
5297
6087
  }
5298
6088
  const count = OpenFemaCountSchema.safeParse(body);
5299
6089
  if (!count.success) throw new KeepsetError(`${entity} did not report a row count`);
5300
- return downloadPages({
6090
+ return {
5301
6091
  resource: entity,
5302
6092
  pageUrl: (offset) => withParameters({
5303
6093
  $format: "csv",
@@ -5309,10 +6099,34 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
5309
6099
  count: count.data.metadata.count,
5310
6100
  port,
5311
6101
  signal
5312
- });
6102
+ };
5313
6103
  }
5314
6104
  /**
5315
- * Downloads a paged source as the one file its pages make, in the way its recipe's paging says.
6105
+ * How a CSV paged source is read, in the way its recipe's paging says.
6106
+ * @param url - The source's URL.
6107
+ * @param paging - How the source is paged.
6108
+ * @param port - The network port.
6109
+ * @param signal - Aborts the download.
6110
+ * @returns The source read.
6111
+ */
6112
+ async function csvRead(url, paging, port, signal) {
6113
+ if (paging.kind === "socrata-resource") return socrataRead(url, paging, port, signal);
6114
+ return paging.kind === "ckan-datastore" ? ckanRead(url, paging.pageSize, port, signal) : openFemaRead(url, paging.pageSize, port, signal);
6115
+ }
6116
+ /**
6117
+ * Reads a CSV paged source a page at a time: the first page with its header and every later page without it, so the pages in order are the one CSV `downloadPaged` joins them into. Only the page being read is held.
6118
+ * @param url - The source's URL.
6119
+ * @param paging - How the source is paged.
6120
+ * @param port - The network port.
6121
+ * @param signal - Aborts the download.
6122
+ * @returns Each page's bytes, in order.
6123
+ * @throws KeepsetError when the source does not report a count, has no rows, or a page is not what it should be.
6124
+ */
6125
+ async function* csvPages(url, paging, port, signal) {
6126
+ yield* resourcePages(await csvRead(url, paging, port, signal));
6127
+ }
6128
+ /**
6129
+ * Downloads a paged source as the one file its pages make, in the way its recipe's paging says: an ArcGIS layer as one GeoJSON feature collection, any other source as one CSV of its pages joined under the first page's header.
5316
6130
  * @param url - The source's URL.
5317
6131
  * @param paging - How the source is paged.
5318
6132
  * @param port - The network port.
@@ -5322,17 +6136,16 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
5322
6136
  */
5323
6137
  async function downloadPaged(url, paging, port, signal) {
5324
6138
  if (paging.kind === "arcgis-query") return downloadArcgisLayer(url, paging.orderBy, port, signal);
5325
- if (paging.kind === "socrata-resource") return downloadSocrataResource(url, paging.pageSize, port, signal);
5326
- return paging.kind === "ckan-datastore" ? downloadCkanDatastore(url, paging.pageSize, port, signal) : downloadOpenFema(url, paging.pageSize, port, signal);
6139
+ return downloadPages(await csvRead(url, paging, port, signal));
5327
6140
  }
5328
6141
  //#endregion
5329
- //#region src/node/fetch-dataset.ts
6142
+ //#region src/node/recipe-run.ts
5330
6143
  /**
5331
- * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
6144
+ * The URL a recipe downloads a source from, as `downloadUrl` gives it. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
5332
6145
  * @param source - The source.
5333
6146
  * @returns The URL to request.
5334
6147
  */
5335
- function downloadUrl(source) {
6148
+ function sourceUrl(source) {
5336
6149
  if (source.status === "dead") {
5337
6150
  if (source.snapshot === void 0) throw new KeepsetError(`dead source ${source.url} has no snapshot`);
5338
6151
  return `https://web.archive.org/web/${source.snapshot.timestamp}id_/${source.url}`;
@@ -5340,6 +6153,18 @@ function downloadUrl(source) {
5340
6153
  return source.url;
5341
6154
  }
5342
6155
  /**
6156
+ * The source record a provenance carries: the entry's source as recorded.
6157
+ * @param source - The source.
6158
+ * @returns Its URL, status and any snapshot.
6159
+ */
6160
+ function sourceRecord(source) {
6161
+ return {
6162
+ url: source.url,
6163
+ status: source.status,
6164
+ ...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
6165
+ };
6166
+ }
6167
+ /**
5343
6168
  * Runs one step, rethrowing any failure as a RecipeError naming the entry and the step, with the original error as its cause.
5344
6169
  * @param entryId - The entry.
5345
6170
  * @param name - The step.
@@ -5366,6 +6191,373 @@ function metaOf(entry) {
5366
6191
  ...entry.properties
5367
6192
  };
5368
6193
  }
6194
+ //#endregion
6195
+ //#region src/node/id-set.ts
6196
+ /** The first table size, in slots: 2 to the 16th, a power of two, so a hash is reduced to a slot by masking (and every doubling keeps it one). */
6197
+ const INITIAL_SLOTS = 65536;
6198
+ /** The first arena size, in bytes: a mebibyte, which a stream of a few tens of thousands of short ids fills before the arena first doubles. */
6199
+ const INITIAL_ARENA_BYTES = 1048576;
6200
+ /** The table grows when more than this share of its slots hold an id, which keeps linear probing short. */
6201
+ const MAX_LOAD = .7;
6202
+ /** The bytes before each id in the arena: its length, as an unsigned 32-bit integer. */
6203
+ const LENGTH_BYTES = 4;
6204
+ /** The largest arena, 2 to the 31st less one: slots hold an arena offset plus one in a signed 32-bit integer. */
6205
+ const MAX_ARENA_BYTES = 2147483647;
6206
+ /** The FNV-1a 32-bit offset basis and prime (http://www.isthe.com/chongo/tech/comp/fnv/). */
6207
+ const FNV_OFFSET = 2166136261;
6208
+ const FNV_PRIME = 16777619;
6209
+ const encoder = new TextEncoder();
6210
+ /**
6211
+ * The FNV-1a hash of bytes.
6212
+ * @param bytes - The bytes.
6213
+ * @returns The 32-bit hash.
6214
+ */
6215
+ function fnv1a(bytes) {
6216
+ let hash = FNV_OFFSET;
6217
+ for (const byte of bytes) hash = Math.imul(hash ^ byte, FNV_PRIME);
6218
+ return hash >>> 0;
6219
+ }
6220
+ /**
6221
+ * Reads a slot of a table.
6222
+ * @param slots - The table.
6223
+ * @param slot - A slot index, always masked into the table's range.
6224
+ * @returns What the slot holds.
6225
+ * @throws RangeError when the index is outside the table, which masking never gives.
6226
+ */
6227
+ function slotOf(slots, slot) {
6228
+ const held = slots[slot];
6229
+ if (held === void 0) throw new RangeError(`slot ${String(slot)} is outside a table of ${String(slots.length)}`);
6230
+ return held;
6231
+ }
6232
+ /**
6233
+ * A set of ids that holds each id as its UTF-8 bytes in one growing arena, indexed by an open-addressed table of arena offsets, so a stream of millions of ids is checked for repeats exactly while holding a few bytes beyond each id's own and no JavaScript string or object per id. Membership compares the bytes themselves, so two different ids are never taken for one.
6234
+ */
6235
+ var IdSet = class {
6236
+ arena = new Uint8Array(INITIAL_ARENA_BYTES);
6237
+ view = new DataView(this.arena.buffer);
6238
+ used = 0;
6239
+ /** Each slot holds an id's arena offset plus one, or zero when empty. */
6240
+ slots = new Int32Array(INITIAL_SLOTS);
6241
+ count = 0;
6242
+ /** @returns How many ids the set holds. */
6243
+ get size() {
6244
+ return this.count;
6245
+ }
6246
+ /**
6247
+ * Adds an id.
6248
+ * @param id - The id.
6249
+ * @returns True when the id was new, false when the set already held it.
6250
+ * @throws KeepsetError when the ids held would pass the largest arena.
6251
+ */
6252
+ add(id) {
6253
+ const bytes = encoder.encode(id);
6254
+ const hash = fnv1a(bytes);
6255
+ const mask = this.slots.length - 1;
6256
+ for (let slot = hash & mask;; slot = slot + 1 & mask) {
6257
+ const held = slotOf(this.slots, slot);
6258
+ if (held === 0) {
6259
+ this.slots[slot] = this.store(bytes) + 1;
6260
+ this.count += 1;
6261
+ if (this.count > this.slots.length * MAX_LOAD) this.grow();
6262
+ return true;
6263
+ }
6264
+ if (this.equals(held - 1, bytes)) return false;
6265
+ }
6266
+ }
6267
+ /**
6268
+ * Whether the id stored at an offset is these bytes.
6269
+ * @param offset - The id's arena offset.
6270
+ * @param bytes - The bytes.
6271
+ * @returns True when they are the same.
6272
+ */
6273
+ equals(offset, bytes) {
6274
+ if (this.lengthAt(offset) !== bytes.length) return false;
6275
+ const start = offset + LENGTH_BYTES;
6276
+ return bytes.every((byte, index) => this.arena[start + index] === byte);
6277
+ }
6278
+ /**
6279
+ * The length of the id stored at an offset.
6280
+ * @param offset - The id's arena offset.
6281
+ * @returns Its byte length.
6282
+ */
6283
+ lengthAt(offset) {
6284
+ return this.view.getUint32(offset, true);
6285
+ }
6286
+ /**
6287
+ * Appends an id's bytes to the arena.
6288
+ * @param bytes - The bytes.
6289
+ * @returns Their offset.
6290
+ */
6291
+ store(bytes) {
6292
+ const needed = this.used + LENGTH_BYTES + bytes.length;
6293
+ if (needed > MAX_ARENA_BYTES) throw new KeepsetError(`the ids of the stream pass ${String(MAX_ARENA_BYTES)} bytes, more than one id set holds`);
6294
+ if (needed > this.arena.length) {
6295
+ const grown = new Uint8Array(Math.min(MAX_ARENA_BYTES, Math.max(needed, this.arena.length * 2)));
6296
+ grown.set(this.arena.subarray(0, this.used));
6297
+ this.arena = grown;
6298
+ this.view = new DataView(grown.buffer);
6299
+ }
6300
+ const offset = this.used;
6301
+ this.view.setUint32(offset, bytes.length, true);
6302
+ this.arena.set(bytes, offset + LENGTH_BYTES);
6303
+ this.used = needed;
6304
+ return offset;
6305
+ }
6306
+ /** Doubles the table, placing every id again by its hash, which is recomputed from the arena so the table holds nothing but offsets. */
6307
+ grow() {
6308
+ const slots = new Int32Array(this.slots.length * 2);
6309
+ const mask = slots.length - 1;
6310
+ for (const held of this.slots) if (held !== 0) {
6311
+ const start = held - 1 + LENGTH_BYTES;
6312
+ let slot = fnv1a(this.arena.subarray(start, start + this.lengthAt(held - 1))) & mask;
6313
+ while (slotOf(slots, slot) !== 0) slot = slot + 1 & mask;
6314
+ slots[slot] = held;
6315
+ }
6316
+ this.slots = slots;
6317
+ }
6318
+ };
6319
+ //#endregion
6320
+ //#region src/node/stream-dataset.ts
6321
+ /**
6322
+ * The parts of a recipe a stream reads, checked to be a streamed recipe: the entry schema refuses a `stream` recipe that is not a CSV paged source read by the csv-table parser with no archive, so this only fails for an entry whose recipe does not stream.
6323
+ * @param entry - The entry.
6324
+ * @returns The recipe's source, paging and parser, and the category.
6325
+ * @throws RecipeError when the recipe is not fetchable, does not stream, or its category has no streamed form.
6326
+ */
6327
+ function streamedRecipe(entry) {
6328
+ const { recipe, category } = entry;
6329
+ if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
6330
+ if ("files" in recipe || recipe.stream !== true) throw new RecipeError(entry.id, "recipe", "the recipe is not streamed, so fetchDataset reads it as one document");
6331
+ const { paging, parser } = recipe;
6332
+ if (paging === void 0 || paging.kind === "arcgis-query" || parser.format !== "csv-table" || !isStreamable(category)) throw new RecipeError(entry.id, "recipe", "a streamed recipe reads a CSV paged source through the csv-table parser, for a category with a streamed form");
6333
+ return {
6334
+ source: recipe.source,
6335
+ paging,
6336
+ parser,
6337
+ category
6338
+ };
6339
+ }
6340
+ /**
6341
+ * Whether a recipe is streamed, so `fetchDataset` refuses it and `streamDataset` reads it.
6342
+ * @param recipe - A recipe.
6343
+ * @returns True for a streamed recipe.
6344
+ */
6345
+ function isStreamed(recipe) {
6346
+ return recipe.status === "fetchable" && !("files" in recipe) && recipe.stream === true;
6347
+ }
6348
+ /**
6349
+ * What a crosswalk path addresses, for the separability check: the document's meta, a field of every node, or anything else.
6350
+ * @param path - A crosswalk path.
6351
+ * @returns Its class.
6352
+ */
6353
+ function pathClass(path) {
6354
+ if (path === "meta" || path.startsWith("meta.")) return "meta";
6355
+ return path === "nodes[]" || path.startsWith("nodes[].") ? "node" : "other";
6356
+ }
6357
+ /**
6358
+ * Why a crosswalk cannot be applied a page of nodes at a time, if it cannot: every rule must read and write the meta alone or one node's fields alone, with no `when`, so encoding a page's nodes under the document's meta gives exactly that page's part of encoding the whole document, and a round trip that holds on every page holds on the whole.
6359
+ * @param crosswalk - The crosswalk.
6360
+ * @returns The first rule that breaks this, described, or undefined when every rule keeps to it.
6361
+ */
6362
+ function inseparableRule(crosswalk) {
6363
+ for (const [index, rule] of [...crosswalk.rules, ...crosswalk.oneWay].entries()) {
6364
+ if ("when" in rule && rule.when !== void 0) return `rule ${String(index)} (${rule.op}) has a when, which may read the whole document`;
6365
+ const paths = ["from" in rule ? rule.from : void 0, "to" in rule ? rule.to : void 0].filter((path) => path !== void 0);
6366
+ const classes = new Set(paths.map(pathClass));
6367
+ const [only] = classes;
6368
+ if (classes.size !== 1 || only === "other") return `rule ${String(index)} (${rule.op}) reads or writes ${paths.join(" and ")}, not the meta alone or one node's fields alone`;
6369
+ }
6370
+ }
6371
+ /**
6372
+ * Passes on what an async iterable yields, rethrowing its failure as a RecipeError naming the entry and the step, as `step` does for one call.
6373
+ * @param entryId - The entry.
6374
+ * @param name - The step.
6375
+ * @param source - What the step yields.
6376
+ * @returns The same items, in order.
6377
+ */
6378
+ async function* inStep(entryId, name, source) {
6379
+ try {
6380
+ yield* source;
6381
+ } catch (error) {
6382
+ if (error instanceof RecipeError) throw error;
6383
+ throw new RecipeError(entryId, name, error instanceof Error ? error.message : String(error), { cause: error });
6384
+ }
6385
+ }
6386
+ /**
6387
+ * Runs a streamed recipe's download and parse steps a page at a time: each page's bytes are hashed, decoded and read into rows, and then let go.
6388
+ * @param dataset - The dataset.
6389
+ * @param recipe - Its streamed recipe.
6390
+ * @param options - The network port, abort signal and clock.
6391
+ * @returns The stream.
6392
+ */
6393
+ function openNative(dataset, recipe, options) {
6394
+ const { entry } = dataset;
6395
+ const { fetch: port = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
6396
+ const source = entry.sources[recipe.source];
6397
+ if (source === void 0) throw new RecipeError(entry.id, "recipe", `the recipe names source ${String(recipe.source)}, which the entry does not have`);
6398
+ const fetchedUrl = sourceUrl(source);
6399
+ const retrievedAt = now().toISOString();
6400
+ const hash = createHash("sha256");
6401
+ let byteLength = 0;
6402
+ async function* pages() {
6403
+ const decoder = new PieceDecoder(recipe.parser);
6404
+ const reader = new TableReader(recipe.parser);
6405
+ for await (const page of inStep(entry.id, "download", csvPages(fetchedUrl, recipe.paging, port, signal))) {
6406
+ hash.update(page);
6407
+ byteLength += page.byteLength;
6408
+ const rows = await step(entry.id, "parse", () => reader.push(decoder.decode(page)));
6409
+ if (rows.length > 0) yield rows;
6410
+ }
6411
+ const rows = await step(entry.id, "parse", () => [...reader.push(decoder.end()), ...reader.end()]);
6412
+ if (rows.length > 0) yield rows;
6413
+ }
6414
+ return {
6415
+ meta: metaOf(entry),
6416
+ pages: pages(),
6417
+ provenance: () => ({
6418
+ entryId: entry.id,
6419
+ catalogue: dataset.catalogue,
6420
+ category: entry.category,
6421
+ source: sourceRecord(source),
6422
+ fetchedUrl,
6423
+ retrievedAt,
6424
+ sha256: hash.copy().digest("hex"),
6425
+ byteLength
6426
+ })
6427
+ };
6428
+ }
6429
+ /**
6430
+ * The nodes of an encoded page: the crosswalk writes only the meta and the nodes, so the result is an object with a `nodes` array.
6431
+ * @param encoded - The crosswalk's output for a page.
6432
+ * @returns Its nodes.
6433
+ * @throws KeepsetError when the output is not an object with a nodes array and nothing beyond meta.
6434
+ */
6435
+ function nodesOf(encoded) {
6436
+ if (typeof encoded !== "object" || encoded === null || Array.isArray(encoded) || !("nodes" in encoded) || !Array.isArray(encoded.nodes)) throw new KeepsetError("the crosswalk's output is not an object with a nodes array");
6437
+ const extra = Object.keys(encoded).filter((key) => key !== "meta" && key !== "nodes");
6438
+ if (extra.length > 0) throw new KeepsetError(`the crosswalk's output has ${extra.join(", ")} beside meta and nodes`);
6439
+ return encoded.nodes;
6440
+ }
6441
+ /**
6442
+ * Checks a crosswalk can run a page at a time, or fails the crosswalk step.
6443
+ * @param entry - The entry.
6444
+ * @param crosswalk - Its crosswalk.
6445
+ * @throws RecipeError when a rule reaches beyond the meta or one node's fields.
6446
+ */
6447
+ function assertSeparable(entry, crosswalk) {
6448
+ const reason = inseparableRule(crosswalk);
6449
+ if (reason !== void 0) throw new RecipeError(entry.id, "crosswalk", `a streamed crosswalk must apply to a page of nodes on its own: ${reason}`);
6450
+ }
6451
+ /**
6452
+ * Streams a table, as `streamEntry` describes.
6453
+ * @param dataset - A dataset whose recipe streams.
6454
+ * @param sink - Where the meta and the nodes go.
6455
+ * @param options - The network port, abort signal and clock.
6456
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
6457
+ * @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
6458
+ */
6459
+ async function streamTable(dataset, sink, options) {
6460
+ const { entry, crosswalk } = dataset;
6461
+ const recipe = streamedRecipe(entry);
6462
+ assertSeparable(entry, crosswalk);
6463
+ const shape = getStreamedShape(recipe.category);
6464
+ const native = openNative(dataset, recipe, options);
6465
+ const meta = await step(entry.id, "validate", async () => {
6466
+ const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, {
6467
+ meta: native.meta,
6468
+ nodes: []
6469
+ }));
6470
+ const parsed = shape.meta.safeParse(typeof encoded === "object" && encoded !== null && !Array.isArray(encoded) && "meta" in encoded ? encoded.meta : void 0);
6471
+ if (!parsed.success) throw new KeepsetError(`the result's meta is not ${entry.category} meta: ${parsed.error.issues.map((issue) => `${["meta", ...issue.path.map(String)].join(".")}: ${issue.message}`).join("; ")}`);
6472
+ return parsed.data;
6473
+ });
6474
+ await sink.meta(meta);
6475
+ const ids = new IdSet();
6476
+ for await (const rows of native.pages) {
6477
+ const offset = ids.size;
6478
+ const encoded = await step(entry.id, "crosswalk", () => nodesOf(encode(crosswalk, {
6479
+ meta: native.meta,
6480
+ nodes: rows
6481
+ })));
6482
+ const nodes = await step(entry.id, "validate", () => encoded.map((candidate, index) => {
6483
+ const parsed = shape.node.safeParse(candidate);
6484
+ if (!parsed.success) throw new KeepsetError(`the result is not ${entry.category}: ${parsed.error.issues.map((issue) => `${[
6485
+ "nodes",
6486
+ String(offset + index),
6487
+ ...issue.path.map(String)
6488
+ ].join(".")}: ${issue.message}`).join("; ")}`);
6489
+ if (!ids.add(parsed.data.id)) throw new KeepsetError(`the result is not ${entry.category}: nodes.${String(offset + index)}.id: duplicate node id ${JSON.stringify(parsed.data.id)}`);
6490
+ return parsed.data;
6491
+ }));
6492
+ await sink.nodes(nodes);
6493
+ }
6494
+ const counts = /* @__PURE__ */ new Map([["nodes", ids.size]]);
6495
+ const warnings = Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`);
6496
+ return {
6497
+ dataset,
6498
+ meta,
6499
+ provenance: native.provenance(),
6500
+ counts,
6501
+ warnings
6502
+ };
6503
+ }
6504
+ /**
6505
+ * Checks a streamed dataset's crosswalk on its real data a page at a time: crossmap's `assertRoundTrip` on the meta alone and then on each page's rows under the meta. The crosswalk applies to a page on its own, so the round trip holds on the whole table exactly when it holds on every page. It stops at the first page that differs and reports the path in the whole document.
6506
+ * @param dataset - A dataset whose recipe streams.
6507
+ * @param options - The network port, abort signal and clock.
6508
+ * @returns Whether the round trip holds, and where it first differs when it does not, with the provenance of the bytes it was checked on: every byte when it holds, those read up to the page that differs when it does not.
6509
+ * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison or the crosswalk cannot run.
6510
+ */
6511
+ async function roundTripTable(dataset, options) {
6512
+ const { entry, crosswalk } = dataset;
6513
+ const recipe = streamedRecipe(entry);
6514
+ assertSeparable(entry, crosswalk);
6515
+ const native = openNative(dataset, recipe, options);
6516
+ const check = (nodes, offset) => {
6517
+ try {
6518
+ assertRoundTrip(crosswalk, [{
6519
+ meta: native.meta,
6520
+ nodes
6521
+ }]);
6522
+ } catch (error) {
6523
+ if (error instanceof RoundTripMismatchError) {
6524
+ const path = error.path.replace(/^nodes\[(\d+)\]/, (_, index) => `nodes[${String(offset + Number(index))}]`);
6525
+ return {
6526
+ holds: false,
6527
+ path,
6528
+ message: error.message.replaceAll(error.path, path),
6529
+ provenance: native.provenance()
6530
+ };
6531
+ }
6532
+ throw new RecipeError(entry.id, "crosswalk", error instanceof Error ? error.message : String(error), { cause: error });
6533
+ }
6534
+ };
6535
+ const metaMismatch = check([], 0);
6536
+ if (metaMismatch !== void 0) return metaMismatch;
6537
+ let offset = 0;
6538
+ for await (const rows of native.pages) {
6539
+ const mismatch = check(rows, offset);
6540
+ if (mismatch !== void 0) {
6541
+ await native.pages.return();
6542
+ return mismatch;
6543
+ }
6544
+ offset += rows.length;
6545
+ }
6546
+ return {
6547
+ holds: true,
6548
+ provenance: native.provenance()
6549
+ };
6550
+ }
6551
+ //#endregion
6552
+ //#region src/node/fetch-dataset.ts
6553
+ /**
6554
+ * The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
6555
+ * @param source - The source.
6556
+ * @returns The URL to request.
6557
+ */
6558
+ function downloadUrl(source) {
6559
+ return sourceUrl(source);
6560
+ }
5369
6561
  /**
5370
6562
  * Puts an instance's id beside the fields of its encoded document. A document that is not an object is left as it is, for the collection schema to refuse.
5371
6563
  * @param document - The crosswalk's output for one member.
@@ -5379,6 +6571,139 @@ function withId(document, id) {
5379
6571
  } : document;
5380
6572
  }
5381
6573
  /**
6574
+ * Downloads one URL's bytes through the network port.
6575
+ * @param port - The network port.
6576
+ * @param url - The URL.
6577
+ * @param signal - Aborts the download.
6578
+ * @returns The bytes.
6579
+ * @throws KeepsetError when the server does not answer successfully.
6580
+ */
6581
+ async function downloadBytes(port, url, signal) {
6582
+ const response = await port(url, signal === void 0 ? {} : { signal });
6583
+ if (!response.ok) throw new KeepsetError(`${url} answered ${String(response.status)} ${response.statusText}`);
6584
+ return new Uint8Array(await response.arrayBuffer());
6585
+ }
6586
+ /**
6587
+ * The record of one download: the source, the URL requested and a hash of exactly the bytes that arrived.
6588
+ * @param source - The entry's source.
6589
+ * @param fetchedUrl - The URL requested.
6590
+ * @param bytes - The bytes.
6591
+ * @returns The download record.
6592
+ */
6593
+ function downloadOf(source, fetchedUrl, bytes) {
6594
+ return {
6595
+ source: sourceRecord(source),
6596
+ fetchedUrl,
6597
+ sha256: createHash("sha256").update(bytes).digest("hex"),
6598
+ byteLength: bytes.byteLength
6599
+ };
6600
+ }
6601
+ /**
6602
+ * Parses one file of a collection into its native document, naming the file when it fails.
6603
+ * @param id - The instance's id.
6604
+ * @param bytes - The file's bytes.
6605
+ * @param parser - The recipe's parser.
6606
+ * @returns The instance's native document, with only its own header values as `meta`.
6607
+ * @throws KeepsetError naming the instance when the file does not parse.
6608
+ */
6609
+ function parseInstance(id, bytes, parser) {
6610
+ try {
6611
+ const parsed = parseNative(bytes, parser);
6612
+ return {
6613
+ id,
6614
+ native: {
6615
+ ...parsed,
6616
+ meta: "meta" in parsed ? parsed.meta : {}
6617
+ }
6618
+ };
6619
+ } catch (error) {
6620
+ throw new KeepsetError(`member ${id}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
6621
+ }
6622
+ }
6623
+ /**
6624
+ * Unpacks and parses the bytes of one archive or file into instances, each member let go once it is parsed.
6625
+ * @param bytes - The bytes.
6626
+ * @param archive - Their archive layout.
6627
+ * @param parser - The recipe's parser.
6628
+ * @param prefix - The path the instance ids are filed under, joined to each member's path, or undefined to use the member's path alone.
6629
+ * @returns The instances, in member order.
6630
+ */
6631
+ function instancesOf(bytes, archive, parser, prefix) {
6632
+ const named = (member) => {
6633
+ if (prefix === void 0) {
6634
+ if (member === void 0) throw new KeepsetError("an instance of a collection has no path");
6635
+ return member;
6636
+ }
6637
+ return member === void 0 ? prefix : `${prefix}/${member}`;
6638
+ };
6639
+ if ("members" in archive) return Array.from(unpackMembers(bytes, archive), (member) => parseInstance(named(member.path), member.bytes, parser));
6640
+ if ("parts" in archive) throw new ArchiveError(`an instance is read from one file or from matching members, not from the named parts ${archive.parts.map((part) => part.name).join(", ")}`);
6641
+ if ("files" in archive) throw new ArchiveError("an instance is read from one file or from matching members, not from every file a multi-file format's files selector matches");
6642
+ const unpacked = unpack(bytes, archive);
6643
+ return [parseInstance(named(unpacked.member), unpacked.bytes, parser)];
6644
+ }
6645
+ /**
6646
+ * Unpacks and parses instances as `instancesOf` does, failing at the extract step when the archive cannot be read or its selector matches nothing, and at the parse step when a member does not parse: the members are unpacked one at a time as they are parsed, so the two steps interleave.
6647
+ * @param entryId - The entry.
6648
+ * @param run - The unpacking and parsing.
6649
+ * @returns The instances.
6650
+ * @throws RecipeError naming the entry and the step that failed.
6651
+ */
6652
+ function extractAndParse(entryId, run) {
6653
+ try {
6654
+ return run();
6655
+ } catch (error) {
6656
+ throw new RecipeError(entryId, error instanceof ArchiveError ? "extract" : "parse", error instanceof Error ? error.message : String(error), { cause: error });
6657
+ }
6658
+ }
6659
+ /**
6660
+ * Runs a files recipe: downloads each file in turn (a dead file from its own pinned snapshot), unpacks it and parses it into its instances, so only one file's bytes are held at a time.
6661
+ * @param dataset - The dataset.
6662
+ * @param recipe - Its files recipe.
6663
+ * @param options - The network port, abort signal and clock.
6664
+ * @returns The instances and the provenance of every file.
6665
+ */
6666
+ async function runFiles(dataset, recipe, options) {
6667
+ const { entry } = dataset;
6668
+ const { fetch: port = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
6669
+ const retrievedAt = now().toISOString();
6670
+ const read = async (index) => {
6671
+ const file = recipe.files[index];
6672
+ if (file === void 0) return [];
6673
+ const source = entry.sources[file.source];
6674
+ if (source === void 0) throw new RecipeError(entry.id, "recipe", `file ${file.path} names source ${String(file.source)}, which the entry does not have`);
6675
+ const fetchedUrl = sourceUrl(source);
6676
+ const bytes = await step(entry.id, "download", async () => downloadBytes(port, fetchedUrl, signal));
6677
+ const documents = extractAndParse(entry.id, () => instancesOf(bytes, file.archive, recipe.parser, file.path));
6678
+ return [{
6679
+ documents,
6680
+ download: {
6681
+ ...downloadOf(source, fetchedUrl, bytes),
6682
+ path: file.path,
6683
+ instances: documents.map((document) => document.id)
6684
+ }
6685
+ }, ...await read(index + 1)];
6686
+ };
6687
+ const files = await read(0);
6688
+ const documents = files.flatMap((file) => file.documents);
6689
+ const ids = /* @__PURE__ */ new Set();
6690
+ for (const { id } of documents) {
6691
+ if (ids.has(id)) throw new RecipeError(entry.id, "extract", `two files give the instance ${id}`);
6692
+ ids.add(id);
6693
+ }
6694
+ return {
6695
+ collection: true,
6696
+ documents,
6697
+ provenance: {
6698
+ entryId: entry.id,
6699
+ catalogue: dataset.catalogue,
6700
+ category: entry.category,
6701
+ retrievedAt,
6702
+ files: files.map((file) => file.download)
6703
+ }
6704
+ };
6705
+ }
6706
+ /**
5382
6707
  * Runs a recipe's download, extract and parse steps, and builds the native documents the crosswalk receives.
5383
6708
  * @param dataset - The dataset.
5384
6709
  * @param options - The network port, abort signal and clock.
@@ -5389,51 +6714,28 @@ async function runToNative(dataset, options) {
5389
6714
  const { fetch: fetchPort = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
5390
6715
  const { recipe } = entry;
5391
6716
  if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
6717
+ if ("files" in recipe) return runFiles(dataset, recipe, options);
6718
+ if (recipe.stream === true) throw new RecipeError(entry.id, "recipe", "the table is too large to hold as one document, so its recipe streams: read it with streamDataset, which hands its nodes on a page at a time");
5392
6719
  const source = entry.sources[recipe.source];
5393
6720
  if (source === void 0) throw new RecipeError(entry.id, "recipe", `the recipe names source ${String(recipe.source)}, which the entry does not have`);
5394
- const fetchedUrl = downloadUrl(source);
6721
+ const fetchedUrl = sourceUrl(source);
5395
6722
  const retrievedAt = now().toISOString();
5396
- const bytes = await step(entry.id, "download", async () => {
5397
- if (recipe.paging !== void 0) return downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal);
5398
- const response = await fetchPort(fetchedUrl, signal === void 0 ? {} : { signal });
5399
- if (!response.ok) throw new KeepsetError(`${fetchedUrl} answered ${String(response.status)} ${response.statusText}`);
5400
- return new Uint8Array(await response.arrayBuffer());
5401
- });
6723
+ const bytes = await step(entry.id, "download", async () => recipe.paging === void 0 ? downloadBytes(fetchPort, fetchedUrl, signal) : downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal));
5402
6724
  const provenanceOf = (extra) => ({
5403
6725
  entryId: entry.id,
5404
6726
  catalogue: dataset.catalogue,
5405
6727
  category: entry.category,
5406
- source: {
5407
- url: source.url,
5408
- status: source.status,
5409
- ...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
5410
- },
5411
- fetchedUrl,
5412
6728
  retrievedAt,
5413
- sha256: createHash("sha256").update(bytes).digest("hex"),
5414
- byteLength: bytes.byteLength,
6729
+ ...downloadOf(source, fetchedUrl, bytes),
5415
6730
  ...extra
5416
6731
  });
5417
6732
  const { archive } = recipe;
5418
6733
  if ("members" in archive) {
5419
- const files = await step(entry.id, "extract", () => unpackMembers(bytes, archive));
6734
+ const documents = extractAndParse(entry.id, () => instancesOf(bytes, archive, recipe.parser, void 0));
5420
6735
  return {
5421
6736
  collection: true,
5422
- documents: await step(entry.id, "parse", () => files.map((file) => {
5423
- try {
5424
- const parsed = parseNative(file.bytes, recipe.parser);
5425
- return {
5426
- id: file.path,
5427
- native: {
5428
- ...parsed,
5429
- meta: "meta" in parsed ? parsed.meta : {}
5430
- }
5431
- };
5432
- } catch (error) {
5433
- throw new KeepsetError(`member ${file.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
5434
- }
5435
- })),
5436
- provenance: provenanceOf({ members: files.map((file) => file.path) })
6737
+ documents,
6738
+ provenance: provenanceOf({ members: documents.map((document) => document.id) })
5437
6739
  };
5438
6740
  }
5439
6741
  if ("parts" in archive) {
@@ -5450,6 +6752,20 @@ async function runToNative(dataset, options) {
5450
6752
  provenance: provenanceOf({ members: parts.map((part) => part.path) })
5451
6753
  };
5452
6754
  }
6755
+ if ("files" in archive) {
6756
+ const files = await step(entry.id, "extract", () => unpackFiles(bytes, archive));
6757
+ return {
6758
+ collection: false,
6759
+ documents: [{
6760
+ id: void 0,
6761
+ native: {
6762
+ ...await step(entry.id, "parse", () => parseNativeFiles(files, recipe.parser)),
6763
+ meta: metaOf(entry)
6764
+ }
6765
+ }],
6766
+ provenance: provenanceOf({ members: files.map((file) => file.path) })
6767
+ };
6768
+ }
5453
6769
  const unpacked = await step(entry.id, "extract", () => unpack(bytes, archive));
5454
6770
  const parsed = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
5455
6771
  return {
@@ -5472,7 +6788,7 @@ async function runToNative(dataset, options) {
5472
6788
  * @param dataset - A dataset from any catalogue.
5473
6789
  * @param options - The network port, abort signal and clock.
5474
6790
  * @returns The resolved target with its provenance and any count warnings.
5475
- * @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
6791
+ * @throws RecipeError naming the entry and the failing step, including for a recipe that streams, which `streamEntry` reads; nothing partial or default-shaped is ever returned.
5476
6792
  */
5477
6793
  async function fetchEntry(dataset, options = {}) {
5478
6794
  const { entry, crosswalk } = dataset;
@@ -5513,13 +6829,14 @@ async function fetchDataset(id, options = {}) {
5513
6829
  return fetchEntry(getDataset(id), options);
5514
6830
  }
5515
6831
  /**
5516
- * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
6832
+ * Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve. A streamed recipe is checked a page at a time, with the round trip holding on the whole table exactly when it holds on every page, so the table is never held.
5517
6833
  * @param dataset - A dataset from any catalogue.
5518
6834
  * @param options - The network port, abort signal and clock.
5519
6835
  * @returns Whether the round trip holds, and where it first differs when it does not.
5520
6836
  * @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
5521
6837
  */
5522
6838
  async function roundTripEntry(dataset, options = {}) {
6839
+ if (isStreamed(dataset.entry.recipe)) return roundTripTable(dataset, options);
5523
6840
  const { documents, provenance } = await runToNative(dataset, options);
5524
6841
  try {
5525
6842
  assertRoundTrip(dataset.crosswalk, documents.map(({ native }) => native));
@@ -5538,6 +6855,28 @@ async function roundTripEntry(dataset, options = {}) {
5538
6855
  };
5539
6856
  }
5540
6857
  /**
6858
+ * Streams a dataset of the bundled catalogue by id, as `streamEntry` does: for a table too large to hold as one document, whose recipe streams.
6859
+ * @param id - The entry id.
6860
+ * @param sink - Where the meta and the nodes go.
6861
+ * @param options - The network port, abort signal and clock.
6862
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
6863
+ * @throws DatasetNotFoundError when no entry has that id, and RecipeError as `streamEntry` does.
6864
+ */
6865
+ async function streamDataset(id, sink, options = {}) {
6866
+ return streamEntry(getDataset(id), sink, options);
6867
+ }
6868
+ /**
6869
+ * Streams a dataset whose table is too large to hold as one document: reads its paged source a page at a time, applies the crosswalk to each page's rows under the document's meta, checks the meta once and each node against the category's node shape and for an id no earlier node had, and hands the nodes on to the sink before reading the next page. What it holds at once is one page's bytes and rows and the ids seen so far (as bytes, not strings), never the table, so the result is the category target `{ meta, nodes }` delivered in order rather than held: the same nodes `fetchEntry` would give, checked by the same rules.
6870
+ * @param dataset - A dataset whose recipe streams.
6871
+ * @param sink - Where the meta and the nodes go.
6872
+ * @param options - The network port, abort signal and clock.
6873
+ * @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
6874
+ * @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
6875
+ */
6876
+ async function streamEntry(dataset, sink, options = {}) {
6877
+ return streamTable(dataset, sink, options);
6878
+ }
6879
+ /**
5541
6880
  * Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
5542
6881
  * @param id - The entry id.
5543
6882
  * @param options - The network port, abort signal and clock.
@@ -5548,4 +6887,4 @@ async function roundTripDataset(id, options = {}) {
5548
6887
  return roundTripEntry(getDataset(id), options);
5549
6888
  }
5550
6889
  //#endregion
5551
- export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry };
6890
+ export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry, streamDataset, streamEntry };