keepset 0.39.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{bundled-6dYCPVD-.js → bundled-B1094k0B.js} +12215 -7937
- package/dist/{categories-DijttH_M.d.ts → categories-BbW0rFCf.d.ts} +2857 -286
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/node/fetch-dataset.d.ts +75 -24
- package/dist/node/fetch-dataset.js +1368 -298
- package/dist-cli/keepset.mjs +80 -9
- package/package.json +1 -1
- package/schemas/entry.schema.json +1 -1
- package/schemas/manifest.schema.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { F as rawSnapshotUrl, H as getOwn, J as getStreamedShape, W as countTarget, X as isStreamable, Y as getTarget, dt as KeepsetError, ft as NativeParseError, pt as RecipeError, q as getCollectionTarget, st as ArchiveError, t as getDataset } from "../bundled-B1094k0B.js";
|
|
2
2
|
import { RoundTripMismatchError, assertRoundTrip, encode } from "crossmap";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
@@ -222,8 +222,8 @@ function parseEdgeList(text, options) {
|
|
|
222
222
|
}
|
|
223
223
|
//#endregion
|
|
224
224
|
//#region src/parsers/gd-contest.ts
|
|
225
|
-
const FORMAT$
|
|
226
|
-
const NUMBER$
|
|
225
|
+
const FORMAT$18 = "gd-contest";
|
|
226
|
+
const NUMBER$10 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
|
|
227
227
|
/**
|
|
228
228
|
* Reads a vertex number, which the format numbers from 0 to N - 1.
|
|
229
229
|
* @param token - The token.
|
|
@@ -231,7 +231,7 @@ const NUMBER$9 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
|
|
|
231
231
|
* @returns The vertex id as text.
|
|
232
232
|
*/
|
|
233
233
|
function vertex$1(token, count) {
|
|
234
|
-
if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$
|
|
234
|
+
if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$18, token.line, `expected a vertex number from 0 to ${String(count - 1)}, found ${JSON.stringify(token.text)}`);
|
|
235
235
|
return token.text;
|
|
236
236
|
}
|
|
237
237
|
/**
|
|
@@ -251,17 +251,17 @@ function parseGdContest(text) {
|
|
|
251
251
|
});
|
|
252
252
|
}
|
|
253
253
|
const [first] = tokens;
|
|
254
|
-
if (first === void 0) throw new NativeParseError(FORMAT$
|
|
255
|
-
if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$
|
|
254
|
+
if (first === void 0) throw new NativeParseError(FORMAT$18, 1, "the file holds no vertex count");
|
|
255
|
+
if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$18, first.line, `the vertex count must be a non-negative integer, found ${JSON.stringify(first.text)}`);
|
|
256
256
|
const count = Number(first.text);
|
|
257
257
|
const coordinates = tokens.slice(1, 1 + 2 * count);
|
|
258
|
-
if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$
|
|
259
|
-
for (const coordinate of coordinates) if (!NUMBER$
|
|
258
|
+
if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$18, lines(text).length, `expected ${String(2 * count)} coordinates for ${String(count)} vertices, found ${String(coordinates.length)}`);
|
|
259
|
+
for (const coordinate of coordinates) if (!NUMBER$10.test(coordinate.text)) throw new NativeParseError(FORMAT$18, coordinate.line, `coordinate ${JSON.stringify(coordinate.text)} is not a number`);
|
|
260
260
|
const ends = tokens.slice(1 + 2 * count);
|
|
261
261
|
const edges = [];
|
|
262
262
|
for (let source = ends.shift(); source !== void 0; source = ends.shift()) {
|
|
263
263
|
const target = ends.shift();
|
|
264
|
-
if (target === void 0) throw new NativeParseError(FORMAT$
|
|
264
|
+
if (target === void 0) throw new NativeParseError(FORMAT$18, source.line, `edge from vertex ${source.text} has no second vertex`);
|
|
265
265
|
edges.push({
|
|
266
266
|
source: vertex$1(source, count),
|
|
267
267
|
target: vertex$1(target, count)
|
|
@@ -275,7 +275,7 @@ function parseGdContest(text) {
|
|
|
275
275
|
//#endregion
|
|
276
276
|
//#region src/parsers/gml.ts
|
|
277
277
|
const KEY = /[A-Za-z_][A-Za-z0-9_]*/y;
|
|
278
|
-
const NUMBER$
|
|
278
|
+
const NUMBER$9 = /[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?/y;
|
|
279
279
|
/**
|
|
280
280
|
* Matches a sticky pattern at a position.
|
|
281
281
|
* @param pattern - A sticky regular expression.
|
|
@@ -326,7 +326,7 @@ function tokenise(text) {
|
|
|
326
326
|
position = end + 1;
|
|
327
327
|
} else {
|
|
328
328
|
const key = matchAt(KEY, text, position);
|
|
329
|
-
const number = key === void 0 ? matchAt(NUMBER$
|
|
329
|
+
const number = key === void 0 ? matchAt(NUMBER$9, text, position) : void 0;
|
|
330
330
|
const match = key ?? number;
|
|
331
331
|
if (match === void 0) throw new NativeParseError("gml", line, `unexpected character ${JSON.stringify(char)}`);
|
|
332
332
|
tokens.push({
|
|
@@ -431,7 +431,7 @@ function parseGml(text) {
|
|
|
431
431
|
}
|
|
432
432
|
//#endregion
|
|
433
433
|
//#region src/parsers/hgb.ts
|
|
434
|
-
const FORMAT$
|
|
434
|
+
const FORMAT$17 = "hgb";
|
|
435
435
|
/** A type code or node id as the HGB files write them: a non-negative decimal integer with no sign or leading zero. */
|
|
436
436
|
const CODE = /^(?:0|[1-9]\d*)$/;
|
|
437
437
|
/** A number as a decimal literal with an optional sign and exponent, so `Number` cannot read hex, an empty string or infinity. */
|
|
@@ -451,7 +451,7 @@ const LINK_FIELDS = 4;
|
|
|
451
451
|
*/
|
|
452
452
|
function resolve(field, types, where) {
|
|
453
453
|
const type = types[CODE.test(field) ? Number(field) : -1];
|
|
454
|
-
if (type === void 0) throw new NativeParseError(FORMAT$
|
|
454
|
+
if (type === void 0) throw new NativeParseError(FORMAT$17, where.line, `${where.file}: the ${where.kind} type ${JSON.stringify(field)} is not one of the ${String(types.length)} the recipe names`);
|
|
455
455
|
return type;
|
|
456
456
|
}
|
|
457
457
|
/**
|
|
@@ -469,9 +469,9 @@ function parseHgb(parts, options) {
|
|
|
469
469
|
forEachLine(parts.nodes, (line, number) => {
|
|
470
470
|
const fields = line.split(" ");
|
|
471
471
|
const [id, , type] = fields;
|
|
472
|
-
if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$
|
|
473
|
-
if (!CODE.test(id)) throw new NativeParseError(FORMAT$
|
|
474
|
-
if (typeOf.has(id)) throw new NativeParseError(FORMAT$
|
|
472
|
+
if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$17, number, `node.dat: expected ${String(NODE_FIELDS)} or ${String(NODE_FIELDS_WITH_ATTRIBUTES)} fields separated by tabs, found ${String(fields.length)}`);
|
|
473
|
+
if (!CODE.test(id)) throw new NativeParseError(FORMAT$17, number, `node.dat: the node id ${JSON.stringify(id)} is not a non-negative integer`);
|
|
474
|
+
if (typeOf.has(id)) throw new NativeParseError(FORMAT$17, number, `node.dat: the node id ${id} is repeated`);
|
|
475
475
|
const nodeType = resolve(type, options.nodeTypes, {
|
|
476
476
|
file: "node.dat",
|
|
477
477
|
line: number,
|
|
@@ -483,12 +483,12 @@ function parseHgb(parts, options) {
|
|
|
483
483
|
type: nodeType
|
|
484
484
|
});
|
|
485
485
|
});
|
|
486
|
-
if (nodes.length === 0) throw new NativeParseError(FORMAT$
|
|
486
|
+
if (nodes.length === 0) throw new NativeParseError(FORMAT$17, 1, "node.dat: the file holds no nodes");
|
|
487
487
|
const edges = [];
|
|
488
488
|
forEachLine(parts.links, (line, number) => {
|
|
489
489
|
const fields = line.split(" ");
|
|
490
490
|
const [source, target, type, weight] = fields;
|
|
491
|
-
if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$
|
|
491
|
+
if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$17, number, `link.dat: expected ${String(LINK_FIELDS)} fields separated by tabs, found ${String(fields.length)}`);
|
|
492
492
|
const linkType = resolve(type, options.linkTypes, {
|
|
493
493
|
file: "link.dat",
|
|
494
494
|
line: number,
|
|
@@ -504,17 +504,17 @@ function parseHgb(parts, options) {
|
|
|
504
504
|
linkType.target
|
|
505
505
|
]]) {
|
|
506
506
|
const actual = typeOf.get(id);
|
|
507
|
-
if (actual === void 0) throw new NativeParseError(FORMAT$
|
|
508
|
-
if (actual !== expected) throw new NativeParseError(FORMAT$
|
|
507
|
+
if (actual === void 0) throw new NativeParseError(FORMAT$17, number, `link.dat: the ${end} ${JSON.stringify(id)} is not a node of node.dat`);
|
|
508
|
+
if (actual !== expected) throw new NativeParseError(FORMAT$17, number, `link.dat: a ${linkType.name} link's ${end} must be a ${expected} node, and ${id} is a ${actual} node`);
|
|
509
509
|
}
|
|
510
|
-
if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$
|
|
510
|
+
if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$17, number, `link.dat: the weight ${JSON.stringify(weight)} is not 1; a weighted HGB file would lose its weights`);
|
|
511
511
|
edges.push({
|
|
512
512
|
source,
|
|
513
513
|
target,
|
|
514
514
|
type: linkType.name
|
|
515
515
|
});
|
|
516
516
|
});
|
|
517
|
-
if (edges.length === 0) throw new NativeParseError(FORMAT$
|
|
517
|
+
if (edges.length === 0) throw new NativeParseError(FORMAT$17, 1, "link.dat: the file holds no links");
|
|
518
518
|
return {
|
|
519
519
|
nodes,
|
|
520
520
|
edges
|
|
@@ -522,7 +522,7 @@ function parseHgb(parts, options) {
|
|
|
522
522
|
}
|
|
523
523
|
//#endregion
|
|
524
524
|
//#region src/parsers/kg-triples.ts
|
|
525
|
-
const FORMAT$
|
|
525
|
+
const FORMAT$16 = "kg-triples";
|
|
526
526
|
/** The character each delimiter option splits a line on. A line is split on exactly that character, so a field cannot hide a second delimiter. */
|
|
527
527
|
const DELIMITERS$1 = {
|
|
528
528
|
tab: " ",
|
|
@@ -544,8 +544,8 @@ const QUOTE$2 = "\"";
|
|
|
544
544
|
*/
|
|
545
545
|
function value(field, quoted, where) {
|
|
546
546
|
const inner = quoted ? field.slice(1, -1) : field;
|
|
547
|
-
if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$
|
|
548
|
-
if (inner === "") throw new NativeParseError(FORMAT$
|
|
547
|
+
if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$16, where.line, `${where.part}the field ${JSON.stringify(field)} is not one value wrapped in double quotes`);
|
|
548
|
+
if (inner === "") throw new NativeParseError(FORMAT$16, where.line, `${where.part}a field is empty`);
|
|
549
549
|
return inner;
|
|
550
550
|
}
|
|
551
551
|
/**
|
|
@@ -569,16 +569,16 @@ function parseKgTriples(parts, options) {
|
|
|
569
569
|
let lastLine = 0;
|
|
570
570
|
forEachLine(part.pieces, (line, number) => {
|
|
571
571
|
lastLine = number;
|
|
572
|
-
if (line.trim() === "") throw new NativeParseError(FORMAT$
|
|
572
|
+
if (line.trim() === "") throw new NativeParseError(FORMAT$16, number, `${label}the line is blank`);
|
|
573
573
|
if (number === 1 && options.header === "names") return;
|
|
574
574
|
if (number === 1 && options.header === "count") {
|
|
575
|
-
if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$
|
|
575
|
+
if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$16, number, `${label}expected the number of triples on the first line, found ${JSON.stringify(line)}`);
|
|
576
576
|
declared = Number(line);
|
|
577
577
|
return;
|
|
578
578
|
}
|
|
579
579
|
const fields = line.split(delimiter);
|
|
580
580
|
const [head, middle, last] = fields;
|
|
581
|
-
if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$
|
|
581
|
+
if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$16, number, `${label}expected ${String(TRIPLE_FIELDS)} fields separated by ${options.delimiter}, found ${String(fields.length)}`);
|
|
582
582
|
const where = {
|
|
583
583
|
line: number,
|
|
584
584
|
part: label
|
|
@@ -604,8 +604,8 @@ function parseKgTriples(parts, options) {
|
|
|
604
604
|
});
|
|
605
605
|
triples += 1;
|
|
606
606
|
});
|
|
607
|
-
if (triples === 0) throw new NativeParseError(FORMAT$
|
|
608
|
-
if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$
|
|
607
|
+
if (triples === 0) throw new NativeParseError(FORMAT$16, lastLine, `${label}the file holds no triples`);
|
|
608
|
+
if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$16, lastLine, `${label}the first line declares ${String(declared)} triples, the file holds ${String(triples)}`);
|
|
609
609
|
}
|
|
610
610
|
return {
|
|
611
611
|
nodes: order.nodes(),
|
|
@@ -614,7 +614,7 @@ function parseKgTriples(parts, options) {
|
|
|
614
614
|
}
|
|
615
615
|
//#endregion
|
|
616
616
|
//#region src/parsers/konect.ts
|
|
617
|
-
const FORMAT$
|
|
617
|
+
const FORMAT$15 = "konect";
|
|
618
618
|
/** The network formats of KONECT's first line (Kunegis, "The KONECT Handbook", table 1): undirected, directed and bipartite. */
|
|
619
619
|
const NETWORK_FORMATS = /* @__PURE__ */ new Set([
|
|
620
620
|
"sym",
|
|
@@ -689,9 +689,9 @@ function parseKonect(pieces, options) {
|
|
|
689
689
|
}));
|
|
690
690
|
});
|
|
691
691
|
const { header } = state;
|
|
692
|
-
if (header === void 0) throw new NativeParseError(FORMAT$
|
|
693
|
-
if (edges.length === 0) throw new NativeParseError(FORMAT$
|
|
694
|
-
if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$
|
|
692
|
+
if (header === void 0) throw new NativeParseError(FORMAT$15, lastLine, "the file has no first line naming its format and weights");
|
|
693
|
+
if (edges.length === 0) throw new NativeParseError(FORMAT$15, lastLine, "the file holds no edges");
|
|
694
|
+
if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$15, lastLine, `the second line declares ${String(header.counts.edges)} edges, the file holds ${String(edges.length)}`);
|
|
695
695
|
return {
|
|
696
696
|
nodes: order.nodes(),
|
|
697
697
|
edges
|
|
@@ -709,9 +709,9 @@ function readHeader(content, number, options) {
|
|
|
709
709
|
const match = FORMAT_LINE.exec(content);
|
|
710
710
|
const network = match?.[1];
|
|
711
711
|
const weights = match?.[2];
|
|
712
|
-
if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$
|
|
713
|
-
if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$
|
|
714
|
-
if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$
|
|
712
|
+
if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$15, number, `expected a first line "% FORMAT WEIGHTS" with FORMAT one of ${[...NETWORK_FORMATS].join(", ")}, found ${JSON.stringify(content)}`);
|
|
713
|
+
if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$15, number, `the weight type ${JSON.stringify(weights)} is not one this parser reads (its third column is not an edge weight or a repetition count)`);
|
|
714
|
+
if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$15, number, `the file's weight type is ${weights}, so a recipe with weighted ${String(WEIGHTED_TYPES.has(weights))} reads it, not one with weighted ${String(options.weighted)}`);
|
|
715
715
|
return {
|
|
716
716
|
line: number,
|
|
717
717
|
network
|
|
@@ -731,15 +731,15 @@ function readEdge(content, number, context) {
|
|
|
731
731
|
const [from, to, third, time] = fields;
|
|
732
732
|
const expected = options.timestamped ? TIMESTAMPED_COLUMNS : options.weighted ? WEIGHTED_COLUMNS : void 0;
|
|
733
733
|
const fits = expected === void 0 ? fields.length === ENDPOINT_COLUMNS || fields.length === WEIGHTED_COLUMNS : fields.length === expected;
|
|
734
|
-
if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$
|
|
734
|
+
if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$15, number, `expected ${expected === void 0 ? `${String(ENDPOINT_COLUMNS)} or ${String(WEIGHTED_COLUMNS)}` : String(expected)} columns for this recipe, found ${String(fields.length)}`);
|
|
735
735
|
const bipartite = header.network === "bip";
|
|
736
736
|
const source = endpoint(from, bipartite ? LEFT_PREFIX : "", header.counts?.left, number);
|
|
737
737
|
const target = endpoint(to, bipartite ? RIGHT_PREFIX : "", header.counts?.right, number);
|
|
738
738
|
order.add(source);
|
|
739
739
|
order.add(target);
|
|
740
|
-
if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$
|
|
741
|
-
if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$
|
|
742
|
-
if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$
|
|
740
|
+
if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$15, number, `the third column ${JSON.stringify(third)} is not 1; an edge that stands for several would be read as one`);
|
|
741
|
+
if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$15, number, `the weight ${JSON.stringify(third)} is not a number`);
|
|
742
|
+
if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$15, number, `the time ${JSON.stringify(time)} is not a number`);
|
|
743
743
|
const edge = {
|
|
744
744
|
source,
|
|
745
745
|
target
|
|
@@ -758,12 +758,12 @@ function readEdge(content, number, context) {
|
|
|
758
758
|
* @throws NativeParseError when the token is not a positive integer or exceeds the set's size.
|
|
759
759
|
*/
|
|
760
760
|
function endpoint(token, prefix, size, number) {
|
|
761
|
-
if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$
|
|
761
|
+
if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$15, number, `${JSON.stringify(token)} is not a node id from 1${size === void 0 ? "" : ` to ${String(size)}`}`);
|
|
762
762
|
return `${prefix}${token}`;
|
|
763
763
|
}
|
|
764
764
|
//#endregion
|
|
765
765
|
//#region src/parsers/ogb.ts
|
|
766
|
-
const FORMAT$
|
|
766
|
+
const FORMAT$14 = "ogb";
|
|
767
767
|
/** The files of an OGB raw directory this reader takes, by their path under the dataset's root directory. A relation's files sit in `raw/relations/<head>___<relation>___<tail>/`. */
|
|
768
768
|
const RAW = /^(?<root>(?:[^/]+\/)*?)raw\/(?<name>num-node-list|num-node-dict|edge|num-edge-list|triplet-type-list|relations\/[^/]+\/(?:edge|num-edge-list))\.csv\.gz$/u;
|
|
769
769
|
const INDEX = /^(?:0|[1-9]\d*)$/;
|
|
@@ -812,7 +812,7 @@ function rows(file) {
|
|
|
812
812
|
function onlyLine(file) {
|
|
813
813
|
const all = rows(file);
|
|
814
814
|
const [only] = all;
|
|
815
|
-
if (all.length !== 1 || only === void 0) throw new NativeParseError(FORMAT$
|
|
815
|
+
if (all.length !== 1 || only === void 0) throw new NativeParseError(FORMAT$14, all[1]?.line ?? 1, `${file.path} must hold one line, found ${String(all.length)}`);
|
|
816
816
|
return only.text.trim();
|
|
817
817
|
}
|
|
818
818
|
/**
|
|
@@ -825,7 +825,7 @@ function onlyLine(file) {
|
|
|
825
825
|
*/
|
|
826
826
|
function index(text, file, line) {
|
|
827
827
|
const trimmed = text.trim();
|
|
828
|
-
if (!INDEX.test(trimmed)) throw new NativeParseError(FORMAT$
|
|
828
|
+
if (!INDEX.test(trimmed)) throw new NativeParseError(FORMAT$14, line, `${file.path}: expected a non-negative integer, found ${JSON.stringify(text)}`);
|
|
829
829
|
return Number(trimmed);
|
|
830
830
|
}
|
|
831
831
|
/**
|
|
@@ -842,17 +842,17 @@ function readEdges(edges, count, ends, emit) {
|
|
|
842
842
|
forEachRow(edges, ({ text, line }) => {
|
|
843
843
|
const fields = text.split(",");
|
|
844
844
|
const [first, second] = fields;
|
|
845
|
-
if (fields.length !== 2 || first === void 0 || second === void 0) throw new NativeParseError(FORMAT$
|
|
845
|
+
if (fields.length !== 2 || first === void 0 || second === void 0) throw new NativeParseError(FORMAT$14, line, `${edges.path}: expected a source and a target index, found ${JSON.stringify(text)}`);
|
|
846
846
|
const source = index(first, edges, line);
|
|
847
847
|
const target = index(second, edges, line);
|
|
848
|
-
if (source >= ends.sources || target >= ends.targets) throw new NativeParseError(FORMAT$
|
|
848
|
+
if (source >= ends.sources || target >= ends.targets) throw new NativeParseError(FORMAT$14, line, `${edges.path}: edge ${text} names a node outside the ${String(ends.sources)} sources and ${String(ends.targets)} targets the dataset records`);
|
|
849
849
|
emit({
|
|
850
850
|
source: ends.source(source),
|
|
851
851
|
target: ends.target(target)
|
|
852
852
|
});
|
|
853
853
|
read += 1;
|
|
854
854
|
});
|
|
855
|
-
if (read !== recorded) throw new NativeParseError(FORMAT$
|
|
855
|
+
if (read !== recorded) throw new NativeParseError(FORMAT$14, read, `${edges.path} holds ${String(read)} edges, but ${count.path} records ${String(recorded)}`);
|
|
856
856
|
}
|
|
857
857
|
/**
|
|
858
858
|
* Takes one file of the dataset directory by its name under `raw/`.
|
|
@@ -864,7 +864,7 @@ function readEdges(edges, count, ends, emit) {
|
|
|
864
864
|
*/
|
|
865
865
|
function required$1(files, name, root) {
|
|
866
866
|
const file = files.get(name);
|
|
867
|
-
if (file === void 0) throw new NativeParseError(FORMAT$
|
|
867
|
+
if (file === void 0) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has no raw/${name}.csv.gz among the files the recipe selects`);
|
|
868
868
|
return file;
|
|
869
869
|
}
|
|
870
870
|
/**
|
|
@@ -880,20 +880,20 @@ function parseOgb(files) {
|
|
|
880
880
|
const match = RAW.exec(file.path);
|
|
881
881
|
const name = match?.groups?.["name"];
|
|
882
882
|
const root = match?.groups?.["root"];
|
|
883
|
-
if (name === void 0 || root === void 0) throw new NativeParseError(FORMAT$
|
|
883
|
+
if (name === void 0 || root === void 0) throw new NativeParseError(FORMAT$14, 1, `${file.path} is not a file of an OGB raw directory this reader takes`);
|
|
884
884
|
roots.add(root);
|
|
885
885
|
byName.set(name, file);
|
|
886
886
|
}
|
|
887
887
|
const [root] = roots;
|
|
888
|
-
if (roots.size !== 1 || root === void 0) throw new NativeParseError(FORMAT$
|
|
888
|
+
if (roots.size !== 1 || root === void 0) throw new NativeParseError(FORMAT$14, 1, `the recipe must select the raw files of one dataset, found ${roots.size === 0 ? "none" : `the directories ${[...roots].join(", ")}`}`);
|
|
889
889
|
const homogeneous = byName.get("num-node-list");
|
|
890
890
|
const heterogeneous = byName.get("num-node-dict");
|
|
891
891
|
const neitherOrBoth = `the dataset directory ${root} must have exactly one of raw/num-node-list.csv.gz (one node type) and raw/num-node-dict.csv.gz (several)`;
|
|
892
892
|
const edges = [];
|
|
893
893
|
if (homogeneous !== void 0) {
|
|
894
|
-
if (heterogeneous !== void 0) throw new NativeParseError(FORMAT$
|
|
894
|
+
if (heterogeneous !== void 0) throw new NativeParseError(FORMAT$14, 1, neitherOrBoth);
|
|
895
895
|
const [relation] = [...byName.keys()].filter((name) => name.startsWith("relations/"));
|
|
896
|
-
if (relation !== void 0 || byName.has("triplet-type-list")) throw new NativeParseError(FORMAT$
|
|
896
|
+
if (relation !== void 0 || byName.has("triplet-type-list")) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has raw/num-node-list.csv.gz, so it has one node type, but the recipe selects ${relation === void 0 ? "raw/triplet-type-list.csv.gz" : `raw/${relation}.csv.gz`} as well`);
|
|
897
897
|
const count = index(onlyLine(homogeneous), homogeneous, 1);
|
|
898
898
|
const name = (node) => String(node);
|
|
899
899
|
readEdges(required$1(byName, "edge", root), required$1(byName, "num-edge-list", root), {
|
|
@@ -909,17 +909,17 @@ function parseOgb(files) {
|
|
|
909
909
|
edges
|
|
910
910
|
};
|
|
911
911
|
}
|
|
912
|
-
if (heterogeneous === void 0) throw new NativeParseError(FORMAT$
|
|
913
|
-
for (const stray of ["edge", "num-edge-list"]) if (byName.has(stray)) throw new NativeParseError(FORMAT$
|
|
912
|
+
if (heterogeneous === void 0) throw new NativeParseError(FORMAT$14, 1, neitherOrBoth);
|
|
913
|
+
for (const stray of ["edge", "num-edge-list"]) if (byName.has(stray)) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has raw/num-node-dict.csv.gz, so its edges are in raw/relations/, but the recipe selects raw/${stray}.csv.gz as well`);
|
|
914
914
|
const header = rows(heterogeneous);
|
|
915
915
|
const [types, counts] = header;
|
|
916
|
-
if (header.length !== 2 || types === void 0 || counts === void 0) throw new NativeParseError(FORMAT$
|
|
916
|
+
if (header.length !== 2 || types === void 0 || counts === void 0) throw new NativeParseError(FORMAT$14, header[2]?.line ?? 1, `${heterogeneous.path} must hold a header of node types and one row of their counts, found ${String(header.length)} lines`);
|
|
917
917
|
const typeNames = types.text.split(",").map((type) => type.trim());
|
|
918
918
|
const typeCounts = counts.text.split(",");
|
|
919
|
-
if (typeNames.length !== typeCounts.length || typeNames.some((type) => type === "") || new Set(typeNames).size !== typeNames.length) throw new NativeParseError(FORMAT$
|
|
919
|
+
if (typeNames.length !== typeCounts.length || typeNames.some((type) => type === "") || new Set(typeNames).size !== typeNames.length) throw new NativeParseError(FORMAT$14, counts.line, `${heterogeneous.path} must name each node type once, over one count per type`);
|
|
920
920
|
const nodeTypes = typeNames.map((type, position) => {
|
|
921
921
|
const count = typeCounts[position];
|
|
922
|
-
if (count === void 0) throw new NativeParseError(FORMAT$
|
|
922
|
+
if (count === void 0) throw new NativeParseError(FORMAT$14, counts.line, `${heterogeneous.path} gives node type ${type} no count`);
|
|
923
923
|
return {
|
|
924
924
|
type,
|
|
925
925
|
count: index(count, heterogeneous, counts.line)
|
|
@@ -932,10 +932,10 @@ function parseOgb(files) {
|
|
|
932
932
|
for (const { text, line } of rows(triplets)) {
|
|
933
933
|
const parts = text.split(",").map((part) => part.trim());
|
|
934
934
|
const [head, relation, tail] = parts;
|
|
935
|
-
if (parts.length !== TRIPLET_PARTS || head === void 0 || relation === void 0 || tail === void 0 || relation === "") throw new NativeParseError(FORMAT$
|
|
935
|
+
if (parts.length !== TRIPLET_PARTS || head === void 0 || relation === void 0 || tail === void 0 || relation === "") throw new NativeParseError(FORMAT$14, line, `${triplets.path}: expected a head type, a relation and a tail type, found ${JSON.stringify(text)}`);
|
|
936
936
|
const sources = nodeCounts.get(head);
|
|
937
937
|
const targets = nodeCounts.get(tail);
|
|
938
|
-
if (sources === void 0 || targets === void 0) throw new NativeParseError(FORMAT$
|
|
938
|
+
if (sources === void 0 || targets === void 0) throw new NativeParseError(FORMAT$14, line, `${triplets.path}: ${text} names a node type that ${heterogeneous.path} does not count`);
|
|
939
939
|
relations.push({
|
|
940
940
|
head,
|
|
941
941
|
relation,
|
|
@@ -948,7 +948,7 @@ function parseOgb(files) {
|
|
|
948
948
|
const listed = new Set(relations.map(({ head, relation, tail }) => `relations/${head}___${relation}___${tail}`));
|
|
949
949
|
for (const name of byName.keys()) {
|
|
950
950
|
const directory = name.slice(0, name.lastIndexOf("/"));
|
|
951
|
-
if (name.startsWith("relations/") && !listed.has(directory)) throw new NativeParseError(FORMAT$
|
|
951
|
+
if (name.startsWith("relations/") && !listed.has(directory)) throw new NativeParseError(FORMAT$14, 1, `raw/${directory}/ is not one of the relations ${triplets.path} lists`);
|
|
952
952
|
}
|
|
953
953
|
for (const { head, relation, tail, sources, targets } of relations) {
|
|
954
954
|
const directory = `relations/${head}___${relation}___${tail}`;
|
|
@@ -1201,69 +1201,71 @@ var RecordScanner = class {
|
|
|
1201
1201
|
}
|
|
1202
1202
|
};
|
|
1203
1203
|
/**
|
|
1204
|
-
*
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
carry = scanned.carry;
|
|
1217
|
-
yield* scanned.completed;
|
|
1204
|
+
* Reads a table file's records as its text arrives in pieces, so neither the whole file nor every cell is held at once: a leading byte order mark is removed, the physical lines the recipe skips are dropped, and the rest is split into records and fields by RFC 4180 (a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR). Blank lines are not records. A piece may end anywhere, even between the two quotes of a doubled quote or between a CR and its LF.
|
|
1205
|
+
*/
|
|
1206
|
+
var RecordReader = class {
|
|
1207
|
+
options;
|
|
1208
|
+
scanner;
|
|
1209
|
+
carry = "";
|
|
1210
|
+
first = true;
|
|
1211
|
+
toSkip;
|
|
1212
|
+
constructor(options) {
|
|
1213
|
+
this.options = options;
|
|
1214
|
+
this.scanner = new RecordScanner(DELIMITERS[options.delimiter].charCodeAt(0), options.skipLines + 1);
|
|
1215
|
+
this.toSkip = options.skipLines;
|
|
1218
1216
|
}
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1217
|
+
/**
|
|
1218
|
+
* Reads the next piece of the file's text.
|
|
1219
|
+
* @param piece - The text, which follows the pieces read before it.
|
|
1220
|
+
* @returns The records the piece completes, in order.
|
|
1221
|
+
* @throws NativeParseError when a quote opens inside an unquoted field.
|
|
1222
|
+
*/
|
|
1223
|
+
push(piece) {
|
|
1224
|
+
let rest = this.first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
|
|
1225
|
+
this.first = false;
|
|
1226
|
+
while (this.toSkip > 0) {
|
|
1227
|
+
const next = rest.indexOf("\n");
|
|
1228
|
+
if (next === -1) return [];
|
|
1229
|
+
rest = rest.slice(next + 1);
|
|
1230
|
+
this.toSkip -= 1;
|
|
1231
|
+
}
|
|
1232
|
+
if (rest === "") return [];
|
|
1233
|
+
const scanned = this.scanner.scan(this.carry + rest, false);
|
|
1234
|
+
this.carry = scanned.carry;
|
|
1235
|
+
return scanned.completed.filter(isNotBlank);
|
|
1236
|
+
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Ends the file.
|
|
1239
|
+
* @returns The records left when the text ends, in order.
|
|
1240
|
+
* @throws NativeParseError when a quoted field is never closed, a quote opens inside an unquoted field, or the file has fewer lines than the recipe skips.
|
|
1241
|
+
*/
|
|
1242
|
+
end() {
|
|
1243
|
+
if (this.toSkip > 0) throw new NativeParseError("csv-table", this.options.skipLines - this.toSkip + 1, `the file has fewer than ${String(this.options.skipLines)} lines to skip before the header`);
|
|
1244
|
+
const completed = this.scanner.scan(this.carry, true).completed;
|
|
1245
|
+
this.carry = "";
|
|
1246
|
+
const last = this.scanner.finish();
|
|
1247
|
+
return [...completed, ...last === void 0 ? [] : [last]].filter(isNotBlank);
|
|
1248
|
+
}
|
|
1249
|
+
};
|
|
1223
1250
|
/**
|
|
1224
|
-
* Splits comma-separated text into its records' fields by RFC 4180, as
|
|
1251
|
+
* Splits comma-separated text into its records' fields by RFC 4180, as the record reader does, keeping every record (blank lines included) at its position: the grid a workbook sheet's CSV makes, for a parser that reads a layout of blocks and not one table.
|
|
1225
1252
|
* @param text - The text.
|
|
1226
1253
|
* @returns Each record's fields, in order.
|
|
1227
1254
|
* @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
|
|
1228
1255
|
*/
|
|
1229
1256
|
function csvGrid(text) {
|
|
1230
|
-
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
* @param pieces - The file text in order.
|
|
1235
|
-
* @param count - How many physical lines to drop.
|
|
1236
|
-
* @returns The rest of the text, in pieces.
|
|
1237
|
-
* @throws NativeParseError when the file has fewer lines than that.
|
|
1238
|
-
*/
|
|
1239
|
-
function* afterLines(pieces, count) {
|
|
1240
|
-
let remaining = count;
|
|
1241
|
-
for (const piece of pieces) {
|
|
1242
|
-
let rest = piece;
|
|
1243
|
-
while (remaining > 0) {
|
|
1244
|
-
const next = rest.indexOf("\n");
|
|
1245
|
-
if (next === -1) {
|
|
1246
|
-
rest = "";
|
|
1247
|
-
break;
|
|
1248
|
-
}
|
|
1249
|
-
rest = rest.slice(next + 1);
|
|
1250
|
-
remaining -= 1;
|
|
1251
|
-
}
|
|
1252
|
-
if (remaining === 0 && rest !== "") yield rest;
|
|
1253
|
-
}
|
|
1254
|
-
if (remaining > 0) throw new NativeParseError("csv-table", count - remaining + 1, `the file has fewer than ${String(count)} lines to skip before the header`);
|
|
1257
|
+
const scanner = new RecordScanner(DELIMITERS.comma.charCodeAt(0), 1);
|
|
1258
|
+
const { completed } = scanner.scan(text, true);
|
|
1259
|
+
const last = scanner.finish();
|
|
1260
|
+
return [...completed, ...last === void 0 ? [] : [last]].map((record) => record.fields);
|
|
1255
1261
|
}
|
|
1256
1262
|
/**
|
|
1257
|
-
*
|
|
1258
|
-
* @param
|
|
1259
|
-
* @returns
|
|
1263
|
+
* Whether a record holds anything: a blank line is a record of one empty field.
|
|
1264
|
+
* @param record - The record.
|
|
1265
|
+
* @returns False for a blank line.
|
|
1260
1266
|
*/
|
|
1261
|
-
function
|
|
1262
|
-
|
|
1263
|
-
for (const piece of typeof text === "string" ? [text] : text) {
|
|
1264
|
-
yield first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
|
|
1265
|
-
first = false;
|
|
1266
|
-
}
|
|
1267
|
+
function isNotBlank(record) {
|
|
1268
|
+
return record.fields.length > 1 || record.fields[0]?.trim() !== "";
|
|
1267
1269
|
}
|
|
1268
1270
|
/**
|
|
1269
1271
|
* Reads a numeric cell.
|
|
@@ -1280,27 +1282,81 @@ function numeric$1(cell, factor, column, line) {
|
|
|
1280
1282
|
return Number(cell) * factor;
|
|
1281
1283
|
}
|
|
1282
1284
|
/**
|
|
1283
|
-
*
|
|
1284
|
-
* @param text - The file text, or its pieces in order, which may begin with a byte order mark.
|
|
1285
|
-
* @param options - The recipe's table parser, of which the delimiter and the lines to skip are read.
|
|
1286
|
-
* @returns Each record.
|
|
1287
|
-
* @throws NativeParseError when a quote is never closed or opens inside an unquoted field, or the file has fewer lines than the recipe skips.
|
|
1285
|
+
* Reads a delimited table into rows as its text arrives in pieces, holding only the record in progress between pieces, so a caller that hands each row on as it is returned holds no more of the table than that. The rows are exactly those `parseCsvTable` gives, in order, with the same errors on the same lines however the text is cut.
|
|
1288
1286
|
*/
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1287
|
+
var TableReader = class {
|
|
1288
|
+
options;
|
|
1289
|
+
records;
|
|
1290
|
+
layout;
|
|
1291
|
+
rows = 0;
|
|
1292
|
+
missing;
|
|
1293
|
+
constructor(options) {
|
|
1294
|
+
this.options = options;
|
|
1295
|
+
this.records = new RecordReader(options);
|
|
1296
|
+
this.missing = new Set(options.nullValues ?? []);
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Reads the next piece of the file's text.
|
|
1300
|
+
* @param piece - The text, which follows the pieces read before it.
|
|
1301
|
+
* @returns The rows the piece completes, in file order.
|
|
1302
|
+
* @throws NativeParseError as `parseCsvTable` does, naming the line of the problem.
|
|
1303
|
+
*/
|
|
1304
|
+
push(piece) {
|
|
1305
|
+
return this.rowsOf(this.records.push(piece));
|
|
1306
|
+
}
|
|
1307
|
+
/**
|
|
1308
|
+
* Ends the file.
|
|
1309
|
+
* @returns The rows left when the text ends, in file order.
|
|
1310
|
+
* @throws NativeParseError as `parseCsvTable` does, including for a file with no header row.
|
|
1311
|
+
*/
|
|
1312
|
+
end() {
|
|
1313
|
+
const rows = this.rowsOf(this.records.end());
|
|
1314
|
+
if (this.layout === void 0) throw new NativeParseError("csv-table", this.options.skipLines + 1, "the file has no header row");
|
|
1315
|
+
return rows;
|
|
1316
|
+
}
|
|
1317
|
+
/**
|
|
1318
|
+
* Builds the rows of records, the first record of the file being its header.
|
|
1319
|
+
* @param records - Records in file order.
|
|
1320
|
+
* @returns The rows among them.
|
|
1321
|
+
*/
|
|
1322
|
+
rowsOf(records) {
|
|
1323
|
+
const rows = [];
|
|
1324
|
+
for (const record of records) if (this.layout === void 0) this.layout = layoutOf(record, this.options);
|
|
1325
|
+
else rows.push(this.rowOf(record, this.layout));
|
|
1326
|
+
return rows;
|
|
1327
|
+
}
|
|
1328
|
+
/**
|
|
1329
|
+
* Builds one row.
|
|
1330
|
+
* @param record - A record after the header.
|
|
1331
|
+
* @param layout - The header's layout.
|
|
1332
|
+
* @returns The row.
|
|
1333
|
+
* @throws NativeParseError when the record's width differs from the header's, its id is empty or a numeric cell is not a number.
|
|
1334
|
+
*/
|
|
1335
|
+
rowOf(record, layout) {
|
|
1336
|
+
const { idColumn } = this.options;
|
|
1337
|
+
if (record.fields.length !== layout.width) throw new NativeParseError("csv-table", record.line, `the row has ${String(record.fields.length)} fields, the header has ${String(layout.width)}`);
|
|
1338
|
+
this.rows += 1;
|
|
1339
|
+
const id = idColumn === null ? String(this.rows) : (record.fields[layout.idPosition] ?? "").trim();
|
|
1340
|
+
if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
|
|
1341
|
+
const cells = layout.kept.map(({ name, position, factor }) => {
|
|
1342
|
+
const trimmed = (record.fields[position] ?? "").trim();
|
|
1343
|
+
const cell = this.missing.has(trimmed) ? "" : trimmed;
|
|
1344
|
+
return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
|
|
1345
|
+
});
|
|
1346
|
+
return {
|
|
1347
|
+
...Object.fromEntries(cells),
|
|
1348
|
+
id
|
|
1349
|
+
};
|
|
1350
|
+
}
|
|
1351
|
+
};
|
|
1293
1352
|
/**
|
|
1294
|
-
*
|
|
1295
|
-
* @param
|
|
1353
|
+
* Reads a table's header into the layout its rows are built by, checking the recipe against it.
|
|
1354
|
+
* @param header - The header record.
|
|
1296
1355
|
* @param options - The recipe's table parser.
|
|
1297
|
-
* @returns The
|
|
1298
|
-
* @throws NativeParseError
|
|
1356
|
+
* @returns The layout.
|
|
1357
|
+
* @throws NativeParseError when two header cells get the same name, the recipe names a column the header lacks, lists the id column among the columns, or declares numeric a column it does not keep.
|
|
1299
1358
|
*/
|
|
1300
|
-
function
|
|
1301
|
-
const table = tableRecords(text, options);
|
|
1302
|
-
const { value: header } = table.next();
|
|
1303
|
-
if (header === void 0) throw new NativeParseError("csv-table", options.skipLines + 1, "the file has no header row");
|
|
1359
|
+
function layoutOf(header, options) {
|
|
1304
1360
|
const names = columnNames(header.fields, options.idColumn, header.line);
|
|
1305
1361
|
const { idColumn, numericColumns, columns } = options;
|
|
1306
1362
|
const wanted = new Set(columns ?? names.filter((name) => name !== idColumn));
|
|
@@ -1312,28 +1368,28 @@ function parseCsvTable(text, options) {
|
|
|
1312
1368
|
if (idColumn !== null && wanted.has(idColumn)) throw new NativeParseError("csv-table", header.line, `the id column ${idColumn} is the row's id, so it is not listed among the columns`);
|
|
1313
1369
|
for (const { column } of numericColumns) if (!wanted.has(column)) throw new NativeParseError("csv-table", header.line, `the column ${column} is declared numeric but is not among the columns kept`);
|
|
1314
1370
|
const factors = new Map(numericColumns.map((entry) => [entry.column, entry.factor]));
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
name,
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1371
|
+
return {
|
|
1372
|
+
width: names.length,
|
|
1373
|
+
idPosition: idColumn === null ? -1 : names.indexOf(idColumn),
|
|
1374
|
+
kept: names.flatMap((name, position) => name !== idColumn && wanted.has(name) ? [{
|
|
1375
|
+
name,
|
|
1376
|
+
position,
|
|
1377
|
+
factor: factors.get(name)
|
|
1378
|
+
}] : [])
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
/**
|
|
1382
|
+
* Parses a delimited table into rows (a cell whose text is one of the recipe's `nullValues` is empty, so null): the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
|
|
1383
|
+
* @param text - The file text, or its pieces in order.
|
|
1384
|
+
* @param options - The recipe's table parser.
|
|
1385
|
+
* @returns The rows, in file order.
|
|
1386
|
+
* @throws NativeParseError naming the line of the first problem, including a listed or numeric column the header lacks or a numeric column that is not listed.
|
|
1387
|
+
*/
|
|
1388
|
+
function parseCsvTable(text, options) {
|
|
1389
|
+
const reader = new TableReader(options);
|
|
1322
1390
|
const nodes = [];
|
|
1323
|
-
for (const
|
|
1324
|
-
|
|
1325
|
-
const id = idColumn === null ? String(nodes.length + 1) : (record.fields[idPosition] ?? "").trim();
|
|
1326
|
-
if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
|
|
1327
|
-
const cells = kept.map(({ name, position, factor }) => {
|
|
1328
|
-
const trimmed = (record.fields[position] ?? "").trim();
|
|
1329
|
-
const cell = missing.has(trimmed) ? "" : trimmed;
|
|
1330
|
-
return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
|
|
1331
|
-
});
|
|
1332
|
-
nodes.push({
|
|
1333
|
-
...Object.fromEntries(cells),
|
|
1334
|
-
id
|
|
1335
|
-
});
|
|
1336
|
-
}
|
|
1391
|
+
for (const piece of typeof text === "string" ? [text] : text) for (const row of reader.push(piece)) nodes.push(row);
|
|
1392
|
+
for (const row of reader.end()) nodes.push(row);
|
|
1337
1393
|
return { nodes };
|
|
1338
1394
|
}
|
|
1339
1395
|
//#endregion
|
|
@@ -1438,7 +1494,7 @@ function parseGeojson(text, options) {
|
|
|
1438
1494
|
}
|
|
1439
1495
|
//#endregion
|
|
1440
1496
|
//#region src/parsers/hhcrsp.ts
|
|
1441
|
-
const FORMAT$
|
|
1497
|
+
const FORMAT$13 = "hhcrsp";
|
|
1442
1498
|
/** The labels of the sheet's blocks, each the top-left cell of a block whose header runs to its right and whose rows run below it. */
|
|
1443
1499
|
const LABEL = {
|
|
1444
1500
|
patients: "Patients",
|
|
@@ -1450,9 +1506,9 @@ const LABEL = {
|
|
|
1450
1506
|
};
|
|
1451
1507
|
/** A service's header, `Service <k>`, a patient's row label and a caregiver's. */
|
|
1452
1508
|
const SERVICE = /^Service (\d+)$/;
|
|
1453
|
-
const NUMBER$
|
|
1509
|
+
const NUMBER$8 = /^-?\d+(?:\.\d+)?$/;
|
|
1454
1510
|
/** How many required services a patient may have: the instances ask for one, or two that must be synchronised, and the parser has a duration column for each. */
|
|
1455
|
-
const MOST_SERVICES = 2;
|
|
1511
|
+
const MOST_SERVICES$1 = 2;
|
|
1456
1512
|
/**
|
|
1457
1513
|
* Finds a block by its label: the one cell of the sheet that holds it, its header the cells to its right up to the first empty one, and its rows those below it up to the first whose label and cells are all empty. Past its header a block has an empty column, so the cell there must be empty in every row and a value the header does not label is refused rather than left unread; `Pat / Ser` instead keeps its count there, with the next block starting right after it, and its count is checked on its own.
|
|
1458
1514
|
* @param grid - The sheet's cells by row.
|
|
@@ -1467,7 +1523,7 @@ function blockOf(grid, label, pastHeader = "empty") {
|
|
|
1467
1523
|
column
|
|
1468
1524
|
}] : []));
|
|
1469
1525
|
const [only] = found;
|
|
1470
|
-
if (only === void 0 || found.length !== 1) throw new NativeParseError(FORMAT$
|
|
1526
|
+
if (only === void 0 || found.length !== 1) throw new NativeParseError(FORMAT$13, 1, `expected one block labelled ${JSON.stringify(label)}, found ${String(found.length)}`);
|
|
1471
1527
|
const headerRow = grid[only.line] ?? [];
|
|
1472
1528
|
const end = headerRow.findIndex((cell, column) => column > only.column && cell.trim() === "");
|
|
1473
1529
|
const header = headerRow.slice(only.column + 1, end === -1 ? headerRow.length : end).map((cell) => cell.trim());
|
|
@@ -1476,7 +1532,7 @@ function blockOf(grid, label, pastHeader = "empty") {
|
|
|
1476
1532
|
const cells = (grid[line] ?? []).slice(only.column, only.column + 1 + header.length).map((cell) => cell.trim());
|
|
1477
1533
|
if (cells.every((cell) => cell === "")) break;
|
|
1478
1534
|
const past = (grid[line] ?? [])[only.column + 1 + header.length]?.trim();
|
|
1479
|
-
if (pastHeader === "empty" && past !== void 0 && past !== "") throw new NativeParseError(FORMAT$
|
|
1535
|
+
if (pastHeader === "empty" && past !== void 0 && past !== "") throw new NativeParseError(FORMAT$13, line + 1, `the ${label} block's row holds ${JSON.stringify(past)} past its header ${JSON.stringify(header)}, which does not label it`);
|
|
1480
1536
|
rows.push({
|
|
1481
1537
|
line: line + 1,
|
|
1482
1538
|
label: cells[0] ?? "",
|
|
@@ -1499,7 +1555,7 @@ function blockOf(grid, label, pastHeader = "empty") {
|
|
|
1499
1555
|
* @throws NativeParseError when the cell is not a decimal number.
|
|
1500
1556
|
*/
|
|
1501
1557
|
function numberOf$4(cell, line, what) {
|
|
1502
|
-
if (cell === void 0 || !NUMBER$
|
|
1558
|
+
if (cell === void 0 || !NUMBER$8.test(cell)) throw new NativeParseError(FORMAT$13, line, `expected ${what}, found ${JSON.stringify(cell)}`);
|
|
1503
1559
|
return Number(cell);
|
|
1504
1560
|
}
|
|
1505
1561
|
/**
|
|
@@ -1511,7 +1567,7 @@ function numberOf$4(cell, line, what) {
|
|
|
1511
1567
|
*/
|
|
1512
1568
|
function checkNumbered(block, noun, label) {
|
|
1513
1569
|
block.rows.forEach((row, index) => {
|
|
1514
|
-
if (row.label !== `${noun} ${String(index + 1)}`) throw new NativeParseError(FORMAT$
|
|
1570
|
+
if (row.label !== `${noun} ${String(index + 1)}`) throw new NativeParseError(FORMAT$13, row.line, `the ${label} block's row ${String(index + 1)} is labelled ${JSON.stringify(row.label)}, not ${noun} ${String(index + 1)}`);
|
|
1515
1571
|
});
|
|
1516
1572
|
}
|
|
1517
1573
|
/**
|
|
@@ -1523,7 +1579,7 @@ function checkNumbered(block, noun, label) {
|
|
|
1523
1579
|
*/
|
|
1524
1580
|
function serviceCount(block, label) {
|
|
1525
1581
|
block.header.forEach((cell, index) => {
|
|
1526
|
-
if (SERVICE.exec(cell)?.[1] !== String(index + 1)) throw new NativeParseError(FORMAT$
|
|
1582
|
+
if (SERVICE.exec(cell)?.[1] !== String(index + 1)) throw new NativeParseError(FORMAT$13, block.line, `the ${label} block's header cell ${String(index + 1)} is ${JSON.stringify(cell)}, not Service ${String(index + 1)}`);
|
|
1527
1583
|
});
|
|
1528
1584
|
return block.header.length;
|
|
1529
1585
|
}
|
|
@@ -1542,7 +1598,7 @@ function parseHhcrsp(text) {
|
|
|
1542
1598
|
const caregiverWindows = blockOf(grid, LABEL.caregiverWindows);
|
|
1543
1599
|
const windows = blockOf(grid, LABEL.patientWindows);
|
|
1544
1600
|
const serviceKinds = serviceCount(requests, LABEL.requests);
|
|
1545
|
-
if (serviceCount(times, LABEL.times) !== serviceKinds || serviceCount(skills, LABEL.skills) !== serviceKinds) throw new NativeParseError(FORMAT$
|
|
1601
|
+
if (serviceCount(times, LABEL.times) !== serviceKinds || serviceCount(skills, LABEL.skills) !== serviceKinds) throw new NativeParseError(FORMAT$13, times.line, `the ${LABEL.requests}, ${LABEL.times} and ${LABEL.skills} blocks name different numbers of services`);
|
|
1546
1602
|
checkNumbered(requests, "Patient", LABEL.requests);
|
|
1547
1603
|
checkNumbered(windows, "Patient", LABEL.patientWindows);
|
|
1548
1604
|
checkNumbered(skills, "Caregiver", LABEL.skills);
|
|
@@ -1557,11 +1613,11 @@ function parseHhcrsp(text) {
|
|
|
1557
1613
|
"",
|
|
1558
1614
|
"center",
|
|
1559
1615
|
"centre"
|
|
1560
|
-
].includes(centreTimes.label.toLowerCase()) || windows.rows.length !== patients || patientTimes.length !== patients || skills.rows.length !== caregiverWindows.rows.length) throw new NativeParseError(FORMAT$
|
|
1561
|
-
if (positions.header.join(",") !== "X,Y" || positions.rows.length !== patients + 2 || positions.rows.some((row, index) => row.label !== String(index))) throw new NativeParseError(FORMAT$
|
|
1616
|
+
].includes(centreTimes.label.toLowerCase()) || windows.rows.length !== patients || patientTimes.length !== patients || skills.rows.length !== caregiverWindows.rows.length) throw new NativeParseError(FORMAT$13, requests.line, `the blocks disagree: ${String(patients)} patients requesting services, ${String(patientTimes.length)} with durations after the centre's row, ${String(windows.rows.length)} with time windows, and ${String(skills.rows.length)} caregivers with skills against ${String(caregiverWindows.rows.length)} with windows`);
|
|
1617
|
+
if (positions.header.join(",") !== "X,Y" || positions.rows.length !== patients + 2 || positions.rows.some((row, index) => row.label !== String(index))) throw new NativeParseError(FORMAT$13, positions.line, `the ${LABEL.patients} block must give X and Y for nodes 0 to ${String(patients + 1)}, the centre, ${String(patients)} patients and the centre again`);
|
|
1562
1618
|
const position = (index) => {
|
|
1563
1619
|
const row = positions.rows[index];
|
|
1564
|
-
if (row === void 0) throw new NativeParseError(FORMAT$
|
|
1620
|
+
if (row === void 0) throw new NativeParseError(FORMAT$13, positions.line, `the ${LABEL.patients} block has no node ${String(index)}`);
|
|
1565
1621
|
return {
|
|
1566
1622
|
x: numberOf$4(row.cells[0], row.line, "an X"),
|
|
1567
1623
|
y: numberOf$4(row.cells[1], row.line, "a Y"),
|
|
@@ -1570,7 +1626,7 @@ function parseHhcrsp(text) {
|
|
|
1570
1626
|
};
|
|
1571
1627
|
const { x, y } = position(0);
|
|
1572
1628
|
const closing = position(patients + 1);
|
|
1573
|
-
if (closing.x !== x || closing.y !== y) throw new NativeParseError(FORMAT$
|
|
1629
|
+
if (closing.x !== x || closing.y !== y) throw new NativeParseError(FORMAT$13, closing.line, `the last node of the ${LABEL.patients} block is not at the centre's position, so it is not the route's end`);
|
|
1574
1630
|
const windowColumn = (name) => {
|
|
1575
1631
|
const index = windows.header.indexOf(name);
|
|
1576
1632
|
return index === -1 ? void 0 : index;
|
|
@@ -1581,7 +1637,7 @@ function parseHhcrsp(text) {
|
|
|
1581
1637
|
earliest2: windowColumn("ETW 2"),
|
|
1582
1638
|
latest2: windowColumn("LTW 2")
|
|
1583
1639
|
};
|
|
1584
|
-
if (columns.earliest1 === void 0 || columns.latest1 === void 0 || columns.earliest2 === void 0 !== (columns.latest2 === void 0) || windows.header.length !== (columns.earliest2 === void 0 ? 2 : 4)) throw new NativeParseError(FORMAT$
|
|
1640
|
+
if (columns.earliest1 === void 0 || columns.latest1 === void 0 || columns.earliest2 === void 0 !== (columns.latest2 === void 0) || windows.header.length !== (columns.earliest2 === void 0 ? 2 : 4)) throw new NativeParseError(FORMAT$13, windows.line, `the ${LABEL.patientWindows} header ${JSON.stringify(windows.header)} is not ETW 1 and LTW 1, with or without ETW 2 and LTW 2`);
|
|
1585
1641
|
const nodes = [{
|
|
1586
1642
|
id: "0",
|
|
1587
1643
|
role: "centre",
|
|
@@ -1597,15 +1653,15 @@ function parseHhcrsp(text) {
|
|
|
1597
1653
|
}];
|
|
1598
1654
|
requests.rows.forEach((request, index) => {
|
|
1599
1655
|
const required = request.cells.flatMap((cell, service) => {
|
|
1600
|
-
if (cell !== "0" && cell !== "1" && cell !== "") throw new NativeParseError(FORMAT$
|
|
1656
|
+
if (cell !== "0" && cell !== "1" && cell !== "") throw new NativeParseError(FORMAT$13, request.line, `${request.label} has ${JSON.stringify(cell)} for Service ${String(service + 1)}, not 0, 1 or empty`);
|
|
1601
1657
|
return cell === "1" ? [service] : [];
|
|
1602
1658
|
});
|
|
1603
1659
|
const stated = (grid[request.line - 1] ?? [])[requests.column + serviceKinds + 1]?.trim();
|
|
1604
|
-
if (stated !== String(required.length)) throw new NativeParseError(FORMAT$
|
|
1605
|
-
if (required.length === 0 || required.length > MOST_SERVICES) throw new NativeParseError(FORMAT$
|
|
1660
|
+
if (stated !== String(required.length)) throw new NativeParseError(FORMAT$13, request.line, `${request.label} requires ${String(required.length)} services by its marks, and the count beside them is ${JSON.stringify(stated)}`);
|
|
1661
|
+
if (required.length === 0 || required.length > MOST_SERVICES$1) throw new NativeParseError(FORMAT$13, request.line, `${request.label} requires ${String(required.length)} services, and a patient requires one or two`);
|
|
1606
1662
|
const timeRow = patientTimes[index];
|
|
1607
1663
|
const windowRow = windows.rows[index];
|
|
1608
|
-
if (timeRow === void 0 || windowRow === void 0) throw new NativeParseError(FORMAT$
|
|
1664
|
+
if (timeRow === void 0 || windowRow === void 0) throw new NativeParseError(FORMAT$13, request.line, `${request.label} has no durations or time windows`);
|
|
1609
1665
|
const duration = (which) => {
|
|
1610
1666
|
const service = required[which];
|
|
1611
1667
|
return service === void 0 ? null : numberOf$4(timeRow.cells[service], timeRow.line, `the duration of Service ${String(service + 1)}`);
|
|
@@ -1636,6 +1692,433 @@ function parseHhcrsp(text) {
|
|
|
1636
1692
|
};
|
|
1637
1693
|
}
|
|
1638
1694
|
//#endregion
|
|
1695
|
+
//#region src/parsers/mankowska.ts
|
|
1696
|
+
const FORMAT$12 = "mankowska";
|
|
1697
|
+
/** The sections of the file, each a keyword on a line of its own followed by its numbers. */
|
|
1698
|
+
const SECTIONS$1 = [
|
|
1699
|
+
"nbNodes",
|
|
1700
|
+
"nbVehi",
|
|
1701
|
+
"nbServi",
|
|
1702
|
+
"r",
|
|
1703
|
+
"DS",
|
|
1704
|
+
"a",
|
|
1705
|
+
"x",
|
|
1706
|
+
"y",
|
|
1707
|
+
"d",
|
|
1708
|
+
"p",
|
|
1709
|
+
"mind",
|
|
1710
|
+
"maxd",
|
|
1711
|
+
"e",
|
|
1712
|
+
"l"
|
|
1713
|
+
];
|
|
1714
|
+
/** How many patients a double service has, and so the most required services a patient can have. */
|
|
1715
|
+
const MOST_SERVICES = 2;
|
|
1716
|
+
/** The depot copies: the first and the last node. */
|
|
1717
|
+
const DEPOTS = 2;
|
|
1718
|
+
const NUMBER$7 = /^-?\d+(?:\.\d+)?$/;
|
|
1719
|
+
const KEYWORD = /^[A-Za-z]+$/;
|
|
1720
|
+
/**
|
|
1721
|
+
* Splits the file into its sections: a line holding one word starts a section, and every number on the lines after it, up to the next such line, is the section's.
|
|
1722
|
+
* @param text - The file text.
|
|
1723
|
+
* @returns Each section by its keyword.
|
|
1724
|
+
* @throws NativeParseError for a keyword the format does not have, one given twice, a value that is not a number, or a number before the first keyword.
|
|
1725
|
+
*/
|
|
1726
|
+
function sectionsOf$1(text) {
|
|
1727
|
+
const sections = /* @__PURE__ */ new Map();
|
|
1728
|
+
let current;
|
|
1729
|
+
for (const [index, raw] of lines(text).entries()) {
|
|
1730
|
+
const tokens = raw.trim().split(/\s+/).filter((token) => token !== "");
|
|
1731
|
+
const [only] = tokens;
|
|
1732
|
+
if (tokens.length === 1 && only !== void 0 && KEYWORD.test(only)) {
|
|
1733
|
+
const name = SECTIONS$1.find((section) => section === only);
|
|
1734
|
+
if (name === void 0 || sections.has(name)) throw new NativeParseError(FORMAT$12, index + 1, name === void 0 ? `the section ${JSON.stringify(only)} is not one of ${SECTIONS$1.join(", ")}` : `the section ${name} is given twice`);
|
|
1735
|
+
current = [];
|
|
1736
|
+
sections.set(name, {
|
|
1737
|
+
line: index + 1,
|
|
1738
|
+
values: current
|
|
1739
|
+
});
|
|
1740
|
+
continue;
|
|
1741
|
+
}
|
|
1742
|
+
for (const token of tokens) {
|
|
1743
|
+
if (current === void 0 || !NUMBER$7.test(token)) throw new NativeParseError(FORMAT$12, index + 1, current === void 0 ? "a number comes before the first section" : `expected a number, found ${JSON.stringify(token)}`);
|
|
1744
|
+
current.push(Number(token));
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
return sections;
|
|
1748
|
+
}
|
|
1749
|
+
/**
|
|
1750
|
+
* Reads a section that must hold a given count of numbers.
|
|
1751
|
+
* @param sections - The sections.
|
|
1752
|
+
* @param name - The section.
|
|
1753
|
+
* @param count - How many numbers it must hold.
|
|
1754
|
+
* @returns Its numbers.
|
|
1755
|
+
* @throws NativeParseError when the section is missing or holds another count.
|
|
1756
|
+
*/
|
|
1757
|
+
function sized(sections, name, count) {
|
|
1758
|
+
const section = sections.get(name);
|
|
1759
|
+
if (section === void 0) throw new NativeParseError(FORMAT$12, 1, `the file has no ${name} section`);
|
|
1760
|
+
if (section.values.length !== count) throw new NativeParseError(FORMAT$12, section.line, `the ${name} section holds ${String(section.values.length)} numbers where ${String(count)} are due`);
|
|
1761
|
+
return section.values;
|
|
1762
|
+
}
|
|
1763
|
+
/**
|
|
1764
|
+
* Reads a count section (`nbNodes`, `nbVehi`, `nbServi`): one whole number of at least a least value.
|
|
1765
|
+
* @param sections - The sections.
|
|
1766
|
+
* @param name - The section.
|
|
1767
|
+
* @param least - The smallest count that makes an instance.
|
|
1768
|
+
* @returns The count.
|
|
1769
|
+
* @throws NativeParseError when the section is missing, holds more than one number, or the count is not a whole number of at least `least`.
|
|
1770
|
+
*/
|
|
1771
|
+
function countOf(sections, name, least) {
|
|
1772
|
+
const [count] = sized(sections, name, 1);
|
|
1773
|
+
if (count === void 0 || !Number.isInteger(count) || count < least) throw new NativeParseError(FORMAT$12, sections.get(name)?.line ?? 1, `${name} must be a whole number of at least ${String(least)}, found ${String(count)}`);
|
|
1774
|
+
return count;
|
|
1775
|
+
}
|
|
1776
|
+
/**
|
|
1777
|
+
* Reads one cell of a row-major matrix section.
|
|
1778
|
+
* @param values - The section's numbers.
|
|
1779
|
+
* @param columns - The matrix's width.
|
|
1780
|
+
* @param row - The row.
|
|
1781
|
+
* @param column - The column.
|
|
1782
|
+
* @returns The cell.
|
|
1783
|
+
* @throws NativeParseError when the cell is outside the section, which `sized` has already ruled out.
|
|
1784
|
+
*/
|
|
1785
|
+
function cell(values, columns, row, column) {
|
|
1786
|
+
const value = values[row * columns + column];
|
|
1787
|
+
if (value === void 0) throw new NativeParseError(FORMAT$12, 1, `cell ${String(row)}, ${String(column)} is outside its section`);
|
|
1788
|
+
return value;
|
|
1789
|
+
}
|
|
1790
|
+
/**
|
|
1791
|
+
* Parses a home health care routing and scheduling instance in the keyword text format of Mankowska, Meisel and Bierwirth (2014), as the OVIG generator writes it: the counts of nodes (`nbNodes`, the patients and two copies of the depot, the first and the last), vehicles (`nbVehi`, the caregivers) and services (`nbServi`); the services each node requires (`r`, a node by service matrix of zeros and ones), the double service nodes (`DS`, node numbers from one), the services each caregiver can give (`a`), the positions (`x`, `y`), the travel matrix (`d`), the processing time of each service at each node by each caregiver (`p`, a row per node and caregiver), the least and most time between a double service's two services (`mind`, `maxd`), and the time windows (`e`, `l`). A node's id is its number from zero. The depot copies are marked as requiring every service, which is the format's convention and not a visit, so they carry no services or processing times. A patient's required services are their numbers from one, joined by spaces, and its first and second processing time are those of its required services in that order. The travel matrix and the caregivers' qualifications are not node fields and are not kept; the processing times must not differ between caregivers, since a node column holds one value. The gaps are kept for a double service node only. The units of time are the generator's own.
|
|
1792
|
+
* @param text - The file text.
|
|
1793
|
+
* @returns The instance: its patient, caregiver and service kind counts as `meta`, and its nodes in node order.
|
|
1794
|
+
* @throws NativeParseError naming the line for a section that is missing, given twice or of the wrong size, a value that is not a number, a requirement that is not zero or one, a node with more than two required services, a double service node with fewer than two or out of range, or processing times that differ between caregivers.
|
|
1795
|
+
*/
|
|
1796
|
+
function parseMankowska(text) {
|
|
1797
|
+
const sections = sectionsOf$1(text);
|
|
1798
|
+
const nodeCount = countOf(sections, "nbNodes", DEPOTS);
|
|
1799
|
+
const caregivers = countOf(sections, "nbVehi", 1);
|
|
1800
|
+
const serviceKinds = countOf(sections, "nbServi", 1);
|
|
1801
|
+
const required = sized(sections, "r", nodeCount * serviceKinds);
|
|
1802
|
+
sized(sections, "a", caregivers * serviceKinds);
|
|
1803
|
+
const xs = sized(sections, "x", nodeCount);
|
|
1804
|
+
const ys = sized(sections, "y", nodeCount);
|
|
1805
|
+
sized(sections, "d", nodeCount * nodeCount);
|
|
1806
|
+
const times = sized(sections, "p", nodeCount * caregivers * serviceKinds);
|
|
1807
|
+
const least = sized(sections, "mind", nodeCount);
|
|
1808
|
+
const most = sized(sections, "maxd", nodeCount);
|
|
1809
|
+
const earliest = sized(sections, "e", nodeCount);
|
|
1810
|
+
const latest = sized(sections, "l", nodeCount);
|
|
1811
|
+
const line = (name) => sections.get(name)?.line ?? 1;
|
|
1812
|
+
const doubles = new Set((sections.get("DS")?.values ?? []).map((number) => {
|
|
1813
|
+
if (!Number.isInteger(number) || number < 1 || number > nodeCount) throw new NativeParseError(FORMAT$12, line("DS"), `the double service node ${String(number)} is not a node number from 1 to ${String(nodeCount)}`);
|
|
1814
|
+
return number - 1;
|
|
1815
|
+
}));
|
|
1816
|
+
const nodes = Array.from({ length: nodeCount }, (_, node) => {
|
|
1817
|
+
const at = (values) => cell(values, 1, node, 0);
|
|
1818
|
+
if (node === 0 || node === nodeCount - 1) return {
|
|
1819
|
+
id: String(node),
|
|
1820
|
+
role: "depot",
|
|
1821
|
+
x: at(xs),
|
|
1822
|
+
y: at(ys),
|
|
1823
|
+
services: null,
|
|
1824
|
+
duration1: null,
|
|
1825
|
+
duration2: null,
|
|
1826
|
+
earliest: at(earliest),
|
|
1827
|
+
latest: at(latest),
|
|
1828
|
+
minGap: null,
|
|
1829
|
+
maxGap: null
|
|
1830
|
+
};
|
|
1831
|
+
const services = Array.from({ length: serviceKinds }, (__, service) => service).filter((service) => {
|
|
1832
|
+
const flag = cell(required, serviceKinds, node, service);
|
|
1833
|
+
if (flag !== 0 && flag !== 1) throw new NativeParseError(FORMAT$12, line("r"), `node ${String(node)} has the requirement ${String(flag)} for service ${String(service + 1)}, not 0 or 1`);
|
|
1834
|
+
return flag === 1;
|
|
1835
|
+
});
|
|
1836
|
+
if (services.length > MOST_SERVICES || doubles.has(node) && services.length !== MOST_SERVICES) throw new NativeParseError(FORMAT$12, line("r"), `node ${String(node)} requires ${String(services.length)} services${doubles.has(node) ? " but is a double service node" : ""}`);
|
|
1837
|
+
const [first, second] = services.map((service) => {
|
|
1838
|
+
const byCaregiver = Array.from({ length: caregivers }, (__, caregiver) => cell(times, serviceKinds, node * caregivers + caregiver, service));
|
|
1839
|
+
const [time] = byCaregiver;
|
|
1840
|
+
if (time === void 0 || byCaregiver.some((other) => other !== time)) throw new NativeParseError(FORMAT$12, line("p"), `node ${String(node)} has processing times for service ${String(service + 1)} that differ between caregivers (${byCaregiver.join(", ")})`);
|
|
1841
|
+
return time;
|
|
1842
|
+
});
|
|
1843
|
+
return {
|
|
1844
|
+
id: String(node),
|
|
1845
|
+
role: "patient",
|
|
1846
|
+
x: at(xs),
|
|
1847
|
+
y: at(ys),
|
|
1848
|
+
services: services.length === 0 ? null : services.map((service) => String(service + 1)).join(" "),
|
|
1849
|
+
duration1: first ?? null,
|
|
1850
|
+
duration2: second ?? null,
|
|
1851
|
+
earliest: at(earliest),
|
|
1852
|
+
latest: at(latest),
|
|
1853
|
+
minGap: doubles.has(node) ? at(least) : null,
|
|
1854
|
+
maxGap: doubles.has(node) ? at(most) : null
|
|
1855
|
+
};
|
|
1856
|
+
});
|
|
1857
|
+
return {
|
|
1858
|
+
meta: {
|
|
1859
|
+
patients: nodeCount - DEPOTS,
|
|
1860
|
+
caregivers,
|
|
1861
|
+
serviceKinds
|
|
1862
|
+
},
|
|
1863
|
+
nodes
|
|
1864
|
+
};
|
|
1865
|
+
}
|
|
1866
|
+
//#endregion
|
|
1867
|
+
//#region src/parsers/tdc.ts
|
|
1868
|
+
const FORMAT$11 = "tdc";
|
|
1869
|
+
/** A technician line: its id, skills, time window and whether it is an investment option. */
|
|
1870
|
+
const TECHNICIAN = /^Technician id \d+, skills \[[^\]]*\], time window \[[^\]]*\], investment option (?:true|false)$/;
|
|
1871
|
+
/** A task line, with the fields the parser keeps captured: id, skills, duration, window opening and closing, priority, penalty, creation, modification (or `nothing`) and cancellation; its relevant technicians follow and are not kept. */
|
|
1872
|
+
const TASK = /^Task id (\d+), skills \[([^\]]*)\], duration (\d+), time window \[([^,\]]+), ([^\]]+)\], priority (\d+), penalty (-?\d+(?:\.\d+)?), created ([^,]+), modified ([^,]+), cancelled (true|false), relevant technician ids (?:Int64)?\[[\d, ]*\]$/;
|
|
1873
|
+
/** A travel time line, between two tasks or between a technician and a task, in either direction. */
|
|
1874
|
+
const TRAVEL = /^(?:task|technician)\d+(?:task|technician)\d+: -?\d+(?:\.\d+)?$/;
|
|
1875
|
+
/** The section headings, in the order the file gives them. */
|
|
1876
|
+
const HEADINGS = [
|
|
1877
|
+
"Technicians:",
|
|
1878
|
+
"Tasks:",
|
|
1879
|
+
"Travel time:"
|
|
1880
|
+
];
|
|
1881
|
+
/** What a line before any heading and a line of each section is, for errors: indexed by the section's position plus one, so the lines before the first heading are first. */
|
|
1882
|
+
const SECTION_NOUNS = [
|
|
1883
|
+
"heading",
|
|
1884
|
+
"technician",
|
|
1885
|
+
"task",
|
|
1886
|
+
"travel time"
|
|
1887
|
+
];
|
|
1888
|
+
/**
|
|
1889
|
+
* Names what a position's lines are, for an error.
|
|
1890
|
+
* @param position - A section's position plus one.
|
|
1891
|
+
* @returns The noun.
|
|
1892
|
+
*/
|
|
1893
|
+
function nounOf(position) {
|
|
1894
|
+
return SECTION_NOUNS[position] ?? `section ${String(position)}`;
|
|
1895
|
+
}
|
|
1896
|
+
/** Where the task pattern captures each field it keeps. */
|
|
1897
|
+
const TASK_AT = {
|
|
1898
|
+
id: 1,
|
|
1899
|
+
skills: 2,
|
|
1900
|
+
duration: 3,
|
|
1901
|
+
earliest: 4,
|
|
1902
|
+
latest: 5,
|
|
1903
|
+
priority: 6,
|
|
1904
|
+
penalty: 7,
|
|
1905
|
+
created: 8,
|
|
1906
|
+
modified: 9,
|
|
1907
|
+
cancelled: 10
|
|
1908
|
+
};
|
|
1909
|
+
/** How much of a refused line an error shows. */
|
|
1910
|
+
const SHOWN = 120;
|
|
1911
|
+
/**
|
|
1912
|
+
* Parses a technician routing and scheduling instance in the sectioned text format of Gamst and Pisinger (2024): a `Technicians:` heading and a line per technician, a `Tasks:` heading and a line per task (`Task id <n>, skills ["<skill>", ...], duration <minutes>, time window [<opening>, <closing>], priority <n>, penalty <n>, created <time>, modified <time or nothing>, cancelled <true or false>, relevant technician ids [<ids>]`), then a `Travel time:` heading and a line per pair, the travel minutes between two tasks or between a technician and a task. The file is read a line at a time and the travel time lines, almost all of a file, are checked and let go, as the tasks/v1 target has no arcs; a technician's line is counted. A task's id is its id, and a modification of `nothing` is null. Times are as the file writes them.
|
|
1913
|
+
* @param pieces - The file's text, in pieces.
|
|
1914
|
+
* @returns The instance: its technician count as `meta`, and its tasks in file order.
|
|
1915
|
+
* @throws NativeParseError naming the line for a heading out of order or missing, a line that is not of its section's shape, or a task id given twice.
|
|
1916
|
+
*/
|
|
1917
|
+
function parseTdc(pieces) {
|
|
1918
|
+
let section = -1;
|
|
1919
|
+
let technicians = 0;
|
|
1920
|
+
const nodes = [];
|
|
1921
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1922
|
+
forEachLine(pieces, (raw, line) => {
|
|
1923
|
+
const text = raw.trim();
|
|
1924
|
+
if (text === "") return;
|
|
1925
|
+
const heading = HEADINGS.findIndex((candidate) => candidate === text);
|
|
1926
|
+
if (heading !== -1) {
|
|
1927
|
+
if (heading !== section + 1) throw new NativeParseError(FORMAT$11, line, `the heading ${text} is out of order`);
|
|
1928
|
+
section = heading;
|
|
1929
|
+
return;
|
|
1930
|
+
}
|
|
1931
|
+
if (section === 0 && TECHNICIAN.test(text)) {
|
|
1932
|
+
technicians += 1;
|
|
1933
|
+
return;
|
|
1934
|
+
}
|
|
1935
|
+
if (section === 2 && TRAVEL.test(text)) return;
|
|
1936
|
+
const task = section === 1 ? TASK.exec(text) : null;
|
|
1937
|
+
if (task === null) throw new NativeParseError(FORMAT$11, line, `the line is not a ${nounOf(section + 1)} line: ${JSON.stringify(text.slice(0, SHOWN))}`);
|
|
1938
|
+
const field = (index) => {
|
|
1939
|
+
const value = task[index];
|
|
1940
|
+
if (value === void 0) throw new NativeParseError(FORMAT$11, line, `the task line has no field ${String(index)}`);
|
|
1941
|
+
return value;
|
|
1942
|
+
};
|
|
1943
|
+
const id = field(TASK_AT.id);
|
|
1944
|
+
if (ids.has(id)) throw new NativeParseError(FORMAT$11, line, `the task ${id} is given twice`);
|
|
1945
|
+
ids.add(id);
|
|
1946
|
+
const skills = field(TASK_AT.skills).split(",").map((skill) => skill.trim().replace(/^"|"$/g, "")).filter((skill) => skill !== "");
|
|
1947
|
+
const modified = field(TASK_AT.modified);
|
|
1948
|
+
nodes.push({
|
|
1949
|
+
id,
|
|
1950
|
+
skills: skills.length === 0 ? null : skills.join(" "),
|
|
1951
|
+
durationMinutes: Number(field(TASK_AT.duration)),
|
|
1952
|
+
earliest: field(TASK_AT.earliest).trim(),
|
|
1953
|
+
latest: field(TASK_AT.latest).trim(),
|
|
1954
|
+
priority: Number(field(TASK_AT.priority)),
|
|
1955
|
+
penalty: Number(field(TASK_AT.penalty)),
|
|
1956
|
+
created: field(TASK_AT.created),
|
|
1957
|
+
modified: modified === "nothing" ? null : modified,
|
|
1958
|
+
cancelled: field(TASK_AT.cancelled)
|
|
1959
|
+
});
|
|
1960
|
+
});
|
|
1961
|
+
if (section !== HEADINGS.length - 1) throw new NativeParseError(FORMAT$11, 1, `the file has no ${nounOf(section + 2)} heading`);
|
|
1962
|
+
return {
|
|
1963
|
+
meta: { technicians },
|
|
1964
|
+
nodes
|
|
1965
|
+
};
|
|
1966
|
+
}
|
|
1967
|
+
//#endregion
|
|
1968
|
+
//#region src/parsers/verolog-2017.ts
|
|
1969
|
+
const FORMAT$10 = "verolog-2017";
|
|
1970
|
+
/** The header lines, with the key of `meta` each fills and whether its value is text. */
|
|
1971
|
+
const HEADER$3 = /* @__PURE__ */ new Map([
|
|
1972
|
+
["DATASET", ["dataset", "text"]],
|
|
1973
|
+
["NAME", ["name", "text"]],
|
|
1974
|
+
["DAYS", ["days", "number"]],
|
|
1975
|
+
["CAPACITY", ["capacity", "number"]],
|
|
1976
|
+
["MAX_TRIP_DISTANCE", ["maxTripDistance", "number"]],
|
|
1977
|
+
["DEPOT_COORDINATE", ["depotCoordinate", "number"]],
|
|
1978
|
+
["VEHICLE_COST", ["vehicleCost", "number"]],
|
|
1979
|
+
["VEHICLE_DAY_COST", ["vehicleDayCost", "number"]],
|
|
1980
|
+
["DISTANCE_COST", ["distanceCost", "number"]]
|
|
1981
|
+
]);
|
|
1982
|
+
/** The sections that hold rows, and how many numbers each row has. */
|
|
1983
|
+
const SECTION_WIDTHS = {
|
|
1984
|
+
TOOLS: 4,
|
|
1985
|
+
COORDINATES: 3,
|
|
1986
|
+
REQUESTS: 7
|
|
1987
|
+
};
|
|
1988
|
+
/** Where a request row keeps its fields. */
|
|
1989
|
+
const REQUEST_AT$1 = {
|
|
1990
|
+
id: 0,
|
|
1991
|
+
location: 1,
|
|
1992
|
+
fromDay: 2,
|
|
1993
|
+
toDay: 3,
|
|
1994
|
+
stayDays: 4,
|
|
1995
|
+
tool: 5,
|
|
1996
|
+
amount: 6
|
|
1997
|
+
};
|
|
1998
|
+
const WHOLE$3 = /^-?\d+$/;
|
|
1999
|
+
/**
|
|
2000
|
+
* Reads a field of a row.
|
|
2001
|
+
* @param row - The row.
|
|
2002
|
+
* @param index - The field's position.
|
|
2003
|
+
* @returns The number.
|
|
2004
|
+
* @throws NativeParseError when the row has no such field.
|
|
2005
|
+
*/
|
|
2006
|
+
function at(row, index) {
|
|
2007
|
+
const value = row.values[index];
|
|
2008
|
+
if (value === void 0) throw new NativeParseError(FORMAT$10, row.line, `the row has no field ${String(index + 1)}`);
|
|
2009
|
+
return value;
|
|
2010
|
+
}
|
|
2011
|
+
/**
|
|
2012
|
+
* Parses an instance of the ORTEC VeRoLog Solver Challenge 2016 to 2017 (Kheiri et al. 2019): `KEY = value` header lines (the dataset and instance names, the planning days, the vehicle capacity, the longest trip, the depot's coordinate id and the three costs), then three sections each opened by `NAME = <count>` and followed by that many rows of whole numbers: `TOOLS` (id, size, how many are available, cost a day), `COORDINATES` (id, x, y) and `REQUESTS` (id, location, first day, last day, days the tools stay, tool kind, how many). Fields are separated by runs of spaces or tabs, and blank lines are skipped. The nodes are the depot, at its coordinate, with the id `depot`, then one node per request under the request's id at the position of its location. The tool table's sizes, availability and costs are not node fields and are not kept; its row count is.
|
|
2013
|
+
* @param text - The file text.
|
|
2014
|
+
* @returns The instance: its header values and tool count as `meta`, and its nodes, the depot first and the requests in file order.
|
|
2015
|
+
* @throws NativeParseError naming the line for an unknown or repeated key, a header value that is not a number where one is due, a missing header line or section, a section with a different number of rows than it says, a row that is not its section's count of whole numbers, a coordinate given twice, a request at a location the file does not give, or a request id given twice.
|
|
2016
|
+
*/
|
|
2017
|
+
function parseVerolog2017(text) {
|
|
2018
|
+
const meta = /* @__PURE__ */ new Map();
|
|
2019
|
+
const sections = /* @__PURE__ */ new Map();
|
|
2020
|
+
let open;
|
|
2021
|
+
const close = () => {
|
|
2022
|
+
if (open !== void 0 && open.rows.length !== open.count) throw new NativeParseError(FORMAT$10, open.line, `the ${open.name} section says ${String(open.count)} rows and has ${String(open.rows.length)}`);
|
|
2023
|
+
open = void 0;
|
|
2024
|
+
};
|
|
2025
|
+
for (const [index, raw] of lines(text).entries()) {
|
|
2026
|
+
const line = index + 1;
|
|
2027
|
+
const trimmed = raw.trim();
|
|
2028
|
+
if (trimmed === "") continue;
|
|
2029
|
+
const keyed = /^([A-Z_]+)\s*=\s*(.*)$/.exec(trimmed);
|
|
2030
|
+
if (keyed?.[1] !== void 0 && keyed[2] !== void 0) {
|
|
2031
|
+
close();
|
|
2032
|
+
const [, key, value] = keyed;
|
|
2033
|
+
const section = Object.keys(SECTION_WIDTHS).find((name) => name === key);
|
|
2034
|
+
if (section !== void 0) {
|
|
2035
|
+
if (sections.has(section) || !WHOLE$3.test(value)) throw new NativeParseError(FORMAT$10, line, sections.has(section) ? `the ${section} section is given twice` : `the ${section} section's count ${JSON.stringify(value)} is not a whole number`);
|
|
2036
|
+
const rows = [];
|
|
2037
|
+
sections.set(section, rows);
|
|
2038
|
+
open = {
|
|
2039
|
+
name: section,
|
|
2040
|
+
rows,
|
|
2041
|
+
count: Number(value),
|
|
2042
|
+
line
|
|
2043
|
+
};
|
|
2044
|
+
continue;
|
|
2045
|
+
}
|
|
2046
|
+
const header = HEADER$3.get(key);
|
|
2047
|
+
if (header === void 0 || meta.has(header[0])) throw new NativeParseError(FORMAT$10, line, header === void 0 ? `the key ${key} is not one this format has` : `the key ${key} is given twice`);
|
|
2048
|
+
const [name, kind] = header;
|
|
2049
|
+
if (kind === "number" && !/^-?\d+(?:\.\d+)?$/.test(value)) throw new NativeParseError(FORMAT$10, line, `${key} must be a number, found ${JSON.stringify(value)}`);
|
|
2050
|
+
meta.set(name, kind === "number" ? Number(value) : value);
|
|
2051
|
+
continue;
|
|
2052
|
+
}
|
|
2053
|
+
const tokens = trimmed.split(/\s+/);
|
|
2054
|
+
if (open === void 0 || tokens.length !== SECTION_WIDTHS[open.name] || !tokens.every((token) => WHOLE$3.test(token))) throw new NativeParseError(FORMAT$10, line, open === void 0 ? `the row ${JSON.stringify(trimmed)} is outside any section` : `a ${open.name} row is ${String(SECTION_WIDTHS[open.name])} whole numbers, found ${JSON.stringify(trimmed)}`);
|
|
2055
|
+
open.rows.push({
|
|
2056
|
+
line,
|
|
2057
|
+
values: tokens.map(Number)
|
|
2058
|
+
});
|
|
2059
|
+
}
|
|
2060
|
+
close();
|
|
2061
|
+
for (const [key, [name]] of HEADER$3) if (!meta.has(name)) throw new NativeParseError(FORMAT$10, 1, `the file has no ${key} line`);
|
|
2062
|
+
const rowsOf = (name) => {
|
|
2063
|
+
const rows = sections.get(name);
|
|
2064
|
+
if (rows === void 0) throw new NativeParseError(FORMAT$10, 1, `the file has no ${name} section`);
|
|
2065
|
+
return rows;
|
|
2066
|
+
};
|
|
2067
|
+
const tools = rowsOf("TOOLS").length;
|
|
2068
|
+
const positions = /* @__PURE__ */ new Map();
|
|
2069
|
+
for (const row of rowsOf("COORDINATES")) {
|
|
2070
|
+
const id = at(row, 0);
|
|
2071
|
+
if (positions.has(id)) throw new NativeParseError(FORMAT$10, row.line, `the coordinate ${String(id)} is given twice`);
|
|
2072
|
+
positions.set(id, [at(row, 1), at(row, 2)]);
|
|
2073
|
+
}
|
|
2074
|
+
const positionOf = (location, line) => {
|
|
2075
|
+
const position = positions.get(location);
|
|
2076
|
+
if (position === void 0) throw new NativeParseError(FORMAT$10, line, `the location ${String(location)} is not among the coordinates`);
|
|
2077
|
+
return position;
|
|
2078
|
+
};
|
|
2079
|
+
const depot = Number(meta.get("depotCoordinate"));
|
|
2080
|
+
const [depotX, depotY] = positionOf(depot, 1);
|
|
2081
|
+
const nodes = [{
|
|
2082
|
+
id: "depot",
|
|
2083
|
+
x: depotX,
|
|
2084
|
+
y: depotY,
|
|
2085
|
+
kind: "depot",
|
|
2086
|
+
location: depot,
|
|
2087
|
+
fromDay: null,
|
|
2088
|
+
toDay: null,
|
|
2089
|
+
stayDays: null,
|
|
2090
|
+
tool: null,
|
|
2091
|
+
amount: null
|
|
2092
|
+
}];
|
|
2093
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2094
|
+
for (const row of rowsOf("REQUESTS")) {
|
|
2095
|
+
const id = String(at(row, REQUEST_AT$1.id));
|
|
2096
|
+
if (ids.has(id)) throw new NativeParseError(FORMAT$10, row.line, `the request ${id} is given twice`);
|
|
2097
|
+
ids.add(id);
|
|
2098
|
+
const location = at(row, REQUEST_AT$1.location);
|
|
2099
|
+
const [x, y] = positionOf(location, row.line);
|
|
2100
|
+
nodes.push({
|
|
2101
|
+
id,
|
|
2102
|
+
x,
|
|
2103
|
+
y,
|
|
2104
|
+
kind: "request",
|
|
2105
|
+
location,
|
|
2106
|
+
fromDay: at(row, REQUEST_AT$1.fromDay),
|
|
2107
|
+
toDay: at(row, REQUEST_AT$1.toDay),
|
|
2108
|
+
stayDays: at(row, REQUEST_AT$1.stayDays),
|
|
2109
|
+
tool: String(at(row, REQUEST_AT$1.tool)),
|
|
2110
|
+
amount: at(row, REQUEST_AT$1.amount)
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
return {
|
|
2114
|
+
meta: {
|
|
2115
|
+
...Object.fromEntries(meta),
|
|
2116
|
+
tools
|
|
2117
|
+
},
|
|
2118
|
+
nodes
|
|
2119
|
+
};
|
|
2120
|
+
}
|
|
2121
|
+
//#endregion
|
|
1639
2122
|
//#region src/parsers/pajek.ts
|
|
1640
2123
|
/** Section keywords, lower-cased, mapped to how their lines are read. `*Network` names the network and starts no section. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
|
|
1641
2124
|
const sections = /* @__PURE__ */ new Map([
|
|
@@ -3161,6 +3644,48 @@ const KIBIBYTE = 1024;
|
|
|
3161
3644
|
/** Bytes decoded at a time for a format that is read in pieces. The decoder holds only a piece's text at once, so this bounds what a large table costs beyond its bytes; it is large enough that the per-piece work is small beside the piece. */
|
|
3162
3645
|
const DECODE_PIECE_BYTES = 8 * KIBIBYTE * KIBIBYTE;
|
|
3163
3646
|
/**
|
|
3647
|
+
* Decodes a file's bytes in the recipe's encoding as they arrive in pieces, as strictly as `decode`: a character whose bytes span two pieces is decoded whole with the later one, and bytes that are not valid in the stated encoding fail.
|
|
3648
|
+
*/
|
|
3649
|
+
var PieceDecoder = class {
|
|
3650
|
+
parser;
|
|
3651
|
+
utf8;
|
|
3652
|
+
constructor(parser) {
|
|
3653
|
+
this.parser = parser;
|
|
3654
|
+
this.utf8 = parser.encoding === "latin1" ? void 0 : new TextDecoder("utf-8", { fatal: true });
|
|
3655
|
+
}
|
|
3656
|
+
/**
|
|
3657
|
+
* Decodes the next piece.
|
|
3658
|
+
* @param piece - Bytes that follow the pieces decoded before them.
|
|
3659
|
+
* @returns Their text, without a character the piece ends partway through.
|
|
3660
|
+
* @throws NativeParseError when the bytes are not valid in the stated encoding.
|
|
3661
|
+
*/
|
|
3662
|
+
decode(piece) {
|
|
3663
|
+
const { utf8 } = this;
|
|
3664
|
+
return utf8 === void 0 ? decode(piece, this.parser) : this.strictly(() => utf8.decode(piece, { stream: true }));
|
|
3665
|
+
}
|
|
3666
|
+
/**
|
|
3667
|
+
* Ends the file.
|
|
3668
|
+
* @returns The text of any bytes still held.
|
|
3669
|
+
* @throws NativeParseError when the file ends partway through a character.
|
|
3670
|
+
*/
|
|
3671
|
+
end() {
|
|
3672
|
+
const { utf8 } = this;
|
|
3673
|
+
return utf8 === void 0 ? "" : this.strictly(() => utf8.decode());
|
|
3674
|
+
}
|
|
3675
|
+
/**
|
|
3676
|
+
* Runs a utf-8 decode, rethrowing its failure in the package's taxonomy: TextDecoder reports invalid bytes only as a TypeError with no position.
|
|
3677
|
+
* @param run - The decode.
|
|
3678
|
+
* @returns Its text.
|
|
3679
|
+
*/
|
|
3680
|
+
strictly(run) {
|
|
3681
|
+
try {
|
|
3682
|
+
return run();
|
|
3683
|
+
} catch (error) {
|
|
3684
|
+
throw new NativeParseError(this.parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
|
|
3685
|
+
}
|
|
3686
|
+
}
|
|
3687
|
+
};
|
|
3688
|
+
/**
|
|
3164
3689
|
* Decodes a file's bytes in the recipe's encoding a piece at a time, as strictly as `decode`, so a file whose text would not fit in one string can still be read.
|
|
3165
3690
|
* @param bytes - The unpacked file.
|
|
3166
3691
|
* @param parser - The recipe's parser, which names the encoding.
|
|
@@ -3169,20 +3694,10 @@ const DECODE_PIECE_BYTES = 8 * KIBIBYTE * KIBIBYTE;
|
|
|
3169
3694
|
* @throws NativeParseError when the bytes are not valid in the stated encoding.
|
|
3170
3695
|
*/
|
|
3171
3696
|
function* decodePieces(bytes, parser, pieceBytes = DECODE_PIECE_BYTES) {
|
|
3172
|
-
const decoder =
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
yield decoder === void 0 ? decode(piece, parser) : decoder.decode(piece, { stream: true });
|
|
3177
|
-
}
|
|
3178
|
-
if (decoder !== void 0) {
|
|
3179
|
-
const rest = decoder.decode();
|
|
3180
|
-
if (rest !== "") yield rest;
|
|
3181
|
-
}
|
|
3182
|
-
} catch (error) {
|
|
3183
|
-
if (error instanceof NativeParseError) throw error;
|
|
3184
|
-
throw new NativeParseError(parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
|
|
3185
|
-
}
|
|
3697
|
+
const decoder = new PieceDecoder(parser);
|
|
3698
|
+
for (let start = 0; start < bytes.length; start += pieceBytes) yield decoder.decode(bytes.subarray(start, start + pieceBytes));
|
|
3699
|
+
const rest = decoder.end();
|
|
3700
|
+
if (rest !== "") yield rest;
|
|
3186
3701
|
}
|
|
3187
3702
|
function parseNative(bytes, parser) {
|
|
3188
3703
|
if (parser.format === "csv-table") return parseCsvTable(decodePieces(bytes, parser), parser);
|
|
@@ -3193,7 +3708,7 @@ function parseNative(bytes, parser) {
|
|
|
3193
3708
|
split: void 0,
|
|
3194
3709
|
pieces: decodePieces(bytes, parser)
|
|
3195
3710
|
}], parser);
|
|
3196
|
-
if (parser.format === "
|
|
3711
|
+
if (parser.format === "tdc") return parseTdc(decodePieces(bytes, parser));
|
|
3197
3712
|
if (parser.format === "edge-list") return parseEdgeList(decodePieces(bytes, parser), parser);
|
|
3198
3713
|
const text = decode(bytes, parser);
|
|
3199
3714
|
if (parser.format === "geojson") return parseGeojson(text, parser);
|
|
@@ -3211,6 +3726,8 @@ function parseNative(bytes, parser) {
|
|
|
3211
3726
|
if (parser.format === "vrp-rep") return parseVrpRep(text);
|
|
3212
3727
|
if (parser.format === "hhcrsp") return parseHhcrsp(text);
|
|
3213
3728
|
if (parser.format === "ogb") throw new NativeParseError(parser.format, 1, "the format reads the several files a files archive selects, not one file");
|
|
3729
|
+
if (parser.format === "mankowska") return parseMankowska(text);
|
|
3730
|
+
if (parser.format === "verolog-2017") return parseVerolog2017(text);
|
|
3214
3731
|
return parsePajek(text);
|
|
3215
3732
|
}
|
|
3216
3733
|
/**
|
|
@@ -5029,18 +5546,20 @@ function unpackShapefile(bytes, member) {
|
|
|
5029
5546
|
};
|
|
5030
5547
|
}
|
|
5031
5548
|
/**
|
|
5032
|
-
* Unpacks downloaded bytes into every file a collection's selector matches, or, with an `xlsx` inner layout, into every sheet of every matched workbook that its `sheets` pattern matches as the sheet's CSV, at the member's path and the sheet's name joined by `/`, members in path order and sheets in the workbook's order.
|
|
5549
|
+
* Unpacks downloaded bytes into every file a collection's selector matches, or, with an `xlsx` inner layout, into every sheet of every matched workbook that its `sheets` pattern matches as the sheet's CSV, at the member's path and the sheet's name joined by `/`, members in path order and sheets in the workbook's order. The files are given one at a time: a zip member is inflated only when it is reached, so a reader that lets each file go before taking the next holds one member's bytes beside the archive, never every member at once.
|
|
5033
5550
|
* @param bytes - The downloaded bytes.
|
|
5034
5551
|
* @param archive - The recipe's archive layout, which selects members with `members`.
|
|
5035
5552
|
* @returns Each file.
|
|
5036
|
-
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind, the selector matches no member, or a member an `xlsx` layout reads is not a workbook or has no sheet the pattern matches.
|
|
5553
|
+
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind, the selector matches no member (at the first file taken), or a member an `xlsx` layout reads is not a workbook or has no sheet the pattern matches (at that member).
|
|
5037
5554
|
*/
|
|
5038
|
-
function unpackMembers(bytes, archive) {
|
|
5039
|
-
const
|
|
5040
|
-
if (archive.inner === void 0) return members;
|
|
5041
|
-
const { sheets } = archive.inner;
|
|
5555
|
+
function* unpackMembers(bytes, archive) {
|
|
5556
|
+
const { sheets } = archive.inner ?? {};
|
|
5042
5557
|
const wanted = sheets === void 0 ? void 0 : new RegExp(`^(?:${sheets})$`, "u");
|
|
5043
|
-
|
|
5558
|
+
for (const member of selectMembers(bytes, archive)) {
|
|
5559
|
+
if (archive.inner === void 0) {
|
|
5560
|
+
yield member;
|
|
5561
|
+
continue;
|
|
5562
|
+
}
|
|
5044
5563
|
let names;
|
|
5045
5564
|
try {
|
|
5046
5565
|
names = sheetNames(member.bytes).filter((sheet) => wanted === void 0 || wanted.test(sheet));
|
|
@@ -5048,24 +5567,24 @@ function unpackMembers(bytes, archive) {
|
|
|
5048
5567
|
throw new ArchiveError(`member ${member.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5049
5568
|
}
|
|
5050
5569
|
if (names.length === 0) throw new ArchiveError(`member ${member.path} has no sheet matching ${sheets ?? ""}`);
|
|
5051
|
-
|
|
5570
|
+
yield* names.map((sheet) => ({
|
|
5052
5571
|
path: `${member.path}/${sheet}`,
|
|
5053
5572
|
bytes: sheetToCsv(member.bytes, {
|
|
5054
5573
|
kind: "xlsx",
|
|
5055
5574
|
sheet
|
|
5056
5575
|
})
|
|
5057
5576
|
}));
|
|
5058
|
-
}
|
|
5577
|
+
}
|
|
5059
5578
|
}
|
|
5060
5579
|
/**
|
|
5061
|
-
* Unpacks every member a collection's selector matches.
|
|
5580
|
+
* Unpacks every member a collection's selector matches, one at a time.
|
|
5062
5581
|
* @param bytes - The downloaded bytes.
|
|
5063
5582
|
* @param archive - The recipe's archive layout.
|
|
5064
5583
|
* @returns Each matching member, in path order.
|
|
5065
|
-
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member.
|
|
5584
|
+
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member, at the first member taken.
|
|
5066
5585
|
*/
|
|
5067
|
-
function selectMembers(bytes, archive) {
|
|
5068
|
-
|
|
5586
|
+
function* selectMembers(bytes, archive) {
|
|
5587
|
+
yield* readMembers(bytes, archive.kind, (paths) => pickAll(paths, archive.members).map((path) => ({ path })));
|
|
5069
5588
|
}
|
|
5070
5589
|
/**
|
|
5071
5590
|
* Unpacks downloaded bytes into the files a parts archive names, each the one member its selector matches.
|
|
@@ -5075,57 +5594,45 @@ function selectMembers(bytes, archive) {
|
|
|
5075
5594
|
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or a part's selector does not match exactly one member.
|
|
5076
5595
|
*/
|
|
5077
5596
|
function unpackParts(bytes, archive) {
|
|
5078
|
-
return readMembers(bytes, archive.kind, (paths) => archive.parts.map((part) => ({
|
|
5597
|
+
return [...readMembers(bytes, archive.kind, (paths) => archive.parts.map((part) => ({
|
|
5079
5598
|
name: part.name,
|
|
5080
5599
|
path: pick(paths, part.member)
|
|
5081
|
-
})));
|
|
5600
|
+
})))];
|
|
5082
5601
|
}
|
|
5083
5602
|
/**
|
|
5084
|
-
* Reads the members of a file archive that a choice makes from its member paths,
|
|
5085
|
-
* @param bytes - The
|
|
5603
|
+
* Reads the members of a file archive that a choice makes from its member paths, one at a time: the archive is listed once and a zip member is inflated only when it is reached, so a reader that lets each member go before taking the next holds one member's bytes beside the archive.
|
|
5604
|
+
* @param bytes - The archive's bytes.
|
|
5086
5605
|
* @param kind - The archive kind.
|
|
5087
5606
|
* @param choose - Given every member path, the members to read, in the order wanted, each naming its path.
|
|
5088
5607
|
* @returns Each choice with the bytes of the member it names, in the order chosen.
|
|
5089
5608
|
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or a chosen member cannot be read; whatever `choose` throws.
|
|
5090
5609
|
*/
|
|
5091
|
-
function readMembers(bytes, kind, choose) {
|
|
5610
|
+
function* readMembers(bytes, kind, choose) {
|
|
5092
5611
|
if (kind === "zip") {
|
|
5093
5612
|
const names = [];
|
|
5094
5613
|
unzip(bytes, (file) => {
|
|
5095
5614
|
names.push(file.name);
|
|
5096
5615
|
return false;
|
|
5097
5616
|
});
|
|
5098
|
-
const
|
|
5099
|
-
|
|
5100
|
-
const files = unzip(bytes, (candidate) => chosen.has(candidate.name));
|
|
5101
|
-
return choices.map((choice) => {
|
|
5102
|
-
const file = getOwn(files, choice.path);
|
|
5617
|
+
for (const choice of choose(names)) {
|
|
5618
|
+
const file = getOwn(unzip(bytes, (candidate) => candidate.name === choice.path), choice.path);
|
|
5103
5619
|
if (file === void 0) throw new ArchiveError(`zip member ${choice.path} could not be read`);
|
|
5104
|
-
|
|
5620
|
+
yield {
|
|
5105
5621
|
...choice,
|
|
5106
5622
|
bytes: file
|
|
5107
5623
|
};
|
|
5108
|
-
}
|
|
5624
|
+
}
|
|
5625
|
+
return;
|
|
5109
5626
|
}
|
|
5110
5627
|
const members = readTar(untarred(bytes, kind));
|
|
5111
|
-
|
|
5628
|
+
for (const choice of choose(members.map((member) => member.path))) {
|
|
5112
5629
|
const found = members.find((member) => member.path === choice.path);
|
|
5113
5630
|
if (found === void 0) throw new ArchiveError(`tar member ${choice.path} could not be read`);
|
|
5114
|
-
|
|
5631
|
+
yield {
|
|
5115
5632
|
...choice,
|
|
5116
5633
|
bytes: found.data
|
|
5117
5634
|
};
|
|
5118
|
-
}
|
|
5119
|
-
}
|
|
5120
|
-
/**
|
|
5121
|
-
* Removes the compression a tar archive kind states, leaving the tar bytes.
|
|
5122
|
-
* @param bytes - The downloaded bytes.
|
|
5123
|
-
* @param kind - The tar archive kind: `tar` is uncompressed, `tar.gz` is gzip and `tar.bz2` is bzip2.
|
|
5124
|
-
* @returns The tar archive.
|
|
5125
|
-
*/
|
|
5126
|
-
function untarred(bytes, kind) {
|
|
5127
|
-
if (kind === "tar") return bytes;
|
|
5128
|
-
return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
|
|
5635
|
+
}
|
|
5129
5636
|
}
|
|
5130
5637
|
/**
|
|
5131
5638
|
* Reads one sheet of a spreadsheet as CSV text, by the spreadsheet's format.
|
|
@@ -5150,10 +5657,10 @@ function sheetToCsv(bytes, archive) {
|
|
|
5150
5657
|
* @throws ArchiveError when the bytes, or a file `inner` gunzips, are not well-formed, or the selector matches no member.
|
|
5151
5658
|
*/
|
|
5152
5659
|
function unpackFiles(bytes, archive) {
|
|
5153
|
-
const files = unpackMembers(bytes, {
|
|
5660
|
+
const files = [...unpackMembers(bytes, {
|
|
5154
5661
|
kind: archive.kind,
|
|
5155
5662
|
members: archive.files
|
|
5156
|
-
});
|
|
5663
|
+
})];
|
|
5157
5664
|
if (archive.inner === void 0) return files;
|
|
5158
5665
|
return files.map((file) => {
|
|
5159
5666
|
try {
|
|
@@ -5167,6 +5674,17 @@ function unpackFiles(bytes, archive) {
|
|
|
5167
5674
|
});
|
|
5168
5675
|
}
|
|
5169
5676
|
/**
|
|
5677
|
+
* Unpacks a tar archive's bytes, which may be compressed: `tar.gz` is gunzipped and `tar.bz2` bunzipped first, `tar` read as it is.
|
|
5678
|
+
* @param bytes - The archive's bytes.
|
|
5679
|
+
* @param kind - The archive kind.
|
|
5680
|
+
* @returns The tar's bytes.
|
|
5681
|
+
* @throws ArchiveError when the bytes are not valid gzip or bzip2 data, as `gunzip` and `bunzip2` report.
|
|
5682
|
+
*/
|
|
5683
|
+
function untarred(bytes, kind) {
|
|
5684
|
+
if (kind === "tar") return bytes;
|
|
5685
|
+
return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
|
|
5686
|
+
}
|
|
5687
|
+
/**
|
|
5170
5688
|
* Gunzips bytes, rethrowing zlib's failure in the package's taxonomy.
|
|
5171
5689
|
* @param bytes - Gzip data.
|
|
5172
5690
|
* @returns The decompressed bytes.
|
|
@@ -5432,15 +5950,25 @@ async function getBytes(port, url, signal) {
|
|
|
5432
5950
|
return new Uint8Array(await response.arrayBuffer());
|
|
5433
5951
|
}
|
|
5434
5952
|
/**
|
|
5435
|
-
* Reads a source's pages, each starting where the rows
|
|
5953
|
+
* Reads a source's pages in order, each starting where the rows read so far end, and checks each holds exactly the rows it should, so a response cut short is an error and not a table with rows missing. Only the page being read and the first page's header are held, so a caller that hands each page on holds no more of the source than that.
|
|
5436
5954
|
* @param read - The source read.
|
|
5437
|
-
* @
|
|
5438
|
-
* @
|
|
5439
|
-
* @returns Every page.
|
|
5440
|
-
* @throws KeepsetError when a page holds a different number of rows than the count says it should, or its header differs from the first page's.
|
|
5955
|
+
* @returns Each page, the first with its header record and every later one without it.
|
|
5956
|
+
* @throws KeepsetError when the source has no rows, a page holds a different number of rows than the count says it should, or its header differs from the first page's.
|
|
5441
5957
|
*/
|
|
5442
|
-
async function
|
|
5443
|
-
if (
|
|
5958
|
+
async function* resourcePages(read) {
|
|
5959
|
+
if (read.count === 0) throw new KeepsetError(`${read.resource} has no rows`);
|
|
5960
|
+
yield* pagesFrom(read, 0, void 0);
|
|
5961
|
+
}
|
|
5962
|
+
/**
|
|
5963
|
+
* Reads one page and then, by recursion, the pages after it.
|
|
5964
|
+
* @param read - The source read.
|
|
5965
|
+
* @param offset - The row the page starts at.
|
|
5966
|
+
* @param header - The first page's header record, or undefined when this is the first page.
|
|
5967
|
+
* @returns This page and every page after it.
|
|
5968
|
+
* @throws KeepsetError as `resourcePages` does.
|
|
5969
|
+
*/
|
|
5970
|
+
async function* pagesFrom(read, offset, header) {
|
|
5971
|
+
if (offset >= read.count) return;
|
|
5444
5972
|
const expected = Math.min(read.pageSize, read.count - offset);
|
|
5445
5973
|
const page = await withAttempts(async () => {
|
|
5446
5974
|
const bytes = await getBytes(read.port, read.pageUrl(offset), read.signal);
|
|
@@ -5448,10 +5976,15 @@ async function readResourcePages(read, held, offset) {
|
|
|
5448
5976
|
if (rows !== expected) throw new KeepsetError(`${read.resource} gave ${String(rows)} rows at offset ${String(offset)} where ${String(expected)} of its ${String(read.count)} were due, so a response was cut short or the table changed during the download`);
|
|
5449
5977
|
return bytes;
|
|
5450
5978
|
}, read.signal);
|
|
5451
|
-
const first = held[0];
|
|
5452
5979
|
const headerEnd = page.indexOf(LINE_FEED) + 1;
|
|
5453
|
-
if (
|
|
5454
|
-
|
|
5980
|
+
if (header === void 0) {
|
|
5981
|
+
yield page;
|
|
5982
|
+
yield* pagesFrom(read, offset + expected, page.slice(0, headerEnd));
|
|
5983
|
+
return;
|
|
5984
|
+
}
|
|
5985
|
+
if (headerEnd !== header.length || !page.subarray(0, headerEnd).every((byte, index) => byte === header[index])) throw new KeepsetError(`${read.resource} gave a different header at offset ${String(offset)} than at the start`);
|
|
5986
|
+
yield page.subarray(headerEnd);
|
|
5987
|
+
yield* pagesFrom(read, offset + expected, header);
|
|
5455
5988
|
}
|
|
5456
5989
|
/**
|
|
5457
5990
|
* Downloads a CSV source a page at a time and joins the pages under the first header.
|
|
@@ -5460,42 +5993,44 @@ async function readResourcePages(read, held, offset) {
|
|
|
5460
5993
|
* @throws KeepsetError when the source has no rows or a page is not what it should be.
|
|
5461
5994
|
*/
|
|
5462
5995
|
async function downloadPages(read) {
|
|
5463
|
-
const pages =
|
|
5464
|
-
|
|
5996
|
+
const pages = [];
|
|
5997
|
+
for await (const page of resourcePages(read)) pages.push(page);
|
|
5465
5998
|
return Buffer.concat(pages);
|
|
5466
5999
|
}
|
|
5467
6000
|
/**
|
|
5468
|
-
*
|
|
6001
|
+
* How a Socrata resource is read: its row count is read first, then pages of `pageSize` rows in the order of the rows' ids, with only the `select` columns when the recipe lists them, so the columns a parser drops are never transferred.
|
|
5469
6002
|
* @param resource - The resource's URL, such as `https://data.example.org/resource/abcd-1234.csv`, without a query.
|
|
5470
|
-
* @param
|
|
6003
|
+
* @param paging - The recipe's paging.
|
|
5471
6004
|
* @param port - The network port.
|
|
5472
6005
|
* @param signal - Aborts the download.
|
|
5473
|
-
* @returns The
|
|
5474
|
-
* @throws KeepsetError when the resource does not report a count
|
|
6006
|
+
* @returns The source read.
|
|
6007
|
+
* @throws KeepsetError when the resource does not report a count.
|
|
5475
6008
|
*/
|
|
5476
|
-
async function
|
|
6009
|
+
async function socrataRead(resource, paging, port, signal) {
|
|
5477
6010
|
const reported = /^"count"\r?\n"(\d+)"/.exec(new TextDecoder("utf-8").decode(await getBytes(port, `${resource}?%24select=count(*)`, signal)));
|
|
5478
6011
|
if (reported?.[1] === void 0) throw new KeepsetError(`${resource} did not report a row count`);
|
|
5479
|
-
|
|
6012
|
+
const { pageSize, select } = paging;
|
|
6013
|
+
const columns = select === void 0 ? "" : `%24select=${encodeURIComponent(select.join(","))}&`;
|
|
6014
|
+
return {
|
|
5480
6015
|
resource,
|
|
5481
|
-
pageUrl: (offset) => `${resource}
|
|
6016
|
+
pageUrl: (offset) => `${resource}?${columns}%24order=%3Aid&%24limit=${String(pageSize)}&%24offset=${String(offset)}`,
|
|
5482
6017
|
pageSize,
|
|
5483
6018
|
count: Number(reported[1]),
|
|
5484
6019
|
port,
|
|
5485
6020
|
signal
|
|
5486
|
-
}
|
|
6021
|
+
};
|
|
5487
6022
|
}
|
|
5488
6023
|
const CountResultSchema = z.object({ result: z.object({ total: z.int().min(0) }) });
|
|
5489
6024
|
/**
|
|
5490
|
-
*
|
|
6025
|
+
* How a CKAN datastore table is read: the row count is read from the portal's datastore API, then pages of `pageSize` rows of the table's CSV dump.
|
|
5491
6026
|
* @param dump - The dump's URL, such as `https://data.example.org/datastore/dump/<resource id>`, without a query; a portal that serves CKAN under a path prefix keeps it before `/datastore/dump/`.
|
|
5492
6027
|
* @param pageSize - How many rows a page holds.
|
|
5493
6028
|
* @param port - The network port.
|
|
5494
6029
|
* @param signal - Aborts the download.
|
|
5495
|
-
* @returns The
|
|
5496
|
-
* @throws KeepsetError when the URL is not a datastore dump
|
|
6030
|
+
* @returns The source read.
|
|
6031
|
+
* @throws KeepsetError when the URL is not a datastore dump or the table does not report a count.
|
|
5497
6032
|
*/
|
|
5498
|
-
async function
|
|
6033
|
+
async function ckanRead(dump, pageSize, port, signal) {
|
|
5499
6034
|
const parsed = /^(.*)\/datastore\/dump\/([^/?#]+)$/.exec(dump);
|
|
5500
6035
|
if (parsed?.[1] === void 0 || parsed[2] === void 0) throw new KeepsetError(`${dump} is not a CKAN datastore dump URL`);
|
|
5501
6036
|
const url = `${parsed[1]}/api/3/action/datastore_search?resource_id=${parsed[2]}&limit=0`;
|
|
@@ -5507,26 +6042,26 @@ async function downloadCkanDatastore(dump, pageSize, port, signal) {
|
|
|
5507
6042
|
}
|
|
5508
6043
|
const count = CountResultSchema.safeParse(body);
|
|
5509
6044
|
if (!count.success) throw new KeepsetError(`${dump} did not report a row count`);
|
|
5510
|
-
return
|
|
6045
|
+
return {
|
|
5511
6046
|
resource: dump,
|
|
5512
6047
|
pageUrl: (offset) => `${dump}?format=csv&limit=${String(pageSize)}&offset=${String(offset)}`,
|
|
5513
6048
|
pageSize,
|
|
5514
6049
|
count: count.data.result.total,
|
|
5515
6050
|
port,
|
|
5516
6051
|
signal
|
|
5517
|
-
}
|
|
6052
|
+
};
|
|
5518
6053
|
}
|
|
5519
6054
|
const OpenFemaCountSchema = z.object({ metadata: z.object({ count: z.int().min(0) }) });
|
|
5520
6055
|
/**
|
|
5521
|
-
*
|
|
6056
|
+
* How an OpenFEMA entity, or the rows of it the source URL selects, is read: the row count is read from the API's own metadata, then pages of `pageSize` rows in the order of the entity's id.
|
|
5522
6057
|
* @param entity - The entity's URL with any `$select` or `$filter` but no `$top`, `$skip`, `$orderby` or `$format`, such as `https://www.fema.gov/api/open/v3/NfipClaims?$select=id,state`.
|
|
5523
6058
|
* @param pageSize - How many rows a page holds.
|
|
5524
6059
|
* @param port - The network port.
|
|
5525
6060
|
* @param signal - Aborts the download.
|
|
5526
|
-
* @returns The
|
|
5527
|
-
* @throws KeepsetError when the source URL sets a paging parameter
|
|
6061
|
+
* @returns The source read.
|
|
6062
|
+
* @throws KeepsetError when the source URL sets a paging parameter or the entity does not report a count.
|
|
5528
6063
|
*/
|
|
5529
|
-
async function
|
|
6064
|
+
async function openFemaRead(entity, pageSize, port, signal) {
|
|
5530
6065
|
const base = new URL(entity);
|
|
5531
6066
|
for (const name of [
|
|
5532
6067
|
"$top",
|
|
@@ -5552,7 +6087,7 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
|
|
|
5552
6087
|
}
|
|
5553
6088
|
const count = OpenFemaCountSchema.safeParse(body);
|
|
5554
6089
|
if (!count.success) throw new KeepsetError(`${entity} did not report a row count`);
|
|
5555
|
-
return
|
|
6090
|
+
return {
|
|
5556
6091
|
resource: entity,
|
|
5557
6092
|
pageUrl: (offset) => withParameters({
|
|
5558
6093
|
$format: "csv",
|
|
@@ -5564,10 +6099,34 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
|
|
|
5564
6099
|
count: count.data.metadata.count,
|
|
5565
6100
|
port,
|
|
5566
6101
|
signal
|
|
5567
|
-
}
|
|
6102
|
+
};
|
|
5568
6103
|
}
|
|
5569
6104
|
/**
|
|
5570
|
-
*
|
|
6105
|
+
* How a CSV paged source is read, in the way its recipe's paging says.
|
|
6106
|
+
* @param url - The source's URL.
|
|
6107
|
+
* @param paging - How the source is paged.
|
|
6108
|
+
* @param port - The network port.
|
|
6109
|
+
* @param signal - Aborts the download.
|
|
6110
|
+
* @returns The source read.
|
|
6111
|
+
*/
|
|
6112
|
+
async function csvRead(url, paging, port, signal) {
|
|
6113
|
+
if (paging.kind === "socrata-resource") return socrataRead(url, paging, port, signal);
|
|
6114
|
+
return paging.kind === "ckan-datastore" ? ckanRead(url, paging.pageSize, port, signal) : openFemaRead(url, paging.pageSize, port, signal);
|
|
6115
|
+
}
|
|
6116
|
+
/**
|
|
6117
|
+
* Reads a CSV paged source a page at a time: the first page with its header and every later page without it, so the pages in order are the one CSV `downloadPaged` joins them into. Only the page being read is held.
|
|
6118
|
+
* @param url - The source's URL.
|
|
6119
|
+
* @param paging - How the source is paged.
|
|
6120
|
+
* @param port - The network port.
|
|
6121
|
+
* @param signal - Aborts the download.
|
|
6122
|
+
* @returns Each page's bytes, in order.
|
|
6123
|
+
* @throws KeepsetError when the source does not report a count, has no rows, or a page is not what it should be.
|
|
6124
|
+
*/
|
|
6125
|
+
async function* csvPages(url, paging, port, signal) {
|
|
6126
|
+
yield* resourcePages(await csvRead(url, paging, port, signal));
|
|
6127
|
+
}
|
|
6128
|
+
/**
|
|
6129
|
+
* Downloads a paged source as the one file its pages make, in the way its recipe's paging says: an ArcGIS layer as one GeoJSON feature collection, any other source as one CSV of its pages joined under the first page's header.
|
|
5571
6130
|
* @param url - The source's URL.
|
|
5572
6131
|
* @param paging - How the source is paged.
|
|
5573
6132
|
* @param port - The network port.
|
|
@@ -5577,24 +6136,35 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
|
|
|
5577
6136
|
*/
|
|
5578
6137
|
async function downloadPaged(url, paging, port, signal) {
|
|
5579
6138
|
if (paging.kind === "arcgis-query") return downloadArcgisLayer(url, paging.orderBy, port, signal);
|
|
5580
|
-
|
|
5581
|
-
return paging.kind === "ckan-datastore" ? downloadCkanDatastore(url, paging.pageSize, port, signal) : downloadOpenFema(url, paging.pageSize, port, signal);
|
|
6139
|
+
return downloadPages(await csvRead(url, paging, port, signal));
|
|
5582
6140
|
}
|
|
5583
6141
|
//#endregion
|
|
5584
|
-
//#region src/node/
|
|
6142
|
+
//#region src/node/recipe-run.ts
|
|
5585
6143
|
/**
|
|
5586
|
-
* The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
|
|
6144
|
+
* The URL a recipe downloads a source from, as `downloadUrl` gives it. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
|
|
5587
6145
|
* @param source - The source.
|
|
5588
6146
|
* @returns The URL to request.
|
|
5589
6147
|
*/
|
|
5590
|
-
function
|
|
6148
|
+
function sourceUrl(source) {
|
|
5591
6149
|
if (source.status === "dead") {
|
|
5592
6150
|
if (source.snapshot === void 0) throw new KeepsetError(`dead source ${source.url} has no snapshot`);
|
|
5593
|
-
return
|
|
6151
|
+
return rawSnapshotUrl(source.snapshot.timestamp, source.url);
|
|
5594
6152
|
}
|
|
5595
6153
|
return source.url;
|
|
5596
6154
|
}
|
|
5597
6155
|
/**
|
|
6156
|
+
* The source record a provenance carries: the entry's source as recorded.
|
|
6157
|
+
* @param source - The source.
|
|
6158
|
+
* @returns Its URL, status and any snapshot.
|
|
6159
|
+
*/
|
|
6160
|
+
function sourceRecord(source) {
|
|
6161
|
+
return {
|
|
6162
|
+
url: source.url,
|
|
6163
|
+
status: source.status,
|
|
6164
|
+
...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
|
|
6165
|
+
};
|
|
6166
|
+
}
|
|
6167
|
+
/**
|
|
5598
6168
|
* Runs one step, rethrowing any failure as a RecipeError naming the entry and the step, with the original error as its cause.
|
|
5599
6169
|
* @param entryId - The entry.
|
|
5600
6170
|
* @param name - The step.
|
|
@@ -5621,6 +6191,373 @@ function metaOf(entry) {
|
|
|
5621
6191
|
...entry.properties
|
|
5622
6192
|
};
|
|
5623
6193
|
}
|
|
6194
|
+
//#endregion
|
|
6195
|
+
//#region src/node/id-set.ts
|
|
6196
|
+
/** The first table size, in slots: 2 to the 16th, a power of two, so a hash is reduced to a slot by masking (and every doubling keeps it one). */
|
|
6197
|
+
const INITIAL_SLOTS = 65536;
|
|
6198
|
+
/** The first arena size, in bytes: a mebibyte, which a stream of a few tens of thousands of short ids fills before the arena first doubles. */
|
|
6199
|
+
const INITIAL_ARENA_BYTES = 1048576;
|
|
6200
|
+
/** The table grows when more than this share of its slots hold an id, which keeps linear probing short. */
|
|
6201
|
+
const MAX_LOAD = .7;
|
|
6202
|
+
/** The bytes before each id in the arena: its length, as an unsigned 32-bit integer. */
|
|
6203
|
+
const LENGTH_BYTES = 4;
|
|
6204
|
+
/** The largest arena, 2 to the 31st less one: slots hold an arena offset plus one in a signed 32-bit integer. */
|
|
6205
|
+
const MAX_ARENA_BYTES = 2147483647;
|
|
6206
|
+
/** The FNV-1a 32-bit offset basis and prime (http://www.isthe.com/chongo/tech/comp/fnv/). */
|
|
6207
|
+
const FNV_OFFSET = 2166136261;
|
|
6208
|
+
const FNV_PRIME = 16777619;
|
|
6209
|
+
const encoder = new TextEncoder();
|
|
6210
|
+
/**
|
|
6211
|
+
* The FNV-1a hash of bytes.
|
|
6212
|
+
* @param bytes - The bytes.
|
|
6213
|
+
* @returns The 32-bit hash.
|
|
6214
|
+
*/
|
|
6215
|
+
function fnv1a(bytes) {
|
|
6216
|
+
let hash = FNV_OFFSET;
|
|
6217
|
+
for (const byte of bytes) hash = Math.imul(hash ^ byte, FNV_PRIME);
|
|
6218
|
+
return hash >>> 0;
|
|
6219
|
+
}
|
|
6220
|
+
/**
|
|
6221
|
+
* Reads a slot of a table.
|
|
6222
|
+
* @param slots - The table.
|
|
6223
|
+
* @param slot - A slot index, always masked into the table's range.
|
|
6224
|
+
* @returns What the slot holds.
|
|
6225
|
+
* @throws RangeError when the index is outside the table, which masking never gives.
|
|
6226
|
+
*/
|
|
6227
|
+
function slotOf(slots, slot) {
|
|
6228
|
+
const held = slots[slot];
|
|
6229
|
+
if (held === void 0) throw new RangeError(`slot ${String(slot)} is outside a table of ${String(slots.length)}`);
|
|
6230
|
+
return held;
|
|
6231
|
+
}
|
|
6232
|
+
/**
|
|
6233
|
+
* A set of ids that holds each id as its UTF-8 bytes in one growing arena, indexed by an open-addressed table of arena offsets, so a stream of millions of ids is checked for repeats exactly while holding a few bytes beyond each id's own and no JavaScript string or object per id. Membership compares the bytes themselves, so two different ids are never taken for one.
|
|
6234
|
+
*/
|
|
6235
|
+
var IdSet = class {
|
|
6236
|
+
arena = new Uint8Array(INITIAL_ARENA_BYTES);
|
|
6237
|
+
view = new DataView(this.arena.buffer);
|
|
6238
|
+
used = 0;
|
|
6239
|
+
/** Each slot holds an id's arena offset plus one, or zero when empty. */
|
|
6240
|
+
slots = new Int32Array(INITIAL_SLOTS);
|
|
6241
|
+
count = 0;
|
|
6242
|
+
/** @returns How many ids the set holds. */
|
|
6243
|
+
get size() {
|
|
6244
|
+
return this.count;
|
|
6245
|
+
}
|
|
6246
|
+
/**
|
|
6247
|
+
* Adds an id.
|
|
6248
|
+
* @param id - The id.
|
|
6249
|
+
* @returns True when the id was new, false when the set already held it.
|
|
6250
|
+
* @throws KeepsetError when the ids held would pass the largest arena.
|
|
6251
|
+
*/
|
|
6252
|
+
add(id) {
|
|
6253
|
+
const bytes = encoder.encode(id);
|
|
6254
|
+
const hash = fnv1a(bytes);
|
|
6255
|
+
const mask = this.slots.length - 1;
|
|
6256
|
+
for (let slot = hash & mask;; slot = slot + 1 & mask) {
|
|
6257
|
+
const held = slotOf(this.slots, slot);
|
|
6258
|
+
if (held === 0) {
|
|
6259
|
+
this.slots[slot] = this.store(bytes) + 1;
|
|
6260
|
+
this.count += 1;
|
|
6261
|
+
if (this.count > this.slots.length * MAX_LOAD) this.grow();
|
|
6262
|
+
return true;
|
|
6263
|
+
}
|
|
6264
|
+
if (this.equals(held - 1, bytes)) return false;
|
|
6265
|
+
}
|
|
6266
|
+
}
|
|
6267
|
+
/**
|
|
6268
|
+
* Whether the id stored at an offset is these bytes.
|
|
6269
|
+
* @param offset - The id's arena offset.
|
|
6270
|
+
* @param bytes - The bytes.
|
|
6271
|
+
* @returns True when they are the same.
|
|
6272
|
+
*/
|
|
6273
|
+
equals(offset, bytes) {
|
|
6274
|
+
if (this.lengthAt(offset) !== bytes.length) return false;
|
|
6275
|
+
const start = offset + LENGTH_BYTES;
|
|
6276
|
+
return bytes.every((byte, index) => this.arena[start + index] === byte);
|
|
6277
|
+
}
|
|
6278
|
+
/**
|
|
6279
|
+
* The length of the id stored at an offset.
|
|
6280
|
+
* @param offset - The id's arena offset.
|
|
6281
|
+
* @returns Its byte length.
|
|
6282
|
+
*/
|
|
6283
|
+
lengthAt(offset) {
|
|
6284
|
+
return this.view.getUint32(offset, true);
|
|
6285
|
+
}
|
|
6286
|
+
/**
|
|
6287
|
+
* Appends an id's bytes to the arena.
|
|
6288
|
+
* @param bytes - The bytes.
|
|
6289
|
+
* @returns Their offset.
|
|
6290
|
+
*/
|
|
6291
|
+
store(bytes) {
|
|
6292
|
+
const needed = this.used + LENGTH_BYTES + bytes.length;
|
|
6293
|
+
if (needed > MAX_ARENA_BYTES) throw new KeepsetError(`the ids of the stream pass ${String(MAX_ARENA_BYTES)} bytes, more than one id set holds`);
|
|
6294
|
+
if (needed > this.arena.length) {
|
|
6295
|
+
const grown = new Uint8Array(Math.min(MAX_ARENA_BYTES, Math.max(needed, this.arena.length * 2)));
|
|
6296
|
+
grown.set(this.arena.subarray(0, this.used));
|
|
6297
|
+
this.arena = grown;
|
|
6298
|
+
this.view = new DataView(grown.buffer);
|
|
6299
|
+
}
|
|
6300
|
+
const offset = this.used;
|
|
6301
|
+
this.view.setUint32(offset, bytes.length, true);
|
|
6302
|
+
this.arena.set(bytes, offset + LENGTH_BYTES);
|
|
6303
|
+
this.used = needed;
|
|
6304
|
+
return offset;
|
|
6305
|
+
}
|
|
6306
|
+
/** Doubles the table, placing every id again by its hash, which is recomputed from the arena so the table holds nothing but offsets. */
|
|
6307
|
+
grow() {
|
|
6308
|
+
const slots = new Int32Array(this.slots.length * 2);
|
|
6309
|
+
const mask = slots.length - 1;
|
|
6310
|
+
for (const held of this.slots) if (held !== 0) {
|
|
6311
|
+
const start = held - 1 + LENGTH_BYTES;
|
|
6312
|
+
let slot = fnv1a(this.arena.subarray(start, start + this.lengthAt(held - 1))) & mask;
|
|
6313
|
+
while (slotOf(slots, slot) !== 0) slot = slot + 1 & mask;
|
|
6314
|
+
slots[slot] = held;
|
|
6315
|
+
}
|
|
6316
|
+
this.slots = slots;
|
|
6317
|
+
}
|
|
6318
|
+
};
|
|
6319
|
+
//#endregion
|
|
6320
|
+
//#region src/node/stream-dataset.ts
|
|
6321
|
+
/**
|
|
6322
|
+
* The parts of a recipe a stream reads, checked to be a streamed recipe: the entry schema refuses a `stream` recipe that is not a CSV paged source read by the csv-table parser with no archive, so this only fails for an entry whose recipe does not stream.
|
|
6323
|
+
* @param entry - The entry.
|
|
6324
|
+
* @returns The recipe's source, paging and parser, and the category.
|
|
6325
|
+
* @throws RecipeError when the recipe is not fetchable, does not stream, or its category has no streamed form.
|
|
6326
|
+
*/
|
|
6327
|
+
function streamedRecipe(entry) {
|
|
6328
|
+
const { recipe, category } = entry;
|
|
6329
|
+
if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
|
|
6330
|
+
if ("files" in recipe || recipe.stream !== true) throw new RecipeError(entry.id, "recipe", "the recipe is not streamed, so fetchDataset reads it as one document");
|
|
6331
|
+
const { paging, parser } = recipe;
|
|
6332
|
+
if (paging === void 0 || paging.kind === "arcgis-query" || parser.format !== "csv-table" || !isStreamable(category)) throw new RecipeError(entry.id, "recipe", "a streamed recipe reads a CSV paged source through the csv-table parser, for a category with a streamed form");
|
|
6333
|
+
return {
|
|
6334
|
+
source: recipe.source,
|
|
6335
|
+
paging,
|
|
6336
|
+
parser,
|
|
6337
|
+
category
|
|
6338
|
+
};
|
|
6339
|
+
}
|
|
6340
|
+
/**
|
|
6341
|
+
* Whether a recipe is streamed, so `fetchDataset` refuses it and `streamDataset` reads it.
|
|
6342
|
+
* @param recipe - A recipe.
|
|
6343
|
+
* @returns True for a streamed recipe.
|
|
6344
|
+
*/
|
|
6345
|
+
function isStreamed(recipe) {
|
|
6346
|
+
return recipe.status === "fetchable" && !("files" in recipe) && recipe.stream === true;
|
|
6347
|
+
}
|
|
6348
|
+
/**
|
|
6349
|
+
* What a crosswalk path addresses, for the separability check: the document's meta, a field of every node, or anything else.
|
|
6350
|
+
* @param path - A crosswalk path.
|
|
6351
|
+
* @returns Its class.
|
|
6352
|
+
*/
|
|
6353
|
+
function pathClass(path) {
|
|
6354
|
+
if (path === "meta" || path.startsWith("meta.")) return "meta";
|
|
6355
|
+
return path === "nodes[]" || path.startsWith("nodes[].") ? "node" : "other";
|
|
6356
|
+
}
|
|
6357
|
+
/**
|
|
6358
|
+
* Why a crosswalk cannot be applied a page of nodes at a time, if it cannot: every rule must read and write the meta alone or one node's fields alone, with no `when`, so encoding a page's nodes under the document's meta gives exactly that page's part of encoding the whole document, and a round trip that holds on every page holds on the whole.
|
|
6359
|
+
* @param crosswalk - The crosswalk.
|
|
6360
|
+
* @returns The first rule that breaks this, described, or undefined when every rule keeps to it.
|
|
6361
|
+
*/
|
|
6362
|
+
function inseparableRule(crosswalk) {
|
|
6363
|
+
for (const [index, rule] of [...crosswalk.rules, ...crosswalk.oneWay].entries()) {
|
|
6364
|
+
if ("when" in rule && rule.when !== void 0) return `rule ${String(index)} (${rule.op}) has a when, which may read the whole document`;
|
|
6365
|
+
const paths = ["from" in rule ? rule.from : void 0, "to" in rule ? rule.to : void 0].filter((path) => path !== void 0);
|
|
6366
|
+
const classes = new Set(paths.map(pathClass));
|
|
6367
|
+
const [only] = classes;
|
|
6368
|
+
if (classes.size !== 1 || only === "other") return `rule ${String(index)} (${rule.op}) reads or writes ${paths.join(" and ")}, not the meta alone or one node's fields alone`;
|
|
6369
|
+
}
|
|
6370
|
+
}
|
|
6371
|
+
/**
|
|
6372
|
+
* Passes on what an async iterable yields, rethrowing its failure as a RecipeError naming the entry and the step, as `step` does for one call.
|
|
6373
|
+
* @param entryId - The entry.
|
|
6374
|
+
* @param name - The step.
|
|
6375
|
+
* @param source - What the step yields.
|
|
6376
|
+
* @returns The same items, in order.
|
|
6377
|
+
*/
|
|
6378
|
+
async function* inStep(entryId, name, source) {
|
|
6379
|
+
try {
|
|
6380
|
+
yield* source;
|
|
6381
|
+
} catch (error) {
|
|
6382
|
+
if (error instanceof RecipeError) throw error;
|
|
6383
|
+
throw new RecipeError(entryId, name, error instanceof Error ? error.message : String(error), { cause: error });
|
|
6384
|
+
}
|
|
6385
|
+
}
|
|
6386
|
+
/**
|
|
6387
|
+
* Runs a streamed recipe's download and parse steps a page at a time: each page's bytes are hashed, decoded and read into rows, and then let go.
|
|
6388
|
+
* @param dataset - The dataset.
|
|
6389
|
+
* @param recipe - Its streamed recipe.
|
|
6390
|
+
* @param options - The network port, abort signal and clock.
|
|
6391
|
+
* @returns The stream.
|
|
6392
|
+
*/
|
|
6393
|
+
function openNative(dataset, recipe, options) {
|
|
6394
|
+
const { entry } = dataset;
|
|
6395
|
+
const { fetch: port = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
|
|
6396
|
+
const source = entry.sources[recipe.source];
|
|
6397
|
+
if (source === void 0) throw new RecipeError(entry.id, "recipe", `the recipe names source ${String(recipe.source)}, which the entry does not have`);
|
|
6398
|
+
const fetchedUrl = sourceUrl(source);
|
|
6399
|
+
const retrievedAt = now().toISOString();
|
|
6400
|
+
const hash = createHash("sha256");
|
|
6401
|
+
let byteLength = 0;
|
|
6402
|
+
async function* pages() {
|
|
6403
|
+
const decoder = new PieceDecoder(recipe.parser);
|
|
6404
|
+
const reader = new TableReader(recipe.parser);
|
|
6405
|
+
for await (const page of inStep(entry.id, "download", csvPages(fetchedUrl, recipe.paging, port, signal))) {
|
|
6406
|
+
hash.update(page);
|
|
6407
|
+
byteLength += page.byteLength;
|
|
6408
|
+
const rows = await step(entry.id, "parse", () => reader.push(decoder.decode(page)));
|
|
6409
|
+
if (rows.length > 0) yield rows;
|
|
6410
|
+
}
|
|
6411
|
+
const rows = await step(entry.id, "parse", () => [...reader.push(decoder.end()), ...reader.end()]);
|
|
6412
|
+
if (rows.length > 0) yield rows;
|
|
6413
|
+
}
|
|
6414
|
+
return {
|
|
6415
|
+
meta: metaOf(entry),
|
|
6416
|
+
pages: pages(),
|
|
6417
|
+
provenance: () => ({
|
|
6418
|
+
entryId: entry.id,
|
|
6419
|
+
catalogue: dataset.catalogue,
|
|
6420
|
+
category: entry.category,
|
|
6421
|
+
source: sourceRecord(source),
|
|
6422
|
+
fetchedUrl,
|
|
6423
|
+
retrievedAt,
|
|
6424
|
+
sha256: hash.copy().digest("hex"),
|
|
6425
|
+
byteLength
|
|
6426
|
+
})
|
|
6427
|
+
};
|
|
6428
|
+
}
|
|
6429
|
+
/**
|
|
6430
|
+
* The nodes of an encoded page: the crosswalk writes only the meta and the nodes, so the result is an object with a `nodes` array.
|
|
6431
|
+
* @param encoded - The crosswalk's output for a page.
|
|
6432
|
+
* @returns Its nodes.
|
|
6433
|
+
* @throws KeepsetError when the output is not an object with a nodes array and nothing beyond meta.
|
|
6434
|
+
*/
|
|
6435
|
+
function nodesOf(encoded) {
|
|
6436
|
+
if (typeof encoded !== "object" || encoded === null || Array.isArray(encoded) || !("nodes" in encoded) || !Array.isArray(encoded.nodes)) throw new KeepsetError("the crosswalk's output is not an object with a nodes array");
|
|
6437
|
+
const extra = Object.keys(encoded).filter((key) => key !== "meta" && key !== "nodes");
|
|
6438
|
+
if (extra.length > 0) throw new KeepsetError(`the crosswalk's output has ${extra.join(", ")} beside meta and nodes`);
|
|
6439
|
+
return encoded.nodes;
|
|
6440
|
+
}
|
|
6441
|
+
/**
|
|
6442
|
+
* Checks a crosswalk can run a page at a time, or fails the crosswalk step.
|
|
6443
|
+
* @param entry - The entry.
|
|
6444
|
+
* @param crosswalk - Its crosswalk.
|
|
6445
|
+
* @throws RecipeError when a rule reaches beyond the meta or one node's fields.
|
|
6446
|
+
*/
|
|
6447
|
+
function assertSeparable(entry, crosswalk) {
|
|
6448
|
+
const reason = inseparableRule(crosswalk);
|
|
6449
|
+
if (reason !== void 0) throw new RecipeError(entry.id, "crosswalk", `a streamed crosswalk must apply to a page of nodes on its own: ${reason}`);
|
|
6450
|
+
}
|
|
6451
|
+
/**
|
|
6452
|
+
* Streams a table, as `streamEntry` describes.
|
|
6453
|
+
* @param dataset - A dataset whose recipe streams.
|
|
6454
|
+
* @param sink - Where the meta and the nodes go.
|
|
6455
|
+
* @param options - The network port, abort signal and clock.
|
|
6456
|
+
* @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
|
|
6457
|
+
* @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
|
|
6458
|
+
*/
|
|
6459
|
+
async function streamTable(dataset, sink, options) {
|
|
6460
|
+
const { entry, crosswalk } = dataset;
|
|
6461
|
+
const recipe = streamedRecipe(entry);
|
|
6462
|
+
assertSeparable(entry, crosswalk);
|
|
6463
|
+
const shape = getStreamedShape(recipe.category);
|
|
6464
|
+
const native = openNative(dataset, recipe, options);
|
|
6465
|
+
const meta = await step(entry.id, "validate", async () => {
|
|
6466
|
+
const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, {
|
|
6467
|
+
meta: native.meta,
|
|
6468
|
+
nodes: []
|
|
6469
|
+
}));
|
|
6470
|
+
const parsed = shape.meta.safeParse(typeof encoded === "object" && encoded !== null && !Array.isArray(encoded) && "meta" in encoded ? encoded.meta : void 0);
|
|
6471
|
+
if (!parsed.success) throw new KeepsetError(`the result's meta is not ${entry.category} meta: ${parsed.error.issues.map((issue) => `${["meta", ...issue.path.map(String)].join(".")}: ${issue.message}`).join("; ")}`);
|
|
6472
|
+
return parsed.data;
|
|
6473
|
+
});
|
|
6474
|
+
await sink.meta(meta);
|
|
6475
|
+
const ids = new IdSet();
|
|
6476
|
+
for await (const rows of native.pages) {
|
|
6477
|
+
const offset = ids.size;
|
|
6478
|
+
const encoded = await step(entry.id, "crosswalk", () => nodesOf(encode(crosswalk, {
|
|
6479
|
+
meta: native.meta,
|
|
6480
|
+
nodes: rows
|
|
6481
|
+
})));
|
|
6482
|
+
const nodes = await step(entry.id, "validate", () => encoded.map((candidate, index) => {
|
|
6483
|
+
const parsed = shape.node.safeParse(candidate);
|
|
6484
|
+
if (!parsed.success) throw new KeepsetError(`the result is not ${entry.category}: ${parsed.error.issues.map((issue) => `${[
|
|
6485
|
+
"nodes",
|
|
6486
|
+
String(offset + index),
|
|
6487
|
+
...issue.path.map(String)
|
|
6488
|
+
].join(".")}: ${issue.message}`).join("; ")}`);
|
|
6489
|
+
if (!ids.add(parsed.data.id)) throw new KeepsetError(`the result is not ${entry.category}: nodes.${String(offset + index)}.id: duplicate node id ${JSON.stringify(parsed.data.id)}`);
|
|
6490
|
+
return parsed.data;
|
|
6491
|
+
}));
|
|
6492
|
+
await sink.nodes(nodes);
|
|
6493
|
+
}
|
|
6494
|
+
const counts = /* @__PURE__ */ new Map([["nodes", ids.size]]);
|
|
6495
|
+
const warnings = Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`);
|
|
6496
|
+
return {
|
|
6497
|
+
dataset,
|
|
6498
|
+
meta,
|
|
6499
|
+
provenance: native.provenance(),
|
|
6500
|
+
counts,
|
|
6501
|
+
warnings
|
|
6502
|
+
};
|
|
6503
|
+
}
|
|
6504
|
+
/**
|
|
6505
|
+
* Checks a streamed dataset's crosswalk on its real data a page at a time: crossmap's `assertRoundTrip` on the meta alone and then on each page's rows under the meta. The crosswalk applies to a page on its own, so the round trip holds on the whole table exactly when it holds on every page. It stops at the first page that differs and reports the path in the whole document.
|
|
6506
|
+
* @param dataset - A dataset whose recipe streams.
|
|
6507
|
+
* @param options - The network port, abort signal and clock.
|
|
6508
|
+
* @returns Whether the round trip holds, and where it first differs when it does not, with the provenance of the bytes it was checked on: every byte when it holds, those read up to the page that differs when it does not.
|
|
6509
|
+
* @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison or the crosswalk cannot run.
|
|
6510
|
+
*/
|
|
6511
|
+
async function roundTripTable(dataset, options) {
|
|
6512
|
+
const { entry, crosswalk } = dataset;
|
|
6513
|
+
const recipe = streamedRecipe(entry);
|
|
6514
|
+
assertSeparable(entry, crosswalk);
|
|
6515
|
+
const native = openNative(dataset, recipe, options);
|
|
6516
|
+
const check = (nodes, offset) => {
|
|
6517
|
+
try {
|
|
6518
|
+
assertRoundTrip(crosswalk, [{
|
|
6519
|
+
meta: native.meta,
|
|
6520
|
+
nodes
|
|
6521
|
+
}]);
|
|
6522
|
+
} catch (error) {
|
|
6523
|
+
if (error instanceof RoundTripMismatchError) {
|
|
6524
|
+
const path = error.path.replace(/^nodes\[(\d+)\]/, (_, index) => `nodes[${String(offset + Number(index))}]`);
|
|
6525
|
+
return {
|
|
6526
|
+
holds: false,
|
|
6527
|
+
path,
|
|
6528
|
+
message: error.message.replaceAll(error.path, path),
|
|
6529
|
+
provenance: native.provenance()
|
|
6530
|
+
};
|
|
6531
|
+
}
|
|
6532
|
+
throw new RecipeError(entry.id, "crosswalk", error instanceof Error ? error.message : String(error), { cause: error });
|
|
6533
|
+
}
|
|
6534
|
+
};
|
|
6535
|
+
const metaMismatch = check([], 0);
|
|
6536
|
+
if (metaMismatch !== void 0) return metaMismatch;
|
|
6537
|
+
let offset = 0;
|
|
6538
|
+
for await (const rows of native.pages) {
|
|
6539
|
+
const mismatch = check(rows, offset);
|
|
6540
|
+
if (mismatch !== void 0) {
|
|
6541
|
+
await native.pages.return();
|
|
6542
|
+
return mismatch;
|
|
6543
|
+
}
|
|
6544
|
+
offset += rows.length;
|
|
6545
|
+
}
|
|
6546
|
+
return {
|
|
6547
|
+
holds: true,
|
|
6548
|
+
provenance: native.provenance()
|
|
6549
|
+
};
|
|
6550
|
+
}
|
|
6551
|
+
//#endregion
|
|
6552
|
+
//#region src/node/fetch-dataset.ts
|
|
6553
|
+
/**
|
|
6554
|
+
* The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
|
|
6555
|
+
* @param source - The source.
|
|
6556
|
+
* @returns The URL to request.
|
|
6557
|
+
*/
|
|
6558
|
+
function downloadUrl(source) {
|
|
6559
|
+
return sourceUrl(source);
|
|
6560
|
+
}
|
|
5624
6561
|
/**
|
|
5625
6562
|
* Puts an instance's id beside the fields of its encoded document. A document that is not an object is left as it is, for the collection schema to refuse.
|
|
5626
6563
|
* @param document - The crosswalk's output for one member.
|
|
@@ -5634,6 +6571,139 @@ function withId(document, id) {
|
|
|
5634
6571
|
} : document;
|
|
5635
6572
|
}
|
|
5636
6573
|
/**
|
|
6574
|
+
* Downloads one URL's bytes through the network port.
|
|
6575
|
+
* @param port - The network port.
|
|
6576
|
+
* @param url - The URL.
|
|
6577
|
+
* @param signal - Aborts the download.
|
|
6578
|
+
* @returns The bytes.
|
|
6579
|
+
* @throws KeepsetError when the server does not answer successfully.
|
|
6580
|
+
*/
|
|
6581
|
+
async function downloadBytes(port, url, signal) {
|
|
6582
|
+
const response = await port(url, signal === void 0 ? {} : { signal });
|
|
6583
|
+
if (!response.ok) throw new KeepsetError(`${url} answered ${String(response.status)} ${response.statusText}`);
|
|
6584
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
6585
|
+
}
|
|
6586
|
+
/**
|
|
6587
|
+
* The record of one download: the source, the URL requested and a hash of exactly the bytes that arrived.
|
|
6588
|
+
* @param source - The entry's source.
|
|
6589
|
+
* @param fetchedUrl - The URL requested.
|
|
6590
|
+
* @param bytes - The bytes.
|
|
6591
|
+
* @returns The download record.
|
|
6592
|
+
*/
|
|
6593
|
+
function downloadOf(source, fetchedUrl, bytes) {
|
|
6594
|
+
return {
|
|
6595
|
+
source: sourceRecord(source),
|
|
6596
|
+
fetchedUrl,
|
|
6597
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
6598
|
+
byteLength: bytes.byteLength
|
|
6599
|
+
};
|
|
6600
|
+
}
|
|
6601
|
+
/**
|
|
6602
|
+
* Parses one file of a collection into its native document, naming the file when it fails.
|
|
6603
|
+
* @param id - The instance's id.
|
|
6604
|
+
* @param bytes - The file's bytes.
|
|
6605
|
+
* @param parser - The recipe's parser.
|
|
6606
|
+
* @returns The instance's native document, with only its own header values as `meta`.
|
|
6607
|
+
* @throws KeepsetError naming the instance when the file does not parse.
|
|
6608
|
+
*/
|
|
6609
|
+
function parseInstance(id, bytes, parser) {
|
|
6610
|
+
try {
|
|
6611
|
+
const parsed = parseNative(bytes, parser);
|
|
6612
|
+
return {
|
|
6613
|
+
id,
|
|
6614
|
+
native: {
|
|
6615
|
+
...parsed,
|
|
6616
|
+
meta: "meta" in parsed ? parsed.meta : {}
|
|
6617
|
+
}
|
|
6618
|
+
};
|
|
6619
|
+
} catch (error) {
|
|
6620
|
+
throw new KeepsetError(`member ${id}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
6621
|
+
}
|
|
6622
|
+
}
|
|
6623
|
+
/**
|
|
6624
|
+
* Unpacks and parses the bytes of one archive or file into instances, each member let go once it is parsed.
|
|
6625
|
+
* @param bytes - The bytes.
|
|
6626
|
+
* @param archive - Their archive layout.
|
|
6627
|
+
* @param parser - The recipe's parser.
|
|
6628
|
+
* @param prefix - The path the instance ids are filed under, joined to each member's path, or undefined to use the member's path alone.
|
|
6629
|
+
* @returns The instances, in member order.
|
|
6630
|
+
*/
|
|
6631
|
+
function instancesOf(bytes, archive, parser, prefix) {
|
|
6632
|
+
const named = (member) => {
|
|
6633
|
+
if (prefix === void 0) {
|
|
6634
|
+
if (member === void 0) throw new KeepsetError("an instance of a collection has no path");
|
|
6635
|
+
return member;
|
|
6636
|
+
}
|
|
6637
|
+
return member === void 0 ? prefix : `${prefix}/${member}`;
|
|
6638
|
+
};
|
|
6639
|
+
if ("members" in archive) return Array.from(unpackMembers(bytes, archive), (member) => parseInstance(named(member.path), member.bytes, parser));
|
|
6640
|
+
if ("parts" in archive) throw new ArchiveError(`an instance is read from one file or from matching members, not from the named parts ${archive.parts.map((part) => part.name).join(", ")}`);
|
|
6641
|
+
if ("files" in archive) throw new ArchiveError("an instance is read from one file or from matching members, not from every file a multi-file format's files selector matches");
|
|
6642
|
+
const unpacked = unpack(bytes, archive);
|
|
6643
|
+
return [parseInstance(named(unpacked.member), unpacked.bytes, parser)];
|
|
6644
|
+
}
|
|
6645
|
+
/**
|
|
6646
|
+
* Unpacks and parses instances as `instancesOf` does, failing at the extract step when the archive cannot be read or its selector matches nothing, and at the parse step when a member does not parse: the members are unpacked one at a time as they are parsed, so the two steps interleave.
|
|
6647
|
+
* @param entryId - The entry.
|
|
6648
|
+
* @param run - The unpacking and parsing.
|
|
6649
|
+
* @returns The instances.
|
|
6650
|
+
* @throws RecipeError naming the entry and the step that failed.
|
|
6651
|
+
*/
|
|
6652
|
+
function extractAndParse(entryId, run) {
|
|
6653
|
+
try {
|
|
6654
|
+
return run();
|
|
6655
|
+
} catch (error) {
|
|
6656
|
+
throw new RecipeError(entryId, error instanceof ArchiveError ? "extract" : "parse", error instanceof Error ? error.message : String(error), { cause: error });
|
|
6657
|
+
}
|
|
6658
|
+
}
|
|
6659
|
+
/**
|
|
6660
|
+
* Runs a files recipe: downloads each file in turn (a dead file from its own pinned snapshot), unpacks it and parses it into its instances, so only one file's bytes are held at a time.
|
|
6661
|
+
* @param dataset - The dataset.
|
|
6662
|
+
* @param recipe - Its files recipe.
|
|
6663
|
+
* @param options - The network port, abort signal and clock.
|
|
6664
|
+
* @returns The instances and the provenance of every file.
|
|
6665
|
+
*/
|
|
6666
|
+
async function runFiles(dataset, recipe, options) {
|
|
6667
|
+
const { entry } = dataset;
|
|
6668
|
+
const { fetch: port = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
|
|
6669
|
+
const retrievedAt = now().toISOString();
|
|
6670
|
+
const read = async (index) => {
|
|
6671
|
+
const file = recipe.files[index];
|
|
6672
|
+
if (file === void 0) return [];
|
|
6673
|
+
const source = entry.sources[file.source];
|
|
6674
|
+
if (source === void 0) throw new RecipeError(entry.id, "recipe", `file ${file.path} names source ${String(file.source)}, which the entry does not have`);
|
|
6675
|
+
const fetchedUrl = sourceUrl(source);
|
|
6676
|
+
const bytes = await step(entry.id, "download", async () => downloadBytes(port, fetchedUrl, signal));
|
|
6677
|
+
const documents = extractAndParse(entry.id, () => instancesOf(bytes, file.archive, recipe.parser, file.path));
|
|
6678
|
+
return [{
|
|
6679
|
+
documents,
|
|
6680
|
+
download: {
|
|
6681
|
+
...downloadOf(source, fetchedUrl, bytes),
|
|
6682
|
+
path: file.path,
|
|
6683
|
+
instances: documents.map((document) => document.id)
|
|
6684
|
+
}
|
|
6685
|
+
}, ...await read(index + 1)];
|
|
6686
|
+
};
|
|
6687
|
+
const files = await read(0);
|
|
6688
|
+
const documents = files.flatMap((file) => file.documents);
|
|
6689
|
+
const ids = /* @__PURE__ */ new Set();
|
|
6690
|
+
for (const { id } of documents) {
|
|
6691
|
+
if (ids.has(id)) throw new RecipeError(entry.id, "extract", `two files give the instance ${id}`);
|
|
6692
|
+
ids.add(id);
|
|
6693
|
+
}
|
|
6694
|
+
return {
|
|
6695
|
+
collection: true,
|
|
6696
|
+
documents,
|
|
6697
|
+
provenance: {
|
|
6698
|
+
entryId: entry.id,
|
|
6699
|
+
catalogue: dataset.catalogue,
|
|
6700
|
+
category: entry.category,
|
|
6701
|
+
retrievedAt,
|
|
6702
|
+
files: files.map((file) => file.download)
|
|
6703
|
+
}
|
|
6704
|
+
};
|
|
6705
|
+
}
|
|
6706
|
+
/**
|
|
5637
6707
|
* Runs a recipe's download, extract and parse steps, and builds the native documents the crosswalk receives.
|
|
5638
6708
|
* @param dataset - The dataset.
|
|
5639
6709
|
* @param options - The network port, abort signal and clock.
|
|
@@ -5644,51 +6714,28 @@ async function runToNative(dataset, options) {
|
|
|
5644
6714
|
const { fetch: fetchPort = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
|
|
5645
6715
|
const { recipe } = entry;
|
|
5646
6716
|
if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
|
|
6717
|
+
if ("files" in recipe) return runFiles(dataset, recipe, options);
|
|
6718
|
+
if (recipe.stream === true) throw new RecipeError(entry.id, "recipe", "the table is too large to hold as one document, so its recipe streams: read it with streamDataset, which hands its nodes on a page at a time");
|
|
5647
6719
|
const source = entry.sources[recipe.source];
|
|
5648
6720
|
if (source === void 0) throw new RecipeError(entry.id, "recipe", `the recipe names source ${String(recipe.source)}, which the entry does not have`);
|
|
5649
|
-
const fetchedUrl =
|
|
6721
|
+
const fetchedUrl = sourceUrl(source);
|
|
5650
6722
|
const retrievedAt = now().toISOString();
|
|
5651
|
-
const bytes = await step(entry.id, "download", async () =>
|
|
5652
|
-
if (recipe.paging !== void 0) return downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal);
|
|
5653
|
-
const response = await fetchPort(fetchedUrl, signal === void 0 ? {} : { signal });
|
|
5654
|
-
if (!response.ok) throw new KeepsetError(`${fetchedUrl} answered ${String(response.status)} ${response.statusText}`);
|
|
5655
|
-
return new Uint8Array(await response.arrayBuffer());
|
|
5656
|
-
});
|
|
6723
|
+
const bytes = await step(entry.id, "download", async () => recipe.paging === void 0 ? downloadBytes(fetchPort, fetchedUrl, signal) : downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal));
|
|
5657
6724
|
const provenanceOf = (extra) => ({
|
|
5658
6725
|
entryId: entry.id,
|
|
5659
6726
|
catalogue: dataset.catalogue,
|
|
5660
6727
|
category: entry.category,
|
|
5661
|
-
source: {
|
|
5662
|
-
url: source.url,
|
|
5663
|
-
status: source.status,
|
|
5664
|
-
...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
|
|
5665
|
-
},
|
|
5666
|
-
fetchedUrl,
|
|
5667
6728
|
retrievedAt,
|
|
5668
|
-
|
|
5669
|
-
byteLength: bytes.byteLength,
|
|
6729
|
+
...downloadOf(source, fetchedUrl, bytes),
|
|
5670
6730
|
...extra
|
|
5671
6731
|
});
|
|
5672
6732
|
const { archive } = recipe;
|
|
5673
6733
|
if ("members" in archive) {
|
|
5674
|
-
const
|
|
6734
|
+
const documents = extractAndParse(entry.id, () => instancesOf(bytes, archive, recipe.parser, void 0));
|
|
5675
6735
|
return {
|
|
5676
6736
|
collection: true,
|
|
5677
|
-
documents
|
|
5678
|
-
|
|
5679
|
-
const parsed = parseNative(file.bytes, recipe.parser);
|
|
5680
|
-
return {
|
|
5681
|
-
id: file.path,
|
|
5682
|
-
native: {
|
|
5683
|
-
...parsed,
|
|
5684
|
-
meta: "meta" in parsed ? parsed.meta : {}
|
|
5685
|
-
}
|
|
5686
|
-
};
|
|
5687
|
-
} catch (error) {
|
|
5688
|
-
throw new KeepsetError(`member ${file.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5689
|
-
}
|
|
5690
|
-
})),
|
|
5691
|
-
provenance: provenanceOf({ members: files.map((file) => file.path) })
|
|
6737
|
+
documents,
|
|
6738
|
+
provenance: provenanceOf({ members: documents.map((document) => document.id) })
|
|
5692
6739
|
};
|
|
5693
6740
|
}
|
|
5694
6741
|
if ("parts" in archive) {
|
|
@@ -5741,7 +6788,7 @@ async function runToNative(dataset, options) {
|
|
|
5741
6788
|
* @param dataset - A dataset from any catalogue.
|
|
5742
6789
|
* @param options - The network port, abort signal and clock.
|
|
5743
6790
|
* @returns The resolved target with its provenance and any count warnings.
|
|
5744
|
-
* @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
|
|
6791
|
+
* @throws RecipeError naming the entry and the failing step, including for a recipe that streams, which `streamEntry` reads; nothing partial or default-shaped is ever returned.
|
|
5745
6792
|
*/
|
|
5746
6793
|
async function fetchEntry(dataset, options = {}) {
|
|
5747
6794
|
const { entry, crosswalk } = dataset;
|
|
@@ -5782,13 +6829,14 @@ async function fetchDataset(id, options = {}) {
|
|
|
5782
6829
|
return fetchEntry(getDataset(id), options);
|
|
5783
6830
|
}
|
|
5784
6831
|
/**
|
|
5785
|
-
* Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
|
|
6832
|
+
* Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve. A streamed recipe is checked a page at a time, with the round trip holding on the whole table exactly when it holds on every page, so the table is never held.
|
|
5786
6833
|
* @param dataset - A dataset from any catalogue.
|
|
5787
6834
|
* @param options - The network port, abort signal and clock.
|
|
5788
6835
|
* @returns Whether the round trip holds, and where it first differs when it does not.
|
|
5789
6836
|
* @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
|
|
5790
6837
|
*/
|
|
5791
6838
|
async function roundTripEntry(dataset, options = {}) {
|
|
6839
|
+
if (isStreamed(dataset.entry.recipe)) return roundTripTable(dataset, options);
|
|
5792
6840
|
const { documents, provenance } = await runToNative(dataset, options);
|
|
5793
6841
|
try {
|
|
5794
6842
|
assertRoundTrip(dataset.crosswalk, documents.map(({ native }) => native));
|
|
@@ -5807,6 +6855,28 @@ async function roundTripEntry(dataset, options = {}) {
|
|
|
5807
6855
|
};
|
|
5808
6856
|
}
|
|
5809
6857
|
/**
|
|
6858
|
+
* Streams a dataset of the bundled catalogue by id, as `streamEntry` does: for a table too large to hold as one document, whose recipe streams.
|
|
6859
|
+
* @param id - The entry id.
|
|
6860
|
+
* @param sink - Where the meta and the nodes go.
|
|
6861
|
+
* @param options - The network port, abort signal and clock.
|
|
6862
|
+
* @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
|
|
6863
|
+
* @throws DatasetNotFoundError when no entry has that id, and RecipeError as `streamEntry` does.
|
|
6864
|
+
*/
|
|
6865
|
+
async function streamDataset(id, sink, options = {}) {
|
|
6866
|
+
return streamEntry(getDataset(id), sink, options);
|
|
6867
|
+
}
|
|
6868
|
+
/**
|
|
6869
|
+
* Streams a dataset whose table is too large to hold as one document: reads its paged source a page at a time, applies the crosswalk to each page's rows under the document's meta, checks the meta once and each node against the category's node shape and for an id no earlier node had, and hands the nodes on to the sink before reading the next page. What it holds at once is one page's bytes and rows and the ids seen so far (as bytes, not strings), never the table, so the result is the category target `{ meta, nodes }` delivered in order rather than held: the same nodes `fetchEntry` would give, checked by the same rules.
|
|
6870
|
+
* @param dataset - A dataset whose recipe streams.
|
|
6871
|
+
* @param sink - Where the meta and the nodes go.
|
|
6872
|
+
* @param options - The network port, abort signal and clock.
|
|
6873
|
+
* @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
|
|
6874
|
+
* @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
|
|
6875
|
+
*/
|
|
6876
|
+
async function streamEntry(dataset, sink, options = {}) {
|
|
6877
|
+
return streamTable(dataset, sink, options);
|
|
6878
|
+
}
|
|
6879
|
+
/**
|
|
5810
6880
|
* Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
|
|
5811
6881
|
* @param id - The entry id.
|
|
5812
6882
|
* @param options - The network port, abort signal and clock.
|
|
@@ -5817,4 +6887,4 @@ async function roundTripDataset(id, options = {}) {
|
|
|
5817
6887
|
return roundTripEntry(getDataset(id), options);
|
|
5818
6888
|
}
|
|
5819
6889
|
//#endregion
|
|
5820
|
-
export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry };
|
|
6890
|
+
export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry, streamDataset, streamEntry };
|