keepset 0.38.0 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{bundled-U2l4WT5T.js → bundled-CL1eRNRR.js} +11918 -7517
- package/dist/{categories-gZh8cn-G.d.ts → categories-BbW0rFCf.d.ts} +2957 -231
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/node/fetch-dataset.d.ts +76 -25
- package/dist/node/fetch-dataset.js +1634 -295
- package/dist-cli/keepset.mjs +80 -9
- package/package.json +1 -1
- package/schemas/entry.schema.json +1 -1
- package/schemas/manifest.schema.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { J as getTarget, K as getCollectionTarget, U as countTarget, V as getOwn, Y as isStreamable, dt as NativeParseError, ft as RecipeError, ot as ArchiveError, q as getStreamedShape, t as getDataset, ut as KeepsetError } from "../bundled-CL1eRNRR.js";
|
|
2
2
|
import { RoundTripMismatchError, assertRoundTrip, encode } from "crossmap";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
@@ -127,6 +127,15 @@ function endpoint$1(fields, column, lineNumber) {
|
|
|
127
127
|
return value.trim();
|
|
128
128
|
}
|
|
129
129
|
/**
|
|
130
|
+
* Whether a line meets one of the recipe's `where` conditions.
|
|
131
|
+
* @param fields - The line's fields, at least as many as the highest column the recipe names.
|
|
132
|
+
* @param condition - The condition.
|
|
133
|
+
* @returns True when the condition's column, trimmed, equals its value.
|
|
134
|
+
*/
|
|
135
|
+
function meets(fields, condition) {
|
|
136
|
+
return fields[condition.column]?.trim() === condition.equals;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
130
139
|
* Resolves `detect` to a fixed delimiter from the first data line: tab when it holds a tab, comma when it holds a comma, whitespace otherwise. The rule is a function of the file's bytes, so a recipe that detects still reads the same way every time.
|
|
131
140
|
* @param dataLine - The first line that is neither skipped, blank nor a comment.
|
|
132
141
|
* @returns The fixed delimiter.
|
|
@@ -149,31 +158,33 @@ function multiplicity(fields, column, lineNumber) {
|
|
|
149
158
|
return Number(value);
|
|
150
159
|
}
|
|
151
160
|
/**
|
|
152
|
-
* Parses a delimited edge list: one edge per line, source and target in the configured columns, after `skipLines` header lines, with blank lines and lines whose first non-blank characters are a comment prefix skipped. A `detect` delimiter is fixed from the first data line. A recipe that names a `keep` column reads only the lines whose field there is one of its values
|
|
153
|
-
* @param text - The file text.
|
|
161
|
+
* Parses a delimited edge list: one edge per line, source and target in the configured columns, after `skipLines` header lines, with blank lines and lines whose first non-blank characters are a comment prefix skipped. A `detect` delimiter is fixed from the first data line. A recipe that names a `keep` column reads only the lines whose field there is one of its values, and one that states `where` reads only the lines whose listed columns, trimmed, equal their values, so a file that records several kinds of row (an interaction file listing genetic beside physical interactions) is read for the rows the dataset is made of; a line that is not read is still checked for the fields it needs. A recipe that names a `multiplicityColumn` emits as many edges for a line as the field says, none for a zero; one that says `mirrored` emits each pair of mirror-image lines once. Nodes are the endpoints of the lines read, in order of first appearance. The text is read line by line, so a file longer than the engine's longest string is read too.
|
|
162
|
+
* @param text - The file text, whole or in pieces.
|
|
154
163
|
* @param options - The recipe's edge-list parser.
|
|
155
164
|
* @returns The native graph.
|
|
156
|
-
* @throws NativeParseError naming the line when a data line has too few fields, an empty endpoint or a multiplicity that is not a whole number, when a mirrored table has a line without its mirror, or when
|
|
165
|
+
* @throws NativeParseError naming the line when a data line has too few fields, an empty endpoint or a multiplicity that is not a whole number, when a mirrored table has a line without its mirror, or when no line is an edge.
|
|
157
166
|
*/
|
|
158
167
|
function parseEdgeList(text, options) {
|
|
159
168
|
let split = options.delimiter === "detect" ? void 0 : splitters[options.delimiter];
|
|
160
|
-
const
|
|
169
|
+
const { where } = options;
|
|
170
|
+
const needed = Math.max(options.sourceColumn, options.targetColumn, options.keep?.column ?? 0, options.multiplicityColumn ?? 0, ...where === void 0 ? [] : where.map((condition) => condition.column)) + 1;
|
|
161
171
|
const order = new NodeOrder();
|
|
162
172
|
const edges = [];
|
|
163
173
|
const awaiting = /* @__PURE__ */ new Map();
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
174
|
+
let lastLine = 0;
|
|
175
|
+
forEachLine(typeof text === "string" ? [text] : text, (line, lineNumber) => {
|
|
176
|
+
lastLine = lineNumber;
|
|
167
177
|
const content = line.trimStart();
|
|
168
|
-
if (
|
|
178
|
+
if (lineNumber <= options.skipLines || content === "" || options.commentPrefixes.some((prefix) => content.startsWith(prefix))) return;
|
|
169
179
|
split ??= splitters[detect(line)];
|
|
170
180
|
const fields = split(line);
|
|
171
181
|
if (fields.length < needed) throw new NativeParseError("edge-list", lineNumber, `expected at least ${String(needed)} fields, found ${String(fields.length)}`);
|
|
172
|
-
if (options.keep !== void 0 && !options.keep.values.includes(fields[options.keep.column]?.trim() ?? ""))
|
|
182
|
+
if (options.keep !== void 0 && !options.keep.values.includes(fields[options.keep.column]?.trim() ?? "")) return;
|
|
183
|
+
if (where !== void 0 && !where.every((condition) => meets(fields, condition))) return;
|
|
173
184
|
const source = endpoint$1(fields, options.sourceColumn, lineNumber);
|
|
174
185
|
const target = endpoint$1(fields, options.targetColumn, lineNumber);
|
|
175
186
|
const count = options.multiplicityColumn === void 0 ? 1 : multiplicity(fields, options.multiplicityColumn, lineNumber);
|
|
176
|
-
if (count === 0)
|
|
187
|
+
if (count === 0) return;
|
|
177
188
|
order.add(source);
|
|
178
189
|
order.add(target);
|
|
179
190
|
if (options.mirrored === true && source !== target) {
|
|
@@ -185,7 +196,7 @@ function parseEdgeList(text, options) {
|
|
|
185
196
|
const waiting = awaiting.get(key);
|
|
186
197
|
if (waiting !== void 0 && waiting.length > 0) {
|
|
187
198
|
waiting.shift();
|
|
188
|
-
|
|
199
|
+
return;
|
|
189
200
|
}
|
|
190
201
|
const mirror = JSON.stringify([
|
|
191
202
|
target,
|
|
@@ -198,12 +209,12 @@ function parseEdgeList(text, options) {
|
|
|
198
209
|
source,
|
|
199
210
|
target
|
|
200
211
|
});
|
|
201
|
-
}
|
|
212
|
+
});
|
|
202
213
|
for (const [key, waiting] of awaiting) {
|
|
203
214
|
const [first] = waiting;
|
|
204
215
|
if (first !== void 0) throw new NativeParseError("edge-list", first, `the table is mirrored but this line has no mirror image (${key} is never listed)`);
|
|
205
216
|
}
|
|
206
|
-
if (edges.length === 0) throw new NativeParseError("edge-list",
|
|
217
|
+
if (edges.length === 0) throw new NativeParseError("edge-list", lastLine, where === void 0 ? "the file holds no edges" : "no line of the file meets the recipe's where conditions");
|
|
207
218
|
return {
|
|
208
219
|
nodes: order.nodes(),
|
|
209
220
|
edges
|
|
@@ -211,8 +222,8 @@ function parseEdgeList(text, options) {
|
|
|
211
222
|
}
|
|
212
223
|
//#endregion
|
|
213
224
|
//#region src/parsers/gd-contest.ts
|
|
214
|
-
const FORMAT$
|
|
215
|
-
const NUMBER$
|
|
225
|
+
const FORMAT$18 = "gd-contest";
|
|
226
|
+
const NUMBER$10 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
|
|
216
227
|
/**
|
|
217
228
|
* Reads a vertex number, which the format numbers from 0 to N - 1.
|
|
218
229
|
* @param token - The token.
|
|
@@ -220,7 +231,7 @@ const NUMBER$9 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
|
|
|
220
231
|
* @returns The vertex id as text.
|
|
221
232
|
*/
|
|
222
233
|
function vertex$1(token, count) {
|
|
223
|
-
if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$
|
|
234
|
+
if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$18, token.line, `expected a vertex number from 0 to ${String(count - 1)}, found ${JSON.stringify(token.text)}`);
|
|
224
235
|
return token.text;
|
|
225
236
|
}
|
|
226
237
|
/**
|
|
@@ -240,17 +251,17 @@ function parseGdContest(text) {
|
|
|
240
251
|
});
|
|
241
252
|
}
|
|
242
253
|
const [first] = tokens;
|
|
243
|
-
if (first === void 0) throw new NativeParseError(FORMAT$
|
|
244
|
-
if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$
|
|
254
|
+
if (first === void 0) throw new NativeParseError(FORMAT$18, 1, "the file holds no vertex count");
|
|
255
|
+
if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$18, first.line, `the vertex count must be a non-negative integer, found ${JSON.stringify(first.text)}`);
|
|
245
256
|
const count = Number(first.text);
|
|
246
257
|
const coordinates = tokens.slice(1, 1 + 2 * count);
|
|
247
|
-
if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$
|
|
248
|
-
for (const coordinate of coordinates) if (!NUMBER$
|
|
258
|
+
if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$18, lines(text).length, `expected ${String(2 * count)} coordinates for ${String(count)} vertices, found ${String(coordinates.length)}`);
|
|
259
|
+
for (const coordinate of coordinates) if (!NUMBER$10.test(coordinate.text)) throw new NativeParseError(FORMAT$18, coordinate.line, `coordinate ${JSON.stringify(coordinate.text)} is not a number`);
|
|
249
260
|
const ends = tokens.slice(1 + 2 * count);
|
|
250
261
|
const edges = [];
|
|
251
262
|
for (let source = ends.shift(); source !== void 0; source = ends.shift()) {
|
|
252
263
|
const target = ends.shift();
|
|
253
|
-
if (target === void 0) throw new NativeParseError(FORMAT$
|
|
264
|
+
if (target === void 0) throw new NativeParseError(FORMAT$18, source.line, `edge from vertex ${source.text} has no second vertex`);
|
|
254
265
|
edges.push({
|
|
255
266
|
source: vertex$1(source, count),
|
|
256
267
|
target: vertex$1(target, count)
|
|
@@ -264,7 +275,7 @@ function parseGdContest(text) {
|
|
|
264
275
|
//#endregion
|
|
265
276
|
//#region src/parsers/gml.ts
|
|
266
277
|
const KEY = /[A-Za-z_][A-Za-z0-9_]*/y;
|
|
267
|
-
const NUMBER$
|
|
278
|
+
const NUMBER$9 = /[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?/y;
|
|
268
279
|
/**
|
|
269
280
|
* Matches a sticky pattern at a position.
|
|
270
281
|
* @param pattern - A sticky regular expression.
|
|
@@ -315,7 +326,7 @@ function tokenise(text) {
|
|
|
315
326
|
position = end + 1;
|
|
316
327
|
} else {
|
|
317
328
|
const key = matchAt(KEY, text, position);
|
|
318
|
-
const number = key === void 0 ? matchAt(NUMBER$
|
|
329
|
+
const number = key === void 0 ? matchAt(NUMBER$9, text, position) : void 0;
|
|
319
330
|
const match = key ?? number;
|
|
320
331
|
if (match === void 0) throw new NativeParseError("gml", line, `unexpected character ${JSON.stringify(char)}`);
|
|
321
332
|
tokens.push({
|
|
@@ -420,7 +431,7 @@ function parseGml(text) {
|
|
|
420
431
|
}
|
|
421
432
|
//#endregion
|
|
422
433
|
//#region src/parsers/hgb.ts
|
|
423
|
-
const FORMAT$
|
|
434
|
+
const FORMAT$17 = "hgb";
|
|
424
435
|
/** A type code or node id as the HGB files write them: a non-negative decimal integer with no sign or leading zero. */
|
|
425
436
|
const CODE = /^(?:0|[1-9]\d*)$/;
|
|
426
437
|
/** A number as a decimal literal with an optional sign and exponent, so `Number` cannot read hex, an empty string or infinity. */
|
|
@@ -440,7 +451,7 @@ const LINK_FIELDS = 4;
|
|
|
440
451
|
*/
|
|
441
452
|
function resolve(field, types, where) {
|
|
442
453
|
const type = types[CODE.test(field) ? Number(field) : -1];
|
|
443
|
-
if (type === void 0) throw new NativeParseError(FORMAT$
|
|
454
|
+
if (type === void 0) throw new NativeParseError(FORMAT$17, where.line, `${where.file}: the ${where.kind} type ${JSON.stringify(field)} is not one of the ${String(types.length)} the recipe names`);
|
|
444
455
|
return type;
|
|
445
456
|
}
|
|
446
457
|
/**
|
|
@@ -458,9 +469,9 @@ function parseHgb(parts, options) {
|
|
|
458
469
|
forEachLine(parts.nodes, (line, number) => {
|
|
459
470
|
const fields = line.split(" ");
|
|
460
471
|
const [id, , type] = fields;
|
|
461
|
-
if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$
|
|
462
|
-
if (!CODE.test(id)) throw new NativeParseError(FORMAT$
|
|
463
|
-
if (typeOf.has(id)) throw new NativeParseError(FORMAT$
|
|
472
|
+
if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$17, number, `node.dat: expected ${String(NODE_FIELDS)} or ${String(NODE_FIELDS_WITH_ATTRIBUTES)} fields separated by tabs, found ${String(fields.length)}`);
|
|
473
|
+
if (!CODE.test(id)) throw new NativeParseError(FORMAT$17, number, `node.dat: the node id ${JSON.stringify(id)} is not a non-negative integer`);
|
|
474
|
+
if (typeOf.has(id)) throw new NativeParseError(FORMAT$17, number, `node.dat: the node id ${id} is repeated`);
|
|
464
475
|
const nodeType = resolve(type, options.nodeTypes, {
|
|
465
476
|
file: "node.dat",
|
|
466
477
|
line: number,
|
|
@@ -472,12 +483,12 @@ function parseHgb(parts, options) {
|
|
|
472
483
|
type: nodeType
|
|
473
484
|
});
|
|
474
485
|
});
|
|
475
|
-
if (nodes.length === 0) throw new NativeParseError(FORMAT$
|
|
486
|
+
if (nodes.length === 0) throw new NativeParseError(FORMAT$17, 1, "node.dat: the file holds no nodes");
|
|
476
487
|
const edges = [];
|
|
477
488
|
forEachLine(parts.links, (line, number) => {
|
|
478
489
|
const fields = line.split(" ");
|
|
479
490
|
const [source, target, type, weight] = fields;
|
|
480
|
-
if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$
|
|
491
|
+
if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$17, number, `link.dat: expected ${String(LINK_FIELDS)} fields separated by tabs, found ${String(fields.length)}`);
|
|
481
492
|
const linkType = resolve(type, options.linkTypes, {
|
|
482
493
|
file: "link.dat",
|
|
483
494
|
line: number,
|
|
@@ -493,17 +504,17 @@ function parseHgb(parts, options) {
|
|
|
493
504
|
linkType.target
|
|
494
505
|
]]) {
|
|
495
506
|
const actual = typeOf.get(id);
|
|
496
|
-
if (actual === void 0) throw new NativeParseError(FORMAT$
|
|
497
|
-
if (actual !== expected) throw new NativeParseError(FORMAT$
|
|
507
|
+
if (actual === void 0) throw new NativeParseError(FORMAT$17, number, `link.dat: the ${end} ${JSON.stringify(id)} is not a node of node.dat`);
|
|
508
|
+
if (actual !== expected) throw new NativeParseError(FORMAT$17, number, `link.dat: a ${linkType.name} link's ${end} must be a ${expected} node, and ${id} is a ${actual} node`);
|
|
498
509
|
}
|
|
499
|
-
if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$
|
|
510
|
+
if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$17, number, `link.dat: the weight ${JSON.stringify(weight)} is not 1; a weighted HGB file would lose its weights`);
|
|
500
511
|
edges.push({
|
|
501
512
|
source,
|
|
502
513
|
target,
|
|
503
514
|
type: linkType.name
|
|
504
515
|
});
|
|
505
516
|
});
|
|
506
|
-
if (edges.length === 0) throw new NativeParseError(FORMAT$
|
|
517
|
+
if (edges.length === 0) throw new NativeParseError(FORMAT$17, 1, "link.dat: the file holds no links");
|
|
507
518
|
return {
|
|
508
519
|
nodes,
|
|
509
520
|
edges
|
|
@@ -511,7 +522,7 @@ function parseHgb(parts, options) {
|
|
|
511
522
|
}
|
|
512
523
|
//#endregion
|
|
513
524
|
//#region src/parsers/kg-triples.ts
|
|
514
|
-
const FORMAT$
|
|
525
|
+
const FORMAT$16 = "kg-triples";
|
|
515
526
|
/** The character each delimiter option splits a line on. A line is split on exactly that character, so a field cannot hide a second delimiter. */
|
|
516
527
|
const DELIMITERS$1 = {
|
|
517
528
|
tab: " ",
|
|
@@ -533,8 +544,8 @@ const QUOTE$2 = "\"";
|
|
|
533
544
|
*/
|
|
534
545
|
function value(field, quoted, where) {
|
|
535
546
|
const inner = quoted ? field.slice(1, -1) : field;
|
|
536
|
-
if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$
|
|
537
|
-
if (inner === "") throw new NativeParseError(FORMAT$
|
|
547
|
+
if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$16, where.line, `${where.part}the field ${JSON.stringify(field)} is not one value wrapped in double quotes`);
|
|
548
|
+
if (inner === "") throw new NativeParseError(FORMAT$16, where.line, `${where.part}a field is empty`);
|
|
538
549
|
return inner;
|
|
539
550
|
}
|
|
540
551
|
/**
|
|
@@ -558,16 +569,16 @@ function parseKgTriples(parts, options) {
|
|
|
558
569
|
let lastLine = 0;
|
|
559
570
|
forEachLine(part.pieces, (line, number) => {
|
|
560
571
|
lastLine = number;
|
|
561
|
-
if (line.trim() === "") throw new NativeParseError(FORMAT$
|
|
572
|
+
if (line.trim() === "") throw new NativeParseError(FORMAT$16, number, `${label}the line is blank`);
|
|
562
573
|
if (number === 1 && options.header === "names") return;
|
|
563
574
|
if (number === 1 && options.header === "count") {
|
|
564
|
-
if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$
|
|
575
|
+
if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$16, number, `${label}expected the number of triples on the first line, found ${JSON.stringify(line)}`);
|
|
565
576
|
declared = Number(line);
|
|
566
577
|
return;
|
|
567
578
|
}
|
|
568
579
|
const fields = line.split(delimiter);
|
|
569
580
|
const [head, middle, last] = fields;
|
|
570
|
-
if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$
|
|
581
|
+
if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$16, number, `${label}expected ${String(TRIPLE_FIELDS)} fields separated by ${options.delimiter}, found ${String(fields.length)}`);
|
|
571
582
|
const where = {
|
|
572
583
|
line: number,
|
|
573
584
|
part: label
|
|
@@ -593,8 +604,8 @@ function parseKgTriples(parts, options) {
|
|
|
593
604
|
});
|
|
594
605
|
triples += 1;
|
|
595
606
|
});
|
|
596
|
-
if (triples === 0) throw new NativeParseError(FORMAT$
|
|
597
|
-
if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$
|
|
607
|
+
if (triples === 0) throw new NativeParseError(FORMAT$16, lastLine, `${label}the file holds no triples`);
|
|
608
|
+
if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$16, lastLine, `${label}the first line declares ${String(declared)} triples, the file holds ${String(triples)}`);
|
|
598
609
|
}
|
|
599
610
|
return {
|
|
600
611
|
nodes: order.nodes(),
|
|
@@ -603,7 +614,7 @@ function parseKgTriples(parts, options) {
|
|
|
603
614
|
}
|
|
604
615
|
//#endregion
|
|
605
616
|
//#region src/parsers/konect.ts
|
|
606
|
-
const FORMAT$
|
|
617
|
+
const FORMAT$15 = "konect";
|
|
607
618
|
/** The network formats of KONECT's first line (Kunegis, "The KONECT Handbook", table 1): undirected, directed and bipartite. */
|
|
608
619
|
const NETWORK_FORMATS = /* @__PURE__ */ new Set([
|
|
609
620
|
"sym",
|
|
@@ -678,9 +689,9 @@ function parseKonect(pieces, options) {
|
|
|
678
689
|
}));
|
|
679
690
|
});
|
|
680
691
|
const { header } = state;
|
|
681
|
-
if (header === void 0) throw new NativeParseError(FORMAT$
|
|
682
|
-
if (edges.length === 0) throw new NativeParseError(FORMAT$
|
|
683
|
-
if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$
|
|
692
|
+
if (header === void 0) throw new NativeParseError(FORMAT$15, lastLine, "the file has no first line naming its format and weights");
|
|
693
|
+
if (edges.length === 0) throw new NativeParseError(FORMAT$15, lastLine, "the file holds no edges");
|
|
694
|
+
if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$15, lastLine, `the second line declares ${String(header.counts.edges)} edges, the file holds ${String(edges.length)}`);
|
|
684
695
|
return {
|
|
685
696
|
nodes: order.nodes(),
|
|
686
697
|
edges
|
|
@@ -698,9 +709,9 @@ function readHeader(content, number, options) {
|
|
|
698
709
|
const match = FORMAT_LINE.exec(content);
|
|
699
710
|
const network = match?.[1];
|
|
700
711
|
const weights = match?.[2];
|
|
701
|
-
if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$
|
|
702
|
-
if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$
|
|
703
|
-
if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$
|
|
712
|
+
if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$15, number, `expected a first line "% FORMAT WEIGHTS" with FORMAT one of ${[...NETWORK_FORMATS].join(", ")}, found ${JSON.stringify(content)}`);
|
|
713
|
+
if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$15, number, `the weight type ${JSON.stringify(weights)} is not one this parser reads (its third column is not an edge weight or a repetition count)`);
|
|
714
|
+
if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$15, number, `the file's weight type is ${weights}, so a recipe with weighted ${String(WEIGHTED_TYPES.has(weights))} reads it, not one with weighted ${String(options.weighted)}`);
|
|
704
715
|
return {
|
|
705
716
|
line: number,
|
|
706
717
|
network
|
|
@@ -720,15 +731,15 @@ function readEdge(content, number, context) {
|
|
|
720
731
|
const [from, to, third, time] = fields;
|
|
721
732
|
const expected = options.timestamped ? TIMESTAMPED_COLUMNS : options.weighted ? WEIGHTED_COLUMNS : void 0;
|
|
722
733
|
const fits = expected === void 0 ? fields.length === ENDPOINT_COLUMNS || fields.length === WEIGHTED_COLUMNS : fields.length === expected;
|
|
723
|
-
if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$
|
|
734
|
+
if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$15, number, `expected ${expected === void 0 ? `${String(ENDPOINT_COLUMNS)} or ${String(WEIGHTED_COLUMNS)}` : String(expected)} columns for this recipe, found ${String(fields.length)}`);
|
|
724
735
|
const bipartite = header.network === "bip";
|
|
725
736
|
const source = endpoint(from, bipartite ? LEFT_PREFIX : "", header.counts?.left, number);
|
|
726
737
|
const target = endpoint(to, bipartite ? RIGHT_PREFIX : "", header.counts?.right, number);
|
|
727
738
|
order.add(source);
|
|
728
739
|
order.add(target);
|
|
729
|
-
if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$
|
|
730
|
-
if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$
|
|
731
|
-
if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$
|
|
740
|
+
if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$15, number, `the third column ${JSON.stringify(third)} is not 1; an edge that stands for several would be read as one`);
|
|
741
|
+
if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$15, number, `the weight ${JSON.stringify(third)} is not a number`);
|
|
742
|
+
if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$15, number, `the time ${JSON.stringify(time)} is not a number`);
|
|
732
743
|
const edge = {
|
|
733
744
|
source,
|
|
734
745
|
target
|
|
@@ -747,10 +758,215 @@ function readEdge(content, number, context) {
|
|
|
747
758
|
* @throws NativeParseError when the token is not a positive integer or exceeds the set's size.
|
|
748
759
|
*/
|
|
749
760
|
function endpoint(token, prefix, size, number) {
|
|
750
|
-
if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$
|
|
761
|
+
if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$15, number, `${JSON.stringify(token)} is not a node id from 1${size === void 0 ? "" : ` to ${String(size)}`}`);
|
|
751
762
|
return `${prefix}${token}`;
|
|
752
763
|
}
|
|
753
764
|
//#endregion
|
|
765
|
+
//#region src/parsers/ogb.ts
|
|
766
|
+
const FORMAT$14 = "ogb";
|
|
767
|
+
/** The files of an OGB raw directory this reader takes, by their path under the dataset's root directory. A relation's files sit in `raw/relations/<head>___<relation>___<tail>/`. */
|
|
768
|
+
const RAW = /^(?<root>(?:[^/]+\/)*?)raw\/(?<name>num-node-list|num-node-dict|edge|num-edge-list|triplet-type-list|relations\/[^/]+\/(?:edge|num-edge-list))\.csv\.gz$/u;
|
|
769
|
+
const INDEX = /^(?:0|[1-9]\d*)$/;
|
|
770
|
+
/** The fields of a line of `triplet-type-list.csv.gz`: a head type, a relation and a tail type. */
|
|
771
|
+
const TRIPLET_PARTS = 3;
|
|
772
|
+
/**
|
|
773
|
+
* Orders two names by code point, the order Python's `sorted` gives the ASCII type and relation names OGB uses.
|
|
774
|
+
* @param left - A name.
|
|
775
|
+
* @param right - Another.
|
|
776
|
+
* @returns Negative, zero or positive.
|
|
777
|
+
*/
|
|
778
|
+
function compare(left, right) {
|
|
779
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* Visits the non-blank lines of a file, read a piece at a time.
|
|
783
|
+
* @param file - The file.
|
|
784
|
+
* @param visit - Called with each non-blank line and its number.
|
|
785
|
+
*/
|
|
786
|
+
function forEachRow(file, visit) {
|
|
787
|
+
forEachLine(file.text(), (text, line) => {
|
|
788
|
+
if (text.trim() !== "") visit({
|
|
789
|
+
text,
|
|
790
|
+
line
|
|
791
|
+
});
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
/**
|
|
795
|
+
* The non-blank lines of a small file, such as a count or the triplet list.
|
|
796
|
+
* @param file - The file.
|
|
797
|
+
* @returns Each non-blank line and its number.
|
|
798
|
+
*/
|
|
799
|
+
function rows(file) {
|
|
800
|
+
const all = [];
|
|
801
|
+
forEachRow(file, (row) => {
|
|
802
|
+
all.push(row);
|
|
803
|
+
});
|
|
804
|
+
return all;
|
|
805
|
+
}
|
|
806
|
+
/**
|
|
807
|
+
* Reads a file that holds one line, such as a count.
|
|
808
|
+
* @param file - The file.
|
|
809
|
+
* @returns The line's text.
|
|
810
|
+
* @throws NativeParseError when the file holds no line or more than one.
|
|
811
|
+
*/
|
|
812
|
+
function onlyLine(file) {
|
|
813
|
+
const all = rows(file);
|
|
814
|
+
const [only] = all;
|
|
815
|
+
if (all.length !== 1 || only === void 0) throw new NativeParseError(FORMAT$14, all[1]?.line ?? 1, `${file.path} must hold one line, found ${String(all.length)}`);
|
|
816
|
+
return only.text.trim();
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Reads a non-negative integer.
|
|
820
|
+
* @param text - The field.
|
|
821
|
+
* @param file - The file, for the error.
|
|
822
|
+
* @param line - The 1-based line, for the error.
|
|
823
|
+
* @returns The integer.
|
|
824
|
+
* @throws NativeParseError when the field is not a non-negative integer.
|
|
825
|
+
*/
|
|
826
|
+
function index(text, file, line) {
|
|
827
|
+
const trimmed = text.trim();
|
|
828
|
+
if (!INDEX.test(trimmed)) throw new NativeParseError(FORMAT$14, line, `${file.path}: expected a non-negative integer, found ${JSON.stringify(text)}`);
|
|
829
|
+
return Number(trimmed);
|
|
830
|
+
}
|
|
831
|
+
/**
|
|
832
|
+
* Reads an `edge.csv.gz` file, one `<source>,<target>` pair of node indices per line, checking each index against its node type's count and the line count against the relation's `num-edge-list.csv.gz`.
|
|
833
|
+
* @param edges - The edge file.
|
|
834
|
+
* @param count - The relation's `num-edge-list.csv.gz`.
|
|
835
|
+
* @param ends - The source and target node types' counts and how each index is named.
|
|
836
|
+
* @param emit - Called with each edge, in file order, so a large file is never held twice.
|
|
837
|
+
* @throws NativeParseError naming the file and line for a line that is not a pair of indices, an index outside its type's nodes, or a line count other than the recorded one.
|
|
838
|
+
*/
|
|
839
|
+
function readEdges(edges, count, ends, emit) {
|
|
840
|
+
const recorded = index(onlyLine(count), count, 1);
|
|
841
|
+
let read = 0;
|
|
842
|
+
forEachRow(edges, ({ text, line }) => {
|
|
843
|
+
const fields = text.split(",");
|
|
844
|
+
const [first, second] = fields;
|
|
845
|
+
if (fields.length !== 2 || first === void 0 || second === void 0) throw new NativeParseError(FORMAT$14, line, `${edges.path}: expected a source and a target index, found ${JSON.stringify(text)}`);
|
|
846
|
+
const source = index(first, edges, line);
|
|
847
|
+
const target = index(second, edges, line);
|
|
848
|
+
if (source >= ends.sources || target >= ends.targets) throw new NativeParseError(FORMAT$14, line, `${edges.path}: edge ${text} names a node outside the ${String(ends.sources)} sources and ${String(ends.targets)} targets the dataset records`);
|
|
849
|
+
emit({
|
|
850
|
+
source: ends.source(source),
|
|
851
|
+
target: ends.target(target)
|
|
852
|
+
});
|
|
853
|
+
read += 1;
|
|
854
|
+
});
|
|
855
|
+
if (read !== recorded) throw new NativeParseError(FORMAT$14, read, `${edges.path} holds ${String(read)} edges, but ${count.path} records ${String(recorded)}`);
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Takes one file of the dataset directory by its name under `raw/`.
|
|
859
|
+
* @param files - The files by name.
|
|
860
|
+
* @param name - The name, such as `edge` or `relations/paper___cites___paper/edge`.
|
|
861
|
+
* @param root - The dataset's root directory, for the error.
|
|
862
|
+
* @returns The file.
|
|
863
|
+
* @throws NativeParseError when the archive's selection holds no such file.
|
|
864
|
+
*/
|
|
865
|
+
function required$1(files, name, root) {
|
|
866
|
+
const file = files.get(name);
|
|
867
|
+
if (file === void 0) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has no raw/${name}.csv.gz among the files the recipe selects`);
|
|
868
|
+
return file;
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* Parses an Open Graph Benchmark dataset from the files of its raw directory, as OGB's own `read_csv_graph_raw` and `read_csv_heterograph_raw` read them (ogb/io/read_graph_raw.py in snap-stanford/ogb). A homogeneous dataset has `raw/num-node-list.csv.gz` (its one graph's node count), `raw/edge.csv.gz` (one `<source>,<target>` pair of node indices per line, no header) and `raw/num-edge-list.csv.gz`: its nodes are `0` to `N - 1`, isolated ones included, and its edges the pairs in file order. A heterogeneous dataset has `raw/num-node-dict.csv.gz` (a header of node types over one row of their counts), `raw/triplet-type-list.csv.gz` (one `<head>,<relation>,<tail>` type triplet per line) and, per triplet, `raw/relations/<head>___<relation>___<tail>/edge.csv.gz` and `num-edge-list.csv.gz`, whose indices count within the head and tail types: its nodes are `<type>:<index>` for each type in sorted order and each index from 0, and its edges are each relation's pairs in file order, relations in the sorted order of their triplets, which is the order OGB reads them in. Edges keep the file's direction. Relation names, node features, labels and splits are not carried, as no graph parser in this version carries more than topology.
|
|
872
|
+
* @param files - The files the recipe's archive selects, gunzipped and decoded: every one must be a file of one dataset's raw directory named above.
|
|
873
|
+
* @returns The native graph.
|
|
874
|
+
* @throws NativeParseError naming the file and line for a file outside the raw layout, files of two datasets, a dataset with neither or both node-count files, a missing file, a graph count other than one, a malformed count or triplet, a relation directory the triplet list does not name, an edge that is not a pair of indices of the dataset's nodes, or an edge file whose line count differs from its recorded count.
|
|
875
|
+
*/
|
|
876
|
+
function parseOgb(files) {
|
|
877
|
+
const byName = /* @__PURE__ */ new Map();
|
|
878
|
+
const roots = /* @__PURE__ */ new Set();
|
|
879
|
+
for (const file of files) {
|
|
880
|
+
const match = RAW.exec(file.path);
|
|
881
|
+
const name = match?.groups?.["name"];
|
|
882
|
+
const root = match?.groups?.["root"];
|
|
883
|
+
if (name === void 0 || root === void 0) throw new NativeParseError(FORMAT$14, 1, `${file.path} is not a file of an OGB raw directory this reader takes`);
|
|
884
|
+
roots.add(root);
|
|
885
|
+
byName.set(name, file);
|
|
886
|
+
}
|
|
887
|
+
const [root] = roots;
|
|
888
|
+
if (roots.size !== 1 || root === void 0) throw new NativeParseError(FORMAT$14, 1, `the recipe must select the raw files of one dataset, found ${roots.size === 0 ? "none" : `the directories ${[...roots].join(", ")}`}`);
|
|
889
|
+
const homogeneous = byName.get("num-node-list");
|
|
890
|
+
const heterogeneous = byName.get("num-node-dict");
|
|
891
|
+
const neitherOrBoth = `the dataset directory ${root} must have exactly one of raw/num-node-list.csv.gz (one node type) and raw/num-node-dict.csv.gz (several)`;
|
|
892
|
+
const edges = [];
|
|
893
|
+
if (homogeneous !== void 0) {
|
|
894
|
+
if (heterogeneous !== void 0) throw new NativeParseError(FORMAT$14, 1, neitherOrBoth);
|
|
895
|
+
const [relation] = [...byName.keys()].filter((name) => name.startsWith("relations/"));
|
|
896
|
+
if (relation !== void 0 || byName.has("triplet-type-list")) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has raw/num-node-list.csv.gz, so it has one node type, but the recipe selects ${relation === void 0 ? "raw/triplet-type-list.csv.gz" : `raw/${relation}.csv.gz`} as well`);
|
|
897
|
+
const count = index(onlyLine(homogeneous), homogeneous, 1);
|
|
898
|
+
const name = (node) => String(node);
|
|
899
|
+
readEdges(required$1(byName, "edge", root), required$1(byName, "num-edge-list", root), {
|
|
900
|
+
sources: count,
|
|
901
|
+
targets: count,
|
|
902
|
+
source: name,
|
|
903
|
+
target: name
|
|
904
|
+
}, (edge) => {
|
|
905
|
+
edges.push(edge);
|
|
906
|
+
});
|
|
907
|
+
return {
|
|
908
|
+
nodes: Array.from({ length: count }, (_, node) => ({ id: String(node) })),
|
|
909
|
+
edges
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
if (heterogeneous === void 0) throw new NativeParseError(FORMAT$14, 1, neitherOrBoth);
|
|
913
|
+
for (const stray of ["edge", "num-edge-list"]) if (byName.has(stray)) throw new NativeParseError(FORMAT$14, 1, `the dataset directory ${root} has raw/num-node-dict.csv.gz, so its edges are in raw/relations/, but the recipe selects raw/${stray}.csv.gz as well`);
|
|
914
|
+
const header = rows(heterogeneous);
|
|
915
|
+
const [types, counts] = header;
|
|
916
|
+
if (header.length !== 2 || types === void 0 || counts === void 0) throw new NativeParseError(FORMAT$14, header[2]?.line ?? 1, `${heterogeneous.path} must hold a header of node types and one row of their counts, found ${String(header.length)} lines`);
|
|
917
|
+
const typeNames = types.text.split(",").map((type) => type.trim());
|
|
918
|
+
const typeCounts = counts.text.split(",");
|
|
919
|
+
if (typeNames.length !== typeCounts.length || typeNames.some((type) => type === "") || new Set(typeNames).size !== typeNames.length) throw new NativeParseError(FORMAT$14, counts.line, `${heterogeneous.path} must name each node type once, over one count per type`);
|
|
920
|
+
const nodeTypes = typeNames.map((type, position) => {
|
|
921
|
+
const count = typeCounts[position];
|
|
922
|
+
if (count === void 0) throw new NativeParseError(FORMAT$14, counts.line, `${heterogeneous.path} gives node type ${type} no count`);
|
|
923
|
+
return {
|
|
924
|
+
type,
|
|
925
|
+
count: index(count, heterogeneous, counts.line)
|
|
926
|
+
};
|
|
927
|
+
});
|
|
928
|
+
nodeTypes.sort((left, right) => compare(left.type, right.type));
|
|
929
|
+
const nodeCounts = new Map(nodeTypes.map(({ type, count }) => [type, count]));
|
|
930
|
+
const triplets = required$1(byName, "triplet-type-list", root);
|
|
931
|
+
const relations = [];
|
|
932
|
+
for (const { text, line } of rows(triplets)) {
|
|
933
|
+
const parts = text.split(",").map((part) => part.trim());
|
|
934
|
+
const [head, relation, tail] = parts;
|
|
935
|
+
if (parts.length !== TRIPLET_PARTS || head === void 0 || relation === void 0 || tail === void 0 || relation === "") throw new NativeParseError(FORMAT$14, line, `${triplets.path}: expected a head type, a relation and a tail type, found ${JSON.stringify(text)}`);
|
|
936
|
+
const sources = nodeCounts.get(head);
|
|
937
|
+
const targets = nodeCounts.get(tail);
|
|
938
|
+
if (sources === void 0 || targets === void 0) throw new NativeParseError(FORMAT$14, line, `${triplets.path}: ${text} names a node type that ${heterogeneous.path} does not count`);
|
|
939
|
+
relations.push({
|
|
940
|
+
head,
|
|
941
|
+
relation,
|
|
942
|
+
tail,
|
|
943
|
+
sources,
|
|
944
|
+
targets
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
relations.sort((left, right) => compare(left.head, right.head) || compare(left.relation, right.relation) || compare(left.tail, right.tail));
|
|
948
|
+
const listed = new Set(relations.map(({ head, relation, tail }) => `relations/${head}___${relation}___${tail}`));
|
|
949
|
+
for (const name of byName.keys()) {
|
|
950
|
+
const directory = name.slice(0, name.lastIndexOf("/"));
|
|
951
|
+
if (name.startsWith("relations/") && !listed.has(directory)) throw new NativeParseError(FORMAT$14, 1, `raw/${directory}/ is not one of the relations ${triplets.path} lists`);
|
|
952
|
+
}
|
|
953
|
+
for (const { head, relation, tail, sources, targets } of relations) {
|
|
954
|
+
const directory = `relations/${head}___${relation}___${tail}`;
|
|
955
|
+
readEdges(required$1(byName, `${directory}/edge`, root), required$1(byName, `${directory}/num-edge-list`, root), {
|
|
956
|
+
sources,
|
|
957
|
+
targets,
|
|
958
|
+
source: (node) => `${head}:${String(node)}`,
|
|
959
|
+
target: (node) => `${tail}:${String(node)}`
|
|
960
|
+
}, (edge) => {
|
|
961
|
+
edges.push(edge);
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
return {
|
|
965
|
+
nodes: nodeTypes.flatMap(({ type, count }) => Array.from({ length: count }, (_, node) => ({ id: `${type}:${String(node)}` }))),
|
|
966
|
+
edges
|
|
967
|
+
};
|
|
968
|
+
}
|
|
969
|
+
//#endregion
|
|
754
970
|
//#region src/parsers/matrix-market.ts
|
|
755
971
|
/** How many tokens an entry line of each field type holds: the coordinates, then the value's parts (none for a pattern, a real and an imaginary part for a complex value). */
|
|
756
972
|
const tokensPerEntry = /* @__PURE__ */ new Map([
|
|
@@ -985,69 +1201,71 @@ var RecordScanner = class {
|
|
|
985
1201
|
}
|
|
986
1202
|
};
|
|
987
1203
|
/**
|
|
988
|
-
*
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
carry = scanned.carry;
|
|
1001
|
-
yield* scanned.completed;
|
|
1204
|
+
* Reads a table file's records as its text arrives in pieces, so neither the whole file nor every cell is held at once: a leading byte order mark is removed, the physical lines the recipe skips are dropped, and the rest is split into records and fields by RFC 4180 (a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR). Blank lines are not records. A piece may end anywhere, even between the two quotes of a doubled quote or between a CR and its LF.
|
|
1205
|
+
*/
|
|
1206
|
+
var RecordReader = class {
|
|
1207
|
+
options;
|
|
1208
|
+
scanner;
|
|
1209
|
+
carry = "";
|
|
1210
|
+
first = true;
|
|
1211
|
+
toSkip;
|
|
1212
|
+
constructor(options) {
|
|
1213
|
+
this.options = options;
|
|
1214
|
+
this.scanner = new RecordScanner(DELIMITERS[options.delimiter].charCodeAt(0), options.skipLines + 1);
|
|
1215
|
+
this.toSkip = options.skipLines;
|
|
1002
1216
|
}
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1217
|
+
/**
|
|
1218
|
+
* Reads the next piece of the file's text.
|
|
1219
|
+
* @param piece - The text, which follows the pieces read before it.
|
|
1220
|
+
* @returns The records the piece completes, in order.
|
|
1221
|
+
* @throws NativeParseError when a quote opens inside an unquoted field.
|
|
1222
|
+
*/
|
|
1223
|
+
push(piece) {
|
|
1224
|
+
let rest = this.first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
|
|
1225
|
+
this.first = false;
|
|
1226
|
+
while (this.toSkip > 0) {
|
|
1227
|
+
const next = rest.indexOf("\n");
|
|
1228
|
+
if (next === -1) return [];
|
|
1229
|
+
rest = rest.slice(next + 1);
|
|
1230
|
+
this.toSkip -= 1;
|
|
1231
|
+
}
|
|
1232
|
+
if (rest === "") return [];
|
|
1233
|
+
const scanned = this.scanner.scan(this.carry + rest, false);
|
|
1234
|
+
this.carry = scanned.carry;
|
|
1235
|
+
return scanned.completed.filter(isNotBlank);
|
|
1236
|
+
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Ends the file.
|
|
1239
|
+
* @returns The records left when the text ends, in order.
|
|
1240
|
+
* @throws NativeParseError when a quoted field is never closed, a quote opens inside an unquoted field, or the file has fewer lines than the recipe skips.
|
|
1241
|
+
*/
|
|
1242
|
+
end() {
|
|
1243
|
+
if (this.toSkip > 0) throw new NativeParseError("csv-table", this.options.skipLines - this.toSkip + 1, `the file has fewer than ${String(this.options.skipLines)} lines to skip before the header`);
|
|
1244
|
+
const completed = this.scanner.scan(this.carry, true).completed;
|
|
1245
|
+
this.carry = "";
|
|
1246
|
+
const last = this.scanner.finish();
|
|
1247
|
+
return [...completed, ...last === void 0 ? [] : [last]].filter(isNotBlank);
|
|
1248
|
+
}
|
|
1249
|
+
};
|
|
1007
1250
|
/**
|
|
1008
|
-
* Splits comma-separated text into its records' fields by RFC 4180, as
|
|
1251
|
+
* Splits comma-separated text into its records' fields by RFC 4180, as the record reader does, keeping every record (blank lines included) at its position: the grid a workbook sheet's CSV makes, for a parser that reads a layout of blocks and not one table.
|
|
1009
1252
|
* @param text - The text.
|
|
1010
1253
|
* @returns Each record's fields, in order.
|
|
1011
1254
|
* @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
|
|
1012
1255
|
*/
|
|
1013
1256
|
function csvGrid(text) {
|
|
1014
|
-
|
|
1015
|
-
}
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
* @param pieces - The file text in order.
|
|
1019
|
-
* @param count - How many physical lines to drop.
|
|
1020
|
-
* @returns The rest of the text, in pieces.
|
|
1021
|
-
* @throws NativeParseError when the file has fewer lines than that.
|
|
1022
|
-
*/
|
|
1023
|
-
function* afterLines(pieces, count) {
|
|
1024
|
-
let remaining = count;
|
|
1025
|
-
for (const piece of pieces) {
|
|
1026
|
-
let rest = piece;
|
|
1027
|
-
while (remaining > 0) {
|
|
1028
|
-
const next = rest.indexOf("\n");
|
|
1029
|
-
if (next === -1) {
|
|
1030
|
-
rest = "";
|
|
1031
|
-
break;
|
|
1032
|
-
}
|
|
1033
|
-
rest = rest.slice(next + 1);
|
|
1034
|
-
remaining -= 1;
|
|
1035
|
-
}
|
|
1036
|
-
if (remaining === 0 && rest !== "") yield rest;
|
|
1037
|
-
}
|
|
1038
|
-
if (remaining > 0) throw new NativeParseError("csv-table", count - remaining + 1, `the file has fewer than ${String(count)} lines to skip before the header`);
|
|
1257
|
+
const scanner = new RecordScanner(DELIMITERS.comma.charCodeAt(0), 1);
|
|
1258
|
+
const { completed } = scanner.scan(text, true);
|
|
1259
|
+
const last = scanner.finish();
|
|
1260
|
+
return [...completed, ...last === void 0 ? [] : [last]].map((record) => record.fields);
|
|
1039
1261
|
}
|
|
1040
1262
|
/**
|
|
1041
|
-
*
|
|
1042
|
-
* @param
|
|
1043
|
-
* @returns
|
|
1263
|
+
* Whether a record holds anything: a blank line is a record of one empty field.
|
|
1264
|
+
* @param record - The record.
|
|
1265
|
+
* @returns False for a blank line.
|
|
1044
1266
|
*/
|
|
1045
|
-
function
|
|
1046
|
-
|
|
1047
|
-
for (const piece of typeof text === "string" ? [text] : text) {
|
|
1048
|
-
yield first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
|
|
1049
|
-
first = false;
|
|
1050
|
-
}
|
|
1267
|
+
function isNotBlank(record) {
|
|
1268
|
+
return record.fields.length > 1 || record.fields[0]?.trim() !== "";
|
|
1051
1269
|
}
|
|
1052
1270
|
/**
|
|
1053
1271
|
* Reads a numeric cell.
|
|
@@ -1064,27 +1282,81 @@ function numeric$1(cell, factor, column, line) {
|
|
|
1064
1282
|
return Number(cell) * factor;
|
|
1065
1283
|
}
|
|
1066
1284
|
/**
|
|
1067
|
-
*
|
|
1068
|
-
* @param text - The file text, or its pieces in order, which may begin with a byte order mark.
|
|
1069
|
-
* @param options - The recipe's table parser, of which the delimiter and the lines to skip are read.
|
|
1070
|
-
* @returns Each record.
|
|
1071
|
-
* @throws NativeParseError when a quote is never closed or opens inside an unquoted field, or the file has fewer lines than the recipe skips.
|
|
1285
|
+
* Reads a delimited table into rows as its text arrives in pieces, holding only the record in progress between pieces, so a caller that hands each row on as it is returned holds no more of the table than that. The rows are exactly those `parseCsvTable` gives, in order, with the same errors on the same lines however the text is cut.
|
|
1072
1286
|
*/
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1287
|
+
var TableReader = class {
|
|
1288
|
+
options;
|
|
1289
|
+
records;
|
|
1290
|
+
layout;
|
|
1291
|
+
rows = 0;
|
|
1292
|
+
missing;
|
|
1293
|
+
constructor(options) {
|
|
1294
|
+
this.options = options;
|
|
1295
|
+
this.records = new RecordReader(options);
|
|
1296
|
+
this.missing = new Set(options.nullValues ?? []);
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Reads the next piece of the file's text.
|
|
1300
|
+
* @param piece - The text, which follows the pieces read before it.
|
|
1301
|
+
* @returns The rows the piece completes, in file order.
|
|
1302
|
+
* @throws NativeParseError as `parseCsvTable` does, naming the line of the problem.
|
|
1303
|
+
*/
|
|
1304
|
+
push(piece) {
|
|
1305
|
+
return this.rowsOf(this.records.push(piece));
|
|
1306
|
+
}
|
|
1307
|
+
/**
|
|
1308
|
+
* Ends the file.
|
|
1309
|
+
* @returns The rows left when the text ends, in file order.
|
|
1310
|
+
* @throws NativeParseError as `parseCsvTable` does, including for a file with no header row.
|
|
1311
|
+
*/
|
|
1312
|
+
end() {
|
|
1313
|
+
const rows = this.rowsOf(this.records.end());
|
|
1314
|
+
if (this.layout === void 0) throw new NativeParseError("csv-table", this.options.skipLines + 1, "the file has no header row");
|
|
1315
|
+
return rows;
|
|
1316
|
+
}
|
|
1317
|
+
/**
|
|
1318
|
+
* Builds the rows of records, the first record of the file being its header.
|
|
1319
|
+
* @param records - Records in file order.
|
|
1320
|
+
* @returns The rows among them.
|
|
1321
|
+
*/
|
|
1322
|
+
rowsOf(records) {
|
|
1323
|
+
const rows = [];
|
|
1324
|
+
for (const record of records) if (this.layout === void 0) this.layout = layoutOf(record, this.options);
|
|
1325
|
+
else rows.push(this.rowOf(record, this.layout));
|
|
1326
|
+
return rows;
|
|
1327
|
+
}
|
|
1328
|
+
/**
|
|
1329
|
+
* Builds one row.
|
|
1330
|
+
* @param record - A record after the header.
|
|
1331
|
+
* @param layout - The header's layout.
|
|
1332
|
+
* @returns The row.
|
|
1333
|
+
* @throws NativeParseError when the record's width differs from the header's, its id is empty or a numeric cell is not a number.
|
|
1334
|
+
*/
|
|
1335
|
+
rowOf(record, layout) {
|
|
1336
|
+
const { idColumn } = this.options;
|
|
1337
|
+
if (record.fields.length !== layout.width) throw new NativeParseError("csv-table", record.line, `the row has ${String(record.fields.length)} fields, the header has ${String(layout.width)}`);
|
|
1338
|
+
this.rows += 1;
|
|
1339
|
+
const id = idColumn === null ? String(this.rows) : (record.fields[layout.idPosition] ?? "").trim();
|
|
1340
|
+
if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
|
|
1341
|
+
const cells = layout.kept.map(({ name, position, factor }) => {
|
|
1342
|
+
const trimmed = (record.fields[position] ?? "").trim();
|
|
1343
|
+
const cell = this.missing.has(trimmed) ? "" : trimmed;
|
|
1344
|
+
return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
|
|
1345
|
+
});
|
|
1346
|
+
return {
|
|
1347
|
+
...Object.fromEntries(cells),
|
|
1348
|
+
id
|
|
1349
|
+
};
|
|
1350
|
+
}
|
|
1351
|
+
};
|
|
1077
1352
|
/**
|
|
1078
|
-
*
|
|
1079
|
-
* @param
|
|
1353
|
+
* Reads a table's header into the layout its rows are built by, checking the recipe against it.
|
|
1354
|
+
* @param header - The header record.
|
|
1080
1355
|
* @param options - The recipe's table parser.
|
|
1081
|
-
* @returns The
|
|
1082
|
-
* @throws NativeParseError
|
|
1356
|
+
* @returns The layout.
|
|
1357
|
+
* @throws NativeParseError when two header cells get the same name, the recipe names a column the header lacks, lists the id column among the columns, or declares numeric a column it does not keep.
|
|
1083
1358
|
*/
|
|
1084
|
-
function
|
|
1085
|
-
const table = tableRecords(text, options);
|
|
1086
|
-
const { value: header } = table.next();
|
|
1087
|
-
if (header === void 0) throw new NativeParseError("csv-table", options.skipLines + 1, "the file has no header row");
|
|
1359
|
+
function layoutOf(header, options) {
|
|
1088
1360
|
const names = columnNames(header.fields, options.idColumn, header.line);
|
|
1089
1361
|
const { idColumn, numericColumns, columns } = options;
|
|
1090
1362
|
const wanted = new Set(columns ?? names.filter((name) => name !== idColumn));
|
|
@@ -1096,28 +1368,28 @@ function parseCsvTable(text, options) {
|
|
|
1096
1368
|
if (idColumn !== null && wanted.has(idColumn)) throw new NativeParseError("csv-table", header.line, `the id column ${idColumn} is the row's id, so it is not listed among the columns`);
|
|
1097
1369
|
for (const { column } of numericColumns) if (!wanted.has(column)) throw new NativeParseError("csv-table", header.line, `the column ${column} is declared numeric but is not among the columns kept`);
|
|
1098
1370
|
const factors = new Map(numericColumns.map((entry) => [entry.column, entry.factor]));
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
name,
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1371
|
+
return {
|
|
1372
|
+
width: names.length,
|
|
1373
|
+
idPosition: idColumn === null ? -1 : names.indexOf(idColumn),
|
|
1374
|
+
kept: names.flatMap((name, position) => name !== idColumn && wanted.has(name) ? [{
|
|
1375
|
+
name,
|
|
1376
|
+
position,
|
|
1377
|
+
factor: factors.get(name)
|
|
1378
|
+
}] : [])
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
/**
|
|
1382
|
+
* Parses a delimited table into rows (a cell whose text is one of the recipe's `nullValues` is empty, so null): the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
|
|
1383
|
+
* @param text - The file text, or its pieces in order.
|
|
1384
|
+
* @param options - The recipe's table parser.
|
|
1385
|
+
* @returns The rows, in file order.
|
|
1386
|
+
* @throws NativeParseError naming the line of the first problem, including a listed or numeric column the header lacks or a numeric column that is not listed.
|
|
1387
|
+
*/
|
|
1388
|
+
function parseCsvTable(text, options) {
|
|
1389
|
+
const reader = new TableReader(options);
|
|
1106
1390
|
const nodes = [];
|
|
1107
|
-
for (const
|
|
1108
|
-
|
|
1109
|
-
const id = idColumn === null ? String(nodes.length + 1) : (record.fields[idPosition] ?? "").trim();
|
|
1110
|
-
if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
|
|
1111
|
-
const cells = kept.map(({ name, position, factor }) => {
|
|
1112
|
-
const trimmed = (record.fields[position] ?? "").trim();
|
|
1113
|
-
const cell = missing.has(trimmed) ? "" : trimmed;
|
|
1114
|
-
return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
|
|
1115
|
-
});
|
|
1116
|
-
nodes.push({
|
|
1117
|
-
...Object.fromEntries(cells),
|
|
1118
|
-
id
|
|
1119
|
-
});
|
|
1120
|
-
}
|
|
1391
|
+
for (const piece of typeof text === "string" ? [text] : text) for (const row of reader.push(piece)) nodes.push(row);
|
|
1392
|
+
for (const row of reader.end()) nodes.push(row);
|
|
1121
1393
|
return { nodes };
|
|
1122
1394
|
}
|
|
1123
1395
|
//#endregion
|
|
@@ -1222,7 +1494,7 @@ function parseGeojson(text, options) {
|
|
|
1222
1494
|
}
|
|
1223
1495
|
//#endregion
|
|
1224
1496
|
//#region src/parsers/hhcrsp.ts
|
|
1225
|
-
const FORMAT$
|
|
1497
|
+
const FORMAT$13 = "hhcrsp";
|
|
1226
1498
|
/** The labels of the sheet's blocks, each the top-left cell of a block whose header runs to its right and whose rows run below it. */
|
|
1227
1499
|
const LABEL = {
|
|
1228
1500
|
patients: "Patients",
|
|
@@ -1234,9 +1506,9 @@ const LABEL = {
|
|
|
1234
1506
|
};
|
|
1235
1507
|
/** A service's header, `Service <k>`, a patient's row label and a caregiver's. */
|
|
1236
1508
|
const SERVICE = /^Service (\d+)$/;
|
|
1237
|
-
const NUMBER$
|
|
1509
|
+
const NUMBER$8 = /^-?\d+(?:\.\d+)?$/;
|
|
1238
1510
|
/** How many required services a patient may have: the instances ask for one, or two that must be synchronised, and the parser has a duration column for each. */
|
|
1239
|
-
const MOST_SERVICES = 2;
|
|
1511
|
+
const MOST_SERVICES$1 = 2;
|
|
1240
1512
|
/**
|
|
1241
1513
|
* Finds a block by its label: the one cell of the sheet that holds it, its header the cells to its right up to the first empty one, and its rows those below it up to the first whose label and cells are all empty. Past its header a block has an empty column, so the cell there must be empty in every row and a value the header does not label is refused rather than left unread; `Pat / Ser` instead keeps its count there, with the next block starting right after it, and its count is checked on its own.
|
|
1242
1514
|
* @param grid - The sheet's cells by row.
|
|
@@ -1251,7 +1523,7 @@ function blockOf(grid, label, pastHeader = "empty") {
|
|
|
1251
1523
|
column
|
|
1252
1524
|
}] : []));
|
|
1253
1525
|
const [only] = found;
|
|
1254
|
-
if (only === void 0 || found.length !== 1) throw new NativeParseError(FORMAT$
|
|
1526
|
+
if (only === void 0 || found.length !== 1) throw new NativeParseError(FORMAT$13, 1, `expected one block labelled ${JSON.stringify(label)}, found ${String(found.length)}`);
|
|
1255
1527
|
const headerRow = grid[only.line] ?? [];
|
|
1256
1528
|
const end = headerRow.findIndex((cell, column) => column > only.column && cell.trim() === "");
|
|
1257
1529
|
const header = headerRow.slice(only.column + 1, end === -1 ? headerRow.length : end).map((cell) => cell.trim());
|
|
@@ -1260,7 +1532,7 @@ function blockOf(grid, label, pastHeader = "empty") {
|
|
|
1260
1532
|
const cells = (grid[line] ?? []).slice(only.column, only.column + 1 + header.length).map((cell) => cell.trim());
|
|
1261
1533
|
if (cells.every((cell) => cell === "")) break;
|
|
1262
1534
|
const past = (grid[line] ?? [])[only.column + 1 + header.length]?.trim();
|
|
1263
|
-
if (pastHeader === "empty" && past !== void 0 && past !== "") throw new NativeParseError(FORMAT$
|
|
1535
|
+
if (pastHeader === "empty" && past !== void 0 && past !== "") throw new NativeParseError(FORMAT$13, line + 1, `the ${label} block's row holds ${JSON.stringify(past)} past its header ${JSON.stringify(header)}, which does not label it`);
|
|
1264
1536
|
rows.push({
|
|
1265
1537
|
line: line + 1,
|
|
1266
1538
|
label: cells[0] ?? "",
|
|
@@ -1283,7 +1555,7 @@ function blockOf(grid, label, pastHeader = "empty") {
|
|
|
1283
1555
|
* @throws NativeParseError when the cell is not a decimal number.
|
|
1284
1556
|
*/
|
|
1285
1557
|
function numberOf$4(cell, line, what) {
|
|
1286
|
-
if (cell === void 0 || !NUMBER$
|
|
1558
|
+
if (cell === void 0 || !NUMBER$8.test(cell)) throw new NativeParseError(FORMAT$13, line, `expected ${what}, found ${JSON.stringify(cell)}`);
|
|
1287
1559
|
return Number(cell);
|
|
1288
1560
|
}
|
|
1289
1561
|
/**
|
|
@@ -1295,7 +1567,7 @@ function numberOf$4(cell, line, what) {
|
|
|
1295
1567
|
*/
|
|
1296
1568
|
function checkNumbered(block, noun, label) {
|
|
1297
1569
|
block.rows.forEach((row, index) => {
|
|
1298
|
-
if (row.label !== `${noun} ${String(index + 1)}`) throw new NativeParseError(FORMAT$
|
|
1570
|
+
if (row.label !== `${noun} ${String(index + 1)}`) throw new NativeParseError(FORMAT$13, row.line, `the ${label} block's row ${String(index + 1)} is labelled ${JSON.stringify(row.label)}, not ${noun} ${String(index + 1)}`);
|
|
1299
1571
|
});
|
|
1300
1572
|
}
|
|
1301
1573
|
/**
|
|
@@ -1307,7 +1579,7 @@ function checkNumbered(block, noun, label) {
|
|
|
1307
1579
|
*/
|
|
1308
1580
|
function serviceCount(block, label) {
|
|
1309
1581
|
block.header.forEach((cell, index) => {
|
|
1310
|
-
if (SERVICE.exec(cell)?.[1] !== String(index + 1)) throw new NativeParseError(FORMAT$
|
|
1582
|
+
if (SERVICE.exec(cell)?.[1] !== String(index + 1)) throw new NativeParseError(FORMAT$13, block.line, `the ${label} block's header cell ${String(index + 1)} is ${JSON.stringify(cell)}, not Service ${String(index + 1)}`);
|
|
1311
1583
|
});
|
|
1312
1584
|
return block.header.length;
|
|
1313
1585
|
}
|
|
@@ -1326,7 +1598,7 @@ function parseHhcrsp(text) {
|
|
|
1326
1598
|
const caregiverWindows = blockOf(grid, LABEL.caregiverWindows);
|
|
1327
1599
|
const windows = blockOf(grid, LABEL.patientWindows);
|
|
1328
1600
|
const serviceKinds = serviceCount(requests, LABEL.requests);
|
|
1329
|
-
if (serviceCount(times, LABEL.times) !== serviceKinds || serviceCount(skills, LABEL.skills) !== serviceKinds) throw new NativeParseError(FORMAT$
|
|
1601
|
+
if (serviceCount(times, LABEL.times) !== serviceKinds || serviceCount(skills, LABEL.skills) !== serviceKinds) throw new NativeParseError(FORMAT$13, times.line, `the ${LABEL.requests}, ${LABEL.times} and ${LABEL.skills} blocks name different numbers of services`);
|
|
1330
1602
|
checkNumbered(requests, "Patient", LABEL.requests);
|
|
1331
1603
|
checkNumbered(windows, "Patient", LABEL.patientWindows);
|
|
1332
1604
|
checkNumbered(skills, "Caregiver", LABEL.skills);
|
|
@@ -1341,11 +1613,11 @@ function parseHhcrsp(text) {
|
|
|
1341
1613
|
"",
|
|
1342
1614
|
"center",
|
|
1343
1615
|
"centre"
|
|
1344
|
-
].includes(centreTimes.label.toLowerCase()) || windows.rows.length !== patients || patientTimes.length !== patients || skills.rows.length !== caregiverWindows.rows.length) throw new NativeParseError(FORMAT$
|
|
1345
|
-
if (positions.header.join(",") !== "X,Y" || positions.rows.length !== patients + 2 || positions.rows.some((row, index) => row.label !== String(index))) throw new NativeParseError(FORMAT$
|
|
1616
|
+
].includes(centreTimes.label.toLowerCase()) || windows.rows.length !== patients || patientTimes.length !== patients || skills.rows.length !== caregiverWindows.rows.length) throw new NativeParseError(FORMAT$13, requests.line, `the blocks disagree: ${String(patients)} patients requesting services, ${String(patientTimes.length)} with durations after the centre's row, ${String(windows.rows.length)} with time windows, and ${String(skills.rows.length)} caregivers with skills against ${String(caregiverWindows.rows.length)} with windows`);
|
|
1617
|
+
if (positions.header.join(",") !== "X,Y" || positions.rows.length !== patients + 2 || positions.rows.some((row, index) => row.label !== String(index))) throw new NativeParseError(FORMAT$13, positions.line, `the ${LABEL.patients} block must give X and Y for nodes 0 to ${String(patients + 1)}, the centre, ${String(patients)} patients and the centre again`);
|
|
1346
1618
|
const position = (index) => {
|
|
1347
1619
|
const row = positions.rows[index];
|
|
1348
|
-
if (row === void 0) throw new NativeParseError(FORMAT$
|
|
1620
|
+
if (row === void 0) throw new NativeParseError(FORMAT$13, positions.line, `the ${LABEL.patients} block has no node ${String(index)}`);
|
|
1349
1621
|
return {
|
|
1350
1622
|
x: numberOf$4(row.cells[0], row.line, "an X"),
|
|
1351
1623
|
y: numberOf$4(row.cells[1], row.line, "a Y"),
|
|
@@ -1354,7 +1626,7 @@ function parseHhcrsp(text) {
|
|
|
1354
1626
|
};
|
|
1355
1627
|
const { x, y } = position(0);
|
|
1356
1628
|
const closing = position(patients + 1);
|
|
1357
|
-
if (closing.x !== x || closing.y !== y) throw new NativeParseError(FORMAT$
|
|
1629
|
+
if (closing.x !== x || closing.y !== y) throw new NativeParseError(FORMAT$13, closing.line, `the last node of the ${LABEL.patients} block is not at the centre's position, so it is not the route's end`);
|
|
1358
1630
|
const windowColumn = (name) => {
|
|
1359
1631
|
const index = windows.header.indexOf(name);
|
|
1360
1632
|
return index === -1 ? void 0 : index;
|
|
@@ -1365,7 +1637,7 @@ function parseHhcrsp(text) {
|
|
|
1365
1637
|
earliest2: windowColumn("ETW 2"),
|
|
1366
1638
|
latest2: windowColumn("LTW 2")
|
|
1367
1639
|
};
|
|
1368
|
-
if (columns.earliest1 === void 0 || columns.latest1 === void 0 || columns.earliest2 === void 0 !== (columns.latest2 === void 0) || windows.header.length !== (columns.earliest2 === void 0 ? 2 : 4)) throw new NativeParseError(FORMAT$
|
|
1640
|
+
if (columns.earliest1 === void 0 || columns.latest1 === void 0 || columns.earliest2 === void 0 !== (columns.latest2 === void 0) || windows.header.length !== (columns.earliest2 === void 0 ? 2 : 4)) throw new NativeParseError(FORMAT$13, windows.line, `the ${LABEL.patientWindows} header ${JSON.stringify(windows.header)} is not ETW 1 and LTW 1, with or without ETW 2 and LTW 2`);
|
|
1369
1641
|
const nodes = [{
|
|
1370
1642
|
id: "0",
|
|
1371
1643
|
role: "centre",
|
|
@@ -1381,15 +1653,15 @@ function parseHhcrsp(text) {
|
|
|
1381
1653
|
}];
|
|
1382
1654
|
requests.rows.forEach((request, index) => {
|
|
1383
1655
|
const required = request.cells.flatMap((cell, service) => {
|
|
1384
|
-
if (cell !== "0" && cell !== "1" && cell !== "") throw new NativeParseError(FORMAT$
|
|
1656
|
+
if (cell !== "0" && cell !== "1" && cell !== "") throw new NativeParseError(FORMAT$13, request.line, `${request.label} has ${JSON.stringify(cell)} for Service ${String(service + 1)}, not 0, 1 or empty`);
|
|
1385
1657
|
return cell === "1" ? [service] : [];
|
|
1386
1658
|
});
|
|
1387
1659
|
const stated = (grid[request.line - 1] ?? [])[requests.column + serviceKinds + 1]?.trim();
|
|
1388
|
-
if (stated !== String(required.length)) throw new NativeParseError(FORMAT$
|
|
1389
|
-
if (required.length === 0 || required.length > MOST_SERVICES) throw new NativeParseError(FORMAT$
|
|
1660
|
+
if (stated !== String(required.length)) throw new NativeParseError(FORMAT$13, request.line, `${request.label} requires ${String(required.length)} services by its marks, and the count beside them is ${JSON.stringify(stated)}`);
|
|
1661
|
+
if (required.length === 0 || required.length > MOST_SERVICES$1) throw new NativeParseError(FORMAT$13, request.line, `${request.label} requires ${String(required.length)} services, and a patient requires one or two`);
|
|
1390
1662
|
const timeRow = patientTimes[index];
|
|
1391
1663
|
const windowRow = windows.rows[index];
|
|
1392
|
-
if (timeRow === void 0 || windowRow === void 0) throw new NativeParseError(FORMAT$
|
|
1664
|
+
if (timeRow === void 0 || windowRow === void 0) throw new NativeParseError(FORMAT$13, request.line, `${request.label} has no durations or time windows`);
|
|
1393
1665
|
const duration = (which) => {
|
|
1394
1666
|
const service = required[which];
|
|
1395
1667
|
return service === void 0 ? null : numberOf$4(timeRow.cells[service], timeRow.line, `the duration of Service ${String(service + 1)}`);
|
|
@@ -1420,6 +1692,433 @@ function parseHhcrsp(text) {
|
|
|
1420
1692
|
};
|
|
1421
1693
|
}
|
|
1422
1694
|
//#endregion
|
|
1695
|
+
//#region src/parsers/mankowska.ts
|
|
1696
|
+
const FORMAT$12 = "mankowska";
|
|
1697
|
+
/** The sections of the file, each a keyword on a line of its own followed by its numbers. */
|
|
1698
|
+
const SECTIONS$1 = [
|
|
1699
|
+
"nbNodes",
|
|
1700
|
+
"nbVehi",
|
|
1701
|
+
"nbServi",
|
|
1702
|
+
"r",
|
|
1703
|
+
"DS",
|
|
1704
|
+
"a",
|
|
1705
|
+
"x",
|
|
1706
|
+
"y",
|
|
1707
|
+
"d",
|
|
1708
|
+
"p",
|
|
1709
|
+
"mind",
|
|
1710
|
+
"maxd",
|
|
1711
|
+
"e",
|
|
1712
|
+
"l"
|
|
1713
|
+
];
|
|
1714
|
+
/** How many patients a double service has, and so the most required services a patient can have. */
|
|
1715
|
+
const MOST_SERVICES = 2;
|
|
1716
|
+
/** The depot copies: the first and the last node. */
|
|
1717
|
+
const DEPOTS = 2;
|
|
1718
|
+
const NUMBER$7 = /^-?\d+(?:\.\d+)?$/;
|
|
1719
|
+
const KEYWORD = /^[A-Za-z]+$/;
|
|
1720
|
+
/**
|
|
1721
|
+
* Splits the file into its sections: a line holding one word starts a section, and every number on the lines after it, up to the next such line, is the section's.
|
|
1722
|
+
* @param text - The file text.
|
|
1723
|
+
* @returns Each section by its keyword.
|
|
1724
|
+
* @throws NativeParseError for a keyword the format does not have, one given twice, a value that is not a number, or a number before the first keyword.
|
|
1725
|
+
*/
|
|
1726
|
+
function sectionsOf$1(text) {
|
|
1727
|
+
const sections = /* @__PURE__ */ new Map();
|
|
1728
|
+
let current;
|
|
1729
|
+
for (const [index, raw] of lines(text).entries()) {
|
|
1730
|
+
const tokens = raw.trim().split(/\s+/).filter((token) => token !== "");
|
|
1731
|
+
const [only] = tokens;
|
|
1732
|
+
if (tokens.length === 1 && only !== void 0 && KEYWORD.test(only)) {
|
|
1733
|
+
const name = SECTIONS$1.find((section) => section === only);
|
|
1734
|
+
if (name === void 0 || sections.has(name)) throw new NativeParseError(FORMAT$12, index + 1, name === void 0 ? `the section ${JSON.stringify(only)} is not one of ${SECTIONS$1.join(", ")}` : `the section ${name} is given twice`);
|
|
1735
|
+
current = [];
|
|
1736
|
+
sections.set(name, {
|
|
1737
|
+
line: index + 1,
|
|
1738
|
+
values: current
|
|
1739
|
+
});
|
|
1740
|
+
continue;
|
|
1741
|
+
}
|
|
1742
|
+
for (const token of tokens) {
|
|
1743
|
+
if (current === void 0 || !NUMBER$7.test(token)) throw new NativeParseError(FORMAT$12, index + 1, current === void 0 ? "a number comes before the first section" : `expected a number, found ${JSON.stringify(token)}`);
|
|
1744
|
+
current.push(Number(token));
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
return sections;
|
|
1748
|
+
}
|
|
1749
|
+
/**
|
|
1750
|
+
* Reads a section that must hold a given count of numbers.
|
|
1751
|
+
* @param sections - The sections.
|
|
1752
|
+
* @param name - The section.
|
|
1753
|
+
* @param count - How many numbers it must hold.
|
|
1754
|
+
* @returns Its numbers.
|
|
1755
|
+
* @throws NativeParseError when the section is missing or holds another count.
|
|
1756
|
+
*/
|
|
1757
|
+
function sized(sections, name, count) {
|
|
1758
|
+
const section = sections.get(name);
|
|
1759
|
+
if (section === void 0) throw new NativeParseError(FORMAT$12, 1, `the file has no ${name} section`);
|
|
1760
|
+
if (section.values.length !== count) throw new NativeParseError(FORMAT$12, section.line, `the ${name} section holds ${String(section.values.length)} numbers where ${String(count)} are due`);
|
|
1761
|
+
return section.values;
|
|
1762
|
+
}
|
|
1763
|
+
/**
|
|
1764
|
+
* Reads a count section (`nbNodes`, `nbVehi`, `nbServi`): one whole number of at least a least value.
|
|
1765
|
+
* @param sections - The sections.
|
|
1766
|
+
* @param name - The section.
|
|
1767
|
+
* @param least - The smallest count that makes an instance.
|
|
1768
|
+
* @returns The count.
|
|
1769
|
+
* @throws NativeParseError when the section is missing, holds more than one number, or the count is not a whole number of at least `least`.
|
|
1770
|
+
*/
|
|
1771
|
+
function countOf(sections, name, least) {
|
|
1772
|
+
const [count] = sized(sections, name, 1);
|
|
1773
|
+
if (count === void 0 || !Number.isInteger(count) || count < least) throw new NativeParseError(FORMAT$12, sections.get(name)?.line ?? 1, `${name} must be a whole number of at least ${String(least)}, found ${String(count)}`);
|
|
1774
|
+
return count;
|
|
1775
|
+
}
|
|
1776
|
+
/**
|
|
1777
|
+
* Reads one cell of a row-major matrix section.
|
|
1778
|
+
* @param values - The section's numbers.
|
|
1779
|
+
* @param columns - The matrix's width.
|
|
1780
|
+
* @param row - The row.
|
|
1781
|
+
* @param column - The column.
|
|
1782
|
+
* @returns The cell.
|
|
1783
|
+
* @throws NativeParseError when the cell is outside the section, which `sized` has already ruled out.
|
|
1784
|
+
*/
|
|
1785
|
+
function cell(values, columns, row, column) {
|
|
1786
|
+
const value = values[row * columns + column];
|
|
1787
|
+
if (value === void 0) throw new NativeParseError(FORMAT$12, 1, `cell ${String(row)}, ${String(column)} is outside its section`);
|
|
1788
|
+
return value;
|
|
1789
|
+
}
|
|
1790
|
+
/**
|
|
1791
|
+
* Parses a home health care routing and scheduling instance in the keyword text format of Mankowska, Meisel and Bierwirth (2014), as the OVIG generator writes it: the counts of nodes (`nbNodes`, the patients and two copies of the depot, the first and the last), vehicles (`nbVehi`, the caregivers) and services (`nbServi`); the services each node requires (`r`, a node by service matrix of zeros and ones), the double service nodes (`DS`, node numbers from one), the services each caregiver can give (`a`), the positions (`x`, `y`), the travel matrix (`d`), the processing time of each service at each node by each caregiver (`p`, a row per node and caregiver), the least and most time between a double service's two services (`mind`, `maxd`), and the time windows (`e`, `l`). A node's id is its number from zero. The depot copies are marked as requiring every service, which is the format's convention and not a visit, so they carry no services or processing times. A patient's required services are their numbers from one, joined by spaces, and its first and second processing time are those of its required services in that order. The travel matrix and the caregivers' qualifications are not node fields and are not kept; the processing times must not differ between caregivers, since a node column holds one value. The gaps are kept for a double service node only. The units of time are the generator's own.
|
|
1792
|
+
* @param text - The file text.
|
|
1793
|
+
* @returns The instance: its patient, caregiver and service kind counts as `meta`, and its nodes in node order.
|
|
1794
|
+
* @throws NativeParseError naming the line for a section that is missing, given twice or of the wrong size, a value that is not a number, a requirement that is not zero or one, a node with more than two required services, a double service node with fewer than two or out of range, or processing times that differ between caregivers.
|
|
1795
|
+
*/
|
|
1796
|
+
function parseMankowska(text) {
|
|
1797
|
+
const sections = sectionsOf$1(text);
|
|
1798
|
+
const nodeCount = countOf(sections, "nbNodes", DEPOTS);
|
|
1799
|
+
const caregivers = countOf(sections, "nbVehi", 1);
|
|
1800
|
+
const serviceKinds = countOf(sections, "nbServi", 1);
|
|
1801
|
+
const required = sized(sections, "r", nodeCount * serviceKinds);
|
|
1802
|
+
sized(sections, "a", caregivers * serviceKinds);
|
|
1803
|
+
const xs = sized(sections, "x", nodeCount);
|
|
1804
|
+
const ys = sized(sections, "y", nodeCount);
|
|
1805
|
+
sized(sections, "d", nodeCount * nodeCount);
|
|
1806
|
+
const times = sized(sections, "p", nodeCount * caregivers * serviceKinds);
|
|
1807
|
+
const least = sized(sections, "mind", nodeCount);
|
|
1808
|
+
const most = sized(sections, "maxd", nodeCount);
|
|
1809
|
+
const earliest = sized(sections, "e", nodeCount);
|
|
1810
|
+
const latest = sized(sections, "l", nodeCount);
|
|
1811
|
+
const line = (name) => sections.get(name)?.line ?? 1;
|
|
1812
|
+
const doubles = new Set((sections.get("DS")?.values ?? []).map((number) => {
|
|
1813
|
+
if (!Number.isInteger(number) || number < 1 || number > nodeCount) throw new NativeParseError(FORMAT$12, line("DS"), `the double service node ${String(number)} is not a node number from 1 to ${String(nodeCount)}`);
|
|
1814
|
+
return number - 1;
|
|
1815
|
+
}));
|
|
1816
|
+
const nodes = Array.from({ length: nodeCount }, (_, node) => {
|
|
1817
|
+
const at = (values) => cell(values, 1, node, 0);
|
|
1818
|
+
if (node === 0 || node === nodeCount - 1) return {
|
|
1819
|
+
id: String(node),
|
|
1820
|
+
role: "depot",
|
|
1821
|
+
x: at(xs),
|
|
1822
|
+
y: at(ys),
|
|
1823
|
+
services: null,
|
|
1824
|
+
duration1: null,
|
|
1825
|
+
duration2: null,
|
|
1826
|
+
earliest: at(earliest),
|
|
1827
|
+
latest: at(latest),
|
|
1828
|
+
minGap: null,
|
|
1829
|
+
maxGap: null
|
|
1830
|
+
};
|
|
1831
|
+
const services = Array.from({ length: serviceKinds }, (__, service) => service).filter((service) => {
|
|
1832
|
+
const flag = cell(required, serviceKinds, node, service);
|
|
1833
|
+
if (flag !== 0 && flag !== 1) throw new NativeParseError(FORMAT$12, line("r"), `node ${String(node)} has the requirement ${String(flag)} for service ${String(service + 1)}, not 0 or 1`);
|
|
1834
|
+
return flag === 1;
|
|
1835
|
+
});
|
|
1836
|
+
if (services.length > MOST_SERVICES || doubles.has(node) && services.length !== MOST_SERVICES) throw new NativeParseError(FORMAT$12, line("r"), `node ${String(node)} requires ${String(services.length)} services${doubles.has(node) ? " but is a double service node" : ""}`);
|
|
1837
|
+
const [first, second] = services.map((service) => {
|
|
1838
|
+
const byCaregiver = Array.from({ length: caregivers }, (__, caregiver) => cell(times, serviceKinds, node * caregivers + caregiver, service));
|
|
1839
|
+
const [time] = byCaregiver;
|
|
1840
|
+
if (time === void 0 || byCaregiver.some((other) => other !== time)) throw new NativeParseError(FORMAT$12, line("p"), `node ${String(node)} has processing times for service ${String(service + 1)} that differ between caregivers (${byCaregiver.join(", ")})`);
|
|
1841
|
+
return time;
|
|
1842
|
+
});
|
|
1843
|
+
return {
|
|
1844
|
+
id: String(node),
|
|
1845
|
+
role: "patient",
|
|
1846
|
+
x: at(xs),
|
|
1847
|
+
y: at(ys),
|
|
1848
|
+
services: services.length === 0 ? null : services.map((service) => String(service + 1)).join(" "),
|
|
1849
|
+
duration1: first ?? null,
|
|
1850
|
+
duration2: second ?? null,
|
|
1851
|
+
earliest: at(earliest),
|
|
1852
|
+
latest: at(latest),
|
|
1853
|
+
minGap: doubles.has(node) ? at(least) : null,
|
|
1854
|
+
maxGap: doubles.has(node) ? at(most) : null
|
|
1855
|
+
};
|
|
1856
|
+
});
|
|
1857
|
+
return {
|
|
1858
|
+
meta: {
|
|
1859
|
+
patients: nodeCount - DEPOTS,
|
|
1860
|
+
caregivers,
|
|
1861
|
+
serviceKinds
|
|
1862
|
+
},
|
|
1863
|
+
nodes
|
|
1864
|
+
};
|
|
1865
|
+
}
|
|
1866
|
+
//#endregion
|
|
1867
|
+
//#region src/parsers/tdc.ts
|
|
1868
|
+
const FORMAT$11 = "tdc";
|
|
1869
|
+
/** A technician line: its id, skills, time window and whether it is an investment option. */
|
|
1870
|
+
const TECHNICIAN = /^Technician id \d+, skills \[[^\]]*\], time window \[[^\]]*\], investment option (?:true|false)$/;
|
|
1871
|
+
/** A task line, with the fields the parser keeps captured: id, skills, duration, window opening and closing, priority, penalty, creation, modification (or `nothing`) and cancellation; its relevant technicians follow and are not kept. */
|
|
1872
|
+
const TASK = /^Task id (\d+), skills \[([^\]]*)\], duration (\d+), time window \[([^,\]]+), ([^\]]+)\], priority (\d+), penalty (-?\d+(?:\.\d+)?), created ([^,]+), modified ([^,]+), cancelled (true|false), relevant technician ids (?:Int64)?\[[\d, ]*\]$/;
|
|
1873
|
+
/** A travel time line, between two tasks or between a technician and a task, in either direction. */
|
|
1874
|
+
const TRAVEL = /^(?:task|technician)\d+(?:task|technician)\d+: -?\d+(?:\.\d+)?$/;
|
|
1875
|
+
/** The section headings, in the order the file gives them. */
|
|
1876
|
+
const HEADINGS = [
|
|
1877
|
+
"Technicians:",
|
|
1878
|
+
"Tasks:",
|
|
1879
|
+
"Travel time:"
|
|
1880
|
+
];
|
|
1881
|
+
/** What a line before any heading and a line of each section is, for errors: indexed by the section's position plus one, so the lines before the first heading are first. */
|
|
1882
|
+
const SECTION_NOUNS = [
|
|
1883
|
+
"heading",
|
|
1884
|
+
"technician",
|
|
1885
|
+
"task",
|
|
1886
|
+
"travel time"
|
|
1887
|
+
];
|
|
1888
|
+
/**
|
|
1889
|
+
* Names what a position's lines are, for an error.
|
|
1890
|
+
* @param position - A section's position plus one.
|
|
1891
|
+
* @returns The noun.
|
|
1892
|
+
*/
|
|
1893
|
+
function nounOf(position) {
|
|
1894
|
+
return SECTION_NOUNS[position] ?? `section ${String(position)}`;
|
|
1895
|
+
}
|
|
1896
|
+
/** Where the task pattern captures each field it keeps. */
|
|
1897
|
+
const TASK_AT = {
|
|
1898
|
+
id: 1,
|
|
1899
|
+
skills: 2,
|
|
1900
|
+
duration: 3,
|
|
1901
|
+
earliest: 4,
|
|
1902
|
+
latest: 5,
|
|
1903
|
+
priority: 6,
|
|
1904
|
+
penalty: 7,
|
|
1905
|
+
created: 8,
|
|
1906
|
+
modified: 9,
|
|
1907
|
+
cancelled: 10
|
|
1908
|
+
};
|
|
1909
|
+
/** How much of a refused line an error shows. */
|
|
1910
|
+
const SHOWN = 120;
|
|
1911
|
+
/**
|
|
1912
|
+
* Parses a technician routing and scheduling instance in the sectioned text format of Gamst and Pisinger (2024): a `Technicians:` heading and a line per technician, a `Tasks:` heading and a line per task (`Task id <n>, skills ["<skill>", ...], duration <minutes>, time window [<opening>, <closing>], priority <n>, penalty <n>, created <time>, modified <time or nothing>, cancelled <true or false>, relevant technician ids [<ids>]`), then a `Travel time:` heading and a line per pair, the travel minutes between two tasks or between a technician and a task. The file is read a line at a time and the travel time lines, almost all of a file, are checked and let go, as the tasks/v1 target has no arcs; a technician's line is counted. A task's id is its id, and a modification of `nothing` is null. Times are as the file writes them.
|
|
1913
|
+
* @param pieces - The file's text, in pieces.
|
|
1914
|
+
* @returns The instance: its technician count as `meta`, and its tasks in file order.
|
|
1915
|
+
* @throws NativeParseError naming the line for a heading out of order or missing, a line that is not of its section's shape, or a task id given twice.
|
|
1916
|
+
*/
|
|
1917
|
+
function parseTdc(pieces) {
|
|
1918
|
+
let section = -1;
|
|
1919
|
+
let technicians = 0;
|
|
1920
|
+
const nodes = [];
|
|
1921
|
+
const ids = /* @__PURE__ */ new Set();
|
|
1922
|
+
forEachLine(pieces, (raw, line) => {
|
|
1923
|
+
const text = raw.trim();
|
|
1924
|
+
if (text === "") return;
|
|
1925
|
+
const heading = HEADINGS.findIndex((candidate) => candidate === text);
|
|
1926
|
+
if (heading !== -1) {
|
|
1927
|
+
if (heading !== section + 1) throw new NativeParseError(FORMAT$11, line, `the heading ${text} is out of order`);
|
|
1928
|
+
section = heading;
|
|
1929
|
+
return;
|
|
1930
|
+
}
|
|
1931
|
+
if (section === 0 && TECHNICIAN.test(text)) {
|
|
1932
|
+
technicians += 1;
|
|
1933
|
+
return;
|
|
1934
|
+
}
|
|
1935
|
+
if (section === 2 && TRAVEL.test(text)) return;
|
|
1936
|
+
const task = section === 1 ? TASK.exec(text) : null;
|
|
1937
|
+
if (task === null) throw new NativeParseError(FORMAT$11, line, `the line is not a ${nounOf(section + 1)} line: ${JSON.stringify(text.slice(0, SHOWN))}`);
|
|
1938
|
+
const field = (index) => {
|
|
1939
|
+
const value = task[index];
|
|
1940
|
+
if (value === void 0) throw new NativeParseError(FORMAT$11, line, `the task line has no field ${String(index)}`);
|
|
1941
|
+
return value;
|
|
1942
|
+
};
|
|
1943
|
+
const id = field(TASK_AT.id);
|
|
1944
|
+
if (ids.has(id)) throw new NativeParseError(FORMAT$11, line, `the task ${id} is given twice`);
|
|
1945
|
+
ids.add(id);
|
|
1946
|
+
const skills = field(TASK_AT.skills).split(",").map((skill) => skill.trim().replace(/^"|"$/g, "")).filter((skill) => skill !== "");
|
|
1947
|
+
const modified = field(TASK_AT.modified);
|
|
1948
|
+
nodes.push({
|
|
1949
|
+
id,
|
|
1950
|
+
skills: skills.length === 0 ? null : skills.join(" "),
|
|
1951
|
+
durationMinutes: Number(field(TASK_AT.duration)),
|
|
1952
|
+
earliest: field(TASK_AT.earliest).trim(),
|
|
1953
|
+
latest: field(TASK_AT.latest).trim(),
|
|
1954
|
+
priority: Number(field(TASK_AT.priority)),
|
|
1955
|
+
penalty: Number(field(TASK_AT.penalty)),
|
|
1956
|
+
created: field(TASK_AT.created),
|
|
1957
|
+
modified: modified === "nothing" ? null : modified,
|
|
1958
|
+
cancelled: field(TASK_AT.cancelled)
|
|
1959
|
+
});
|
|
1960
|
+
});
|
|
1961
|
+
if (section !== HEADINGS.length - 1) throw new NativeParseError(FORMAT$11, 1, `the file has no ${nounOf(section + 2)} heading`);
|
|
1962
|
+
return {
|
|
1963
|
+
meta: { technicians },
|
|
1964
|
+
nodes
|
|
1965
|
+
};
|
|
1966
|
+
}
|
|
1967
|
+
//#endregion
|
|
1968
|
+
//#region src/parsers/verolog-2017.ts
|
|
1969
|
+
const FORMAT$10 = "verolog-2017";
|
|
1970
|
+
/** The header lines, with the key of `meta` each fills and whether its value is text. */
|
|
1971
|
+
const HEADER$3 = /* @__PURE__ */ new Map([
|
|
1972
|
+
["DATASET", ["dataset", "text"]],
|
|
1973
|
+
["NAME", ["name", "text"]],
|
|
1974
|
+
["DAYS", ["days", "number"]],
|
|
1975
|
+
["CAPACITY", ["capacity", "number"]],
|
|
1976
|
+
["MAX_TRIP_DISTANCE", ["maxTripDistance", "number"]],
|
|
1977
|
+
["DEPOT_COORDINATE", ["depotCoordinate", "number"]],
|
|
1978
|
+
["VEHICLE_COST", ["vehicleCost", "number"]],
|
|
1979
|
+
["VEHICLE_DAY_COST", ["vehicleDayCost", "number"]],
|
|
1980
|
+
["DISTANCE_COST", ["distanceCost", "number"]]
|
|
1981
|
+
]);
|
|
1982
|
+
/** The sections that hold rows, and how many numbers each row has. */
|
|
1983
|
+
const SECTION_WIDTHS = {
|
|
1984
|
+
TOOLS: 4,
|
|
1985
|
+
COORDINATES: 3,
|
|
1986
|
+
REQUESTS: 7
|
|
1987
|
+
};
|
|
1988
|
+
/** Where a request row keeps its fields. */
|
|
1989
|
+
const REQUEST_AT$1 = {
|
|
1990
|
+
id: 0,
|
|
1991
|
+
location: 1,
|
|
1992
|
+
fromDay: 2,
|
|
1993
|
+
toDay: 3,
|
|
1994
|
+
stayDays: 4,
|
|
1995
|
+
tool: 5,
|
|
1996
|
+
amount: 6
|
|
1997
|
+
};
|
|
1998
|
+
const WHOLE$3 = /^-?\d+$/;
|
|
1999
|
+
/**
|
|
2000
|
+
* Reads a field of a row.
|
|
2001
|
+
* @param row - The row.
|
|
2002
|
+
* @param index - The field's position.
|
|
2003
|
+
* @returns The number.
|
|
2004
|
+
* @throws NativeParseError when the row has no such field.
|
|
2005
|
+
*/
|
|
2006
|
+
function at(row, index) {
|
|
2007
|
+
const value = row.values[index];
|
|
2008
|
+
if (value === void 0) throw new NativeParseError(FORMAT$10, row.line, `the row has no field ${String(index + 1)}`);
|
|
2009
|
+
return value;
|
|
2010
|
+
}
|
|
2011
|
+
/**
|
|
2012
|
+
* Parses an instance of the ORTEC VeRoLog Solver Challenge 2016 to 2017 (Kheiri et al. 2019): `KEY = value` header lines (the dataset and instance names, the planning days, the vehicle capacity, the longest trip, the depot's coordinate id and the three costs), then three sections each opened by `NAME = <count>` and followed by that many rows of whole numbers: `TOOLS` (id, size, how many are available, cost a day), `COORDINATES` (id, x, y) and `REQUESTS` (id, location, first day, last day, days the tools stay, tool kind, how many). Fields are separated by runs of spaces or tabs, and blank lines are skipped. The nodes are the depot, at its coordinate, with the id `depot`, then one node per request under the request's id at the position of its location. The tool table's sizes, availability and costs are not node fields and are not kept; its row count is.
|
|
2013
|
+
* @param text - The file text.
|
|
2014
|
+
* @returns The instance: its header values and tool count as `meta`, and its nodes, the depot first and the requests in file order.
|
|
2015
|
+
* @throws NativeParseError naming the line for an unknown or repeated key, a header value that is not a number where one is due, a missing header line or section, a section with a different number of rows than it says, a row that is not its section's count of whole numbers, a coordinate given twice, a request at a location the file does not give, or a request id given twice.
|
|
2016
|
+
*/
|
|
2017
|
+
function parseVerolog2017(text) {
|
|
2018
|
+
const meta = /* @__PURE__ */ new Map();
|
|
2019
|
+
const sections = /* @__PURE__ */ new Map();
|
|
2020
|
+
let open;
|
|
2021
|
+
const close = () => {
|
|
2022
|
+
if (open !== void 0 && open.rows.length !== open.count) throw new NativeParseError(FORMAT$10, open.line, `the ${open.name} section says ${String(open.count)} rows and has ${String(open.rows.length)}`);
|
|
2023
|
+
open = void 0;
|
|
2024
|
+
};
|
|
2025
|
+
for (const [index, raw] of lines(text).entries()) {
|
|
2026
|
+
const line = index + 1;
|
|
2027
|
+
const trimmed = raw.trim();
|
|
2028
|
+
if (trimmed === "") continue;
|
|
2029
|
+
const keyed = /^([A-Z_]+)\s*=\s*(.*)$/.exec(trimmed);
|
|
2030
|
+
if (keyed?.[1] !== void 0 && keyed[2] !== void 0) {
|
|
2031
|
+
close();
|
|
2032
|
+
const [, key, value] = keyed;
|
|
2033
|
+
const section = Object.keys(SECTION_WIDTHS).find((name) => name === key);
|
|
2034
|
+
if (section !== void 0) {
|
|
2035
|
+
if (sections.has(section) || !WHOLE$3.test(value)) throw new NativeParseError(FORMAT$10, line, sections.has(section) ? `the ${section} section is given twice` : `the ${section} section's count ${JSON.stringify(value)} is not a whole number`);
|
|
2036
|
+
const rows = [];
|
|
2037
|
+
sections.set(section, rows);
|
|
2038
|
+
open = {
|
|
2039
|
+
name: section,
|
|
2040
|
+
rows,
|
|
2041
|
+
count: Number(value),
|
|
2042
|
+
line
|
|
2043
|
+
};
|
|
2044
|
+
continue;
|
|
2045
|
+
}
|
|
2046
|
+
const header = HEADER$3.get(key);
|
|
2047
|
+
if (header === void 0 || meta.has(header[0])) throw new NativeParseError(FORMAT$10, line, header === void 0 ? `the key ${key} is not one this format has` : `the key ${key} is given twice`);
|
|
2048
|
+
const [name, kind] = header;
|
|
2049
|
+
if (kind === "number" && !/^-?\d+(?:\.\d+)?$/.test(value)) throw new NativeParseError(FORMAT$10, line, `${key} must be a number, found ${JSON.stringify(value)}`);
|
|
2050
|
+
meta.set(name, kind === "number" ? Number(value) : value);
|
|
2051
|
+
continue;
|
|
2052
|
+
}
|
|
2053
|
+
const tokens = trimmed.split(/\s+/);
|
|
2054
|
+
if (open === void 0 || tokens.length !== SECTION_WIDTHS[open.name] || !tokens.every((token) => WHOLE$3.test(token))) throw new NativeParseError(FORMAT$10, line, open === void 0 ? `the row ${JSON.stringify(trimmed)} is outside any section` : `a ${open.name} row is ${String(SECTION_WIDTHS[open.name])} whole numbers, found ${JSON.stringify(trimmed)}`);
|
|
2055
|
+
open.rows.push({
|
|
2056
|
+
line,
|
|
2057
|
+
values: tokens.map(Number)
|
|
2058
|
+
});
|
|
2059
|
+
}
|
|
2060
|
+
close();
|
|
2061
|
+
for (const [key, [name]] of HEADER$3) if (!meta.has(name)) throw new NativeParseError(FORMAT$10, 1, `the file has no ${key} line`);
|
|
2062
|
+
const rowsOf = (name) => {
|
|
2063
|
+
const rows = sections.get(name);
|
|
2064
|
+
if (rows === void 0) throw new NativeParseError(FORMAT$10, 1, `the file has no ${name} section`);
|
|
2065
|
+
return rows;
|
|
2066
|
+
};
|
|
2067
|
+
const tools = rowsOf("TOOLS").length;
|
|
2068
|
+
const positions = /* @__PURE__ */ new Map();
|
|
2069
|
+
for (const row of rowsOf("COORDINATES")) {
|
|
2070
|
+
const id = at(row, 0);
|
|
2071
|
+
if (positions.has(id)) throw new NativeParseError(FORMAT$10, row.line, `the coordinate ${String(id)} is given twice`);
|
|
2072
|
+
positions.set(id, [at(row, 1), at(row, 2)]);
|
|
2073
|
+
}
|
|
2074
|
+
const positionOf = (location, line) => {
|
|
2075
|
+
const position = positions.get(location);
|
|
2076
|
+
if (position === void 0) throw new NativeParseError(FORMAT$10, line, `the location ${String(location)} is not among the coordinates`);
|
|
2077
|
+
return position;
|
|
2078
|
+
};
|
|
2079
|
+
const depot = Number(meta.get("depotCoordinate"));
|
|
2080
|
+
const [depotX, depotY] = positionOf(depot, 1);
|
|
2081
|
+
const nodes = [{
|
|
2082
|
+
id: "depot",
|
|
2083
|
+
x: depotX,
|
|
2084
|
+
y: depotY,
|
|
2085
|
+
kind: "depot",
|
|
2086
|
+
location: depot,
|
|
2087
|
+
fromDay: null,
|
|
2088
|
+
toDay: null,
|
|
2089
|
+
stayDays: null,
|
|
2090
|
+
tool: null,
|
|
2091
|
+
amount: null
|
|
2092
|
+
}];
|
|
2093
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2094
|
+
for (const row of rowsOf("REQUESTS")) {
|
|
2095
|
+
const id = String(at(row, REQUEST_AT$1.id));
|
|
2096
|
+
if (ids.has(id)) throw new NativeParseError(FORMAT$10, row.line, `the request ${id} is given twice`);
|
|
2097
|
+
ids.add(id);
|
|
2098
|
+
const location = at(row, REQUEST_AT$1.location);
|
|
2099
|
+
const [x, y] = positionOf(location, row.line);
|
|
2100
|
+
nodes.push({
|
|
2101
|
+
id,
|
|
2102
|
+
x,
|
|
2103
|
+
y,
|
|
2104
|
+
kind: "request",
|
|
2105
|
+
location,
|
|
2106
|
+
fromDay: at(row, REQUEST_AT$1.fromDay),
|
|
2107
|
+
toDay: at(row, REQUEST_AT$1.toDay),
|
|
2108
|
+
stayDays: at(row, REQUEST_AT$1.stayDays),
|
|
2109
|
+
tool: String(at(row, REQUEST_AT$1.tool)),
|
|
2110
|
+
amount: at(row, REQUEST_AT$1.amount)
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
return {
|
|
2114
|
+
meta: {
|
|
2115
|
+
...Object.fromEntries(meta),
|
|
2116
|
+
tools
|
|
2117
|
+
},
|
|
2118
|
+
nodes
|
|
2119
|
+
};
|
|
2120
|
+
}
|
|
2121
|
+
//#endregion
|
|
1423
2122
|
//#region src/parsers/pajek.ts
|
|
1424
2123
|
/** Section keywords, lower-cased, mapped to how their lines are read. `*Network` names the network and starts no section. `*Partition` and `*Vector` hold node attributes this version does not carry, so their lines are skipped, as are the `*Vertices` lines inside them. */
|
|
1425
2124
|
const sections = /* @__PURE__ */ new Map([
|
|
@@ -2945,28 +3644,60 @@ const KIBIBYTE = 1024;
|
|
|
2945
3644
|
/** Bytes decoded at a time for a format that is read in pieces. The decoder holds only a piece's text at once, so this bounds what a large table costs beyond its bytes; it is large enough that the per-piece work is small beside the piece. */
|
|
2946
3645
|
const DECODE_PIECE_BYTES = 8 * KIBIBYTE * KIBIBYTE;
|
|
2947
3646
|
/**
|
|
2948
|
-
* Decodes a file's bytes in the recipe's encoding
|
|
2949
|
-
* @param bytes - The unpacked file.
|
|
2950
|
-
* @param parser - The recipe's parser, which names the encoding.
|
|
2951
|
-
* @param pieceBytes - How many bytes to decode at a time.
|
|
2952
|
-
* @returns The text of each piece, in order; a character that spans two pieces is yielded whole with the later one.
|
|
2953
|
-
* @throws NativeParseError when the bytes are not valid in the stated encoding.
|
|
3647
|
+
* Decodes a file's bytes in the recipe's encoding as they arrive in pieces, as strictly as `decode`: a character whose bytes span two pieces is decoded whole with the later one, and bytes that are not valid in the stated encoding fail.
|
|
2954
3648
|
*/
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
3649
|
+
var PieceDecoder = class {
|
|
3650
|
+
parser;
|
|
3651
|
+
utf8;
|
|
3652
|
+
constructor(parser) {
|
|
3653
|
+
this.parser = parser;
|
|
3654
|
+
this.utf8 = parser.encoding === "latin1" ? void 0 : new TextDecoder("utf-8", { fatal: true });
|
|
3655
|
+
}
|
|
3656
|
+
/**
|
|
3657
|
+
* Decodes the next piece.
|
|
3658
|
+
* @param piece - Bytes that follow the pieces decoded before them.
|
|
3659
|
+
* @returns Their text, without a character the piece ends partway through.
|
|
3660
|
+
* @throws NativeParseError when the bytes are not valid in the stated encoding.
|
|
3661
|
+
*/
|
|
3662
|
+
decode(piece) {
|
|
3663
|
+
const { utf8 } = this;
|
|
3664
|
+
return utf8 === void 0 ? decode(piece, this.parser) : this.strictly(() => utf8.decode(piece, { stream: true }));
|
|
3665
|
+
}
|
|
3666
|
+
/**
|
|
3667
|
+
* Ends the file.
|
|
3668
|
+
* @returns The text of any bytes still held.
|
|
3669
|
+
* @throws NativeParseError when the file ends partway through a character.
|
|
3670
|
+
*/
|
|
3671
|
+
end() {
|
|
3672
|
+
const { utf8 } = this;
|
|
3673
|
+
return utf8 === void 0 ? "" : this.strictly(() => utf8.decode());
|
|
3674
|
+
}
|
|
3675
|
+
/**
|
|
3676
|
+
* Runs a utf-8 decode, rethrowing its failure in the package's taxonomy: TextDecoder reports invalid bytes only as a TypeError with no position.
|
|
3677
|
+
* @param run - The decode.
|
|
3678
|
+
* @returns Its text.
|
|
3679
|
+
*/
|
|
3680
|
+
strictly(run) {
|
|
3681
|
+
try {
|
|
3682
|
+
return run();
|
|
3683
|
+
} catch (error) {
|
|
3684
|
+
throw new NativeParseError(this.parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
|
|
2965
3685
|
}
|
|
2966
|
-
} catch (error) {
|
|
2967
|
-
if (error instanceof NativeParseError) throw error;
|
|
2968
|
-
throw new NativeParseError(parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
|
|
2969
3686
|
}
|
|
3687
|
+
};
|
|
3688
|
+
/**
|
|
3689
|
+
* Decodes a file's bytes in the recipe's encoding a piece at a time, as strictly as `decode`, so a file whose text would not fit in one string can still be read.
|
|
3690
|
+
* @param bytes - The unpacked file.
|
|
3691
|
+
* @param parser - The recipe's parser, which names the encoding.
|
|
3692
|
+
* @param pieceBytes - How many bytes to decode at a time.
|
|
3693
|
+
* @returns The text of each piece, in order; a character that spans two pieces is yielded whole with the later one.
|
|
3694
|
+
* @throws NativeParseError when the bytes are not valid in the stated encoding.
|
|
3695
|
+
*/
|
|
3696
|
+
function* decodePieces(bytes, parser, pieceBytes = DECODE_PIECE_BYTES) {
|
|
3697
|
+
const decoder = new PieceDecoder(parser);
|
|
3698
|
+
for (let start = 0; start < bytes.length; start += pieceBytes) yield decoder.decode(bytes.subarray(start, start + pieceBytes));
|
|
3699
|
+
const rest = decoder.end();
|
|
3700
|
+
if (rest !== "") yield rest;
|
|
2970
3701
|
}
|
|
2971
3702
|
function parseNative(bytes, parser) {
|
|
2972
3703
|
if (parser.format === "csv-table") return parseCsvTable(decodePieces(bytes, parser), parser);
|
|
@@ -2977,10 +3708,10 @@ function parseNative(bytes, parser) {
|
|
|
2977
3708
|
split: void 0,
|
|
2978
3709
|
pieces: decodePieces(bytes, parser)
|
|
2979
3710
|
}], parser);
|
|
2980
|
-
if (parser.format === "
|
|
3711
|
+
if (parser.format === "tdc") return parseTdc(decodePieces(bytes, parser));
|
|
3712
|
+
if (parser.format === "edge-list") return parseEdgeList(decodePieces(bytes, parser), parser);
|
|
2981
3713
|
const text = decode(bytes, parser);
|
|
2982
3714
|
if (parser.format === "geojson") return parseGeojson(text, parser);
|
|
2983
|
-
if (parser.format === "edge-list") return parseEdgeList(text, parser);
|
|
2984
3715
|
if (parser.format === "matrix-market") return parseMatrixMarket(text);
|
|
2985
3716
|
if (parser.format === "gml") return parseGml(text);
|
|
2986
3717
|
if (parser.format === "ucinet-dl") return parseUcinetDl(text, parser);
|
|
@@ -2994,6 +3725,9 @@ function parseNative(bytes, parser) {
|
|
|
2994
3725
|
if (parser.format === "verolog-2019") return parseVerolog2019(text);
|
|
2995
3726
|
if (parser.format === "vrp-rep") return parseVrpRep(text);
|
|
2996
3727
|
if (parser.format === "hhcrsp") return parseHhcrsp(text);
|
|
3728
|
+
if (parser.format === "ogb") throw new NativeParseError(parser.format, 1, "the format reads the several files a files archive selects, not one file");
|
|
3729
|
+
if (parser.format === "mankowska") return parseMankowska(text);
|
|
3730
|
+
if (parser.format === "verolog-2017") return parseVerolog2017(text);
|
|
2997
3731
|
return parsePajek(text);
|
|
2998
3732
|
}
|
|
2999
3733
|
/**
|
|
@@ -3019,6 +3753,20 @@ function parseNativeParts(parts, parser) {
|
|
|
3019
3753
|
}
|
|
3020
3754
|
throw new NativeParseError(parser.format, 1, `the ${parser.format} parser reads one file, not named parts`);
|
|
3021
3755
|
}
|
|
3756
|
+
/**
|
|
3757
|
+
* Reads the several files of a multi-file format into the one native document they make together.
|
|
3758
|
+
* @param files - The files a `files` archive selects, each with its path in the archive, in path order.
|
|
3759
|
+
* @param parser - The recipe's parser, which must be a multi-file format.
|
|
3760
|
+
* @returns The native graph.
|
|
3761
|
+
* @throws NativeParseError when a file is not valid in the stated encoding or the files are not in the stated format, or when the parser reads one file and not several.
|
|
3762
|
+
*/
|
|
3763
|
+
function parseNativeFiles(files, parser) {
|
|
3764
|
+
if (parser.format !== "ogb") throw new NativeParseError(parser.format, 1, "the format reads one file, not the several a files archive selects");
|
|
3765
|
+
return parseOgb(files.map((file) => ({
|
|
3766
|
+
path: file.path,
|
|
3767
|
+
text: () => decodePieces(file.bytes, parser)
|
|
3768
|
+
})));
|
|
3769
|
+
}
|
|
3022
3770
|
//#endregion
|
|
3023
3771
|
//#region src/node/bzip2.ts
|
|
3024
3772
|
/** The four bytes every bzip2 stream starts with, as ASCII: `BZh` and the block size digit, `1` to `9`. */
|
|
@@ -4798,18 +5546,20 @@ function unpackShapefile(bytes, member) {
|
|
|
4798
5546
|
};
|
|
4799
5547
|
}
|
|
4800
5548
|
/**
|
|
4801
|
-
* Unpacks downloaded bytes into every file a collection's selector matches, or, with an `xlsx` inner layout, into every sheet of every matched workbook that its `sheets` pattern matches as the sheet's CSV, at the member's path and the sheet's name joined by `/`, members in path order and sheets in the workbook's order.
|
|
5549
|
+
* Unpacks downloaded bytes into every file a collection's selector matches, or, with an `xlsx` inner layout, into every sheet of every matched workbook that its `sheets` pattern matches as the sheet's CSV, at the member's path and the sheet's name joined by `/`, members in path order and sheets in the workbook's order. The files are given one at a time: a zip member is inflated only when it is reached, so a reader that lets each file go before taking the next holds one member's bytes beside the archive, never every member at once.
|
|
4802
5550
|
* @param bytes - The downloaded bytes.
|
|
4803
5551
|
* @param archive - The recipe's archive layout, which selects members with `members`.
|
|
4804
5552
|
* @returns Each file.
|
|
4805
|
-
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind, the selector matches no member, or a member an `xlsx` layout reads is not a workbook or has no sheet the pattern matches.
|
|
5553
|
+
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind, the selector matches no member (at the first file taken), or a member an `xlsx` layout reads is not a workbook or has no sheet the pattern matches (at that member).
|
|
4806
5554
|
*/
|
|
4807
|
-
function unpackMembers(bytes, archive) {
|
|
4808
|
-
const
|
|
4809
|
-
if (archive.inner === void 0) return members;
|
|
4810
|
-
const { sheets } = archive.inner;
|
|
5555
|
+
function* unpackMembers(bytes, archive) {
|
|
5556
|
+
const { sheets } = archive.inner ?? {};
|
|
4811
5557
|
const wanted = sheets === void 0 ? void 0 : new RegExp(`^(?:${sheets})$`, "u");
|
|
4812
|
-
|
|
5558
|
+
for (const member of selectMembers(bytes, archive)) {
|
|
5559
|
+
if (archive.inner === void 0) {
|
|
5560
|
+
yield member;
|
|
5561
|
+
continue;
|
|
5562
|
+
}
|
|
4813
5563
|
let names;
|
|
4814
5564
|
try {
|
|
4815
5565
|
names = sheetNames(member.bytes).filter((sheet) => wanted === void 0 || wanted.test(sheet));
|
|
@@ -4817,24 +5567,24 @@ function unpackMembers(bytes, archive) {
|
|
|
4817
5567
|
throw new ArchiveError(`member ${member.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
4818
5568
|
}
|
|
4819
5569
|
if (names.length === 0) throw new ArchiveError(`member ${member.path} has no sheet matching ${sheets ?? ""}`);
|
|
4820
|
-
|
|
5570
|
+
yield* names.map((sheet) => ({
|
|
4821
5571
|
path: `${member.path}/${sheet}`,
|
|
4822
5572
|
bytes: sheetToCsv(member.bytes, {
|
|
4823
5573
|
kind: "xlsx",
|
|
4824
5574
|
sheet
|
|
4825
5575
|
})
|
|
4826
5576
|
}));
|
|
4827
|
-
}
|
|
5577
|
+
}
|
|
4828
5578
|
}
|
|
4829
5579
|
/**
|
|
4830
|
-
* Unpacks every member a collection's selector matches.
|
|
5580
|
+
* Unpacks every member a collection's selector matches, one at a time.
|
|
4831
5581
|
* @param bytes - The downloaded bytes.
|
|
4832
5582
|
* @param archive - The recipe's archive layout.
|
|
4833
5583
|
* @returns Each matching member, in path order.
|
|
4834
|
-
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member.
|
|
5584
|
+
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or the selector matches no member, at the first member taken.
|
|
4835
5585
|
*/
|
|
4836
|
-
function selectMembers(bytes, archive) {
|
|
4837
|
-
|
|
5586
|
+
function* selectMembers(bytes, archive) {
|
|
5587
|
+
yield* readMembers(bytes, archive.kind, (paths) => pickAll(paths, archive.members).map((path) => ({ path })));
|
|
4838
5588
|
}
|
|
4839
5589
|
/**
|
|
4840
5590
|
* Unpacks downloaded bytes into the files a parts archive names, each the one member its selector matches.
|
|
@@ -4844,57 +5594,45 @@ function selectMembers(bytes, archive) {
|
|
|
4844
5594
|
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or a part's selector does not match exactly one member.
|
|
4845
5595
|
*/
|
|
4846
5596
|
function unpackParts(bytes, archive) {
|
|
4847
|
-
return readMembers(bytes, archive.kind, (paths) => archive.parts.map((part) => ({
|
|
5597
|
+
return [...readMembers(bytes, archive.kind, (paths) => archive.parts.map((part) => ({
|
|
4848
5598
|
name: part.name,
|
|
4849
5599
|
path: pick(paths, part.member)
|
|
4850
|
-
})));
|
|
5600
|
+
})))];
|
|
4851
5601
|
}
|
|
4852
5602
|
/**
|
|
4853
|
-
* Reads the members of a file archive that a choice makes from its member paths,
|
|
4854
|
-
* @param bytes - The
|
|
5603
|
+
* Reads the members of a file archive that a choice makes from its member paths, one at a time: the archive is listed once and a zip member is inflated only when it is reached, so a reader that lets each member go before taking the next holds one member's bytes beside the archive.
|
|
5604
|
+
* @param bytes - The archive's bytes.
|
|
4855
5605
|
* @param kind - The archive kind.
|
|
4856
5606
|
* @param choose - Given every member path, the members to read, in the order wanted, each naming its path.
|
|
4857
5607
|
* @returns Each choice with the bytes of the member it names, in the order chosen.
|
|
4858
5608
|
* @throws ArchiveError when the bytes are not a well-formed archive of the stated kind or a chosen member cannot be read; whatever `choose` throws.
|
|
4859
5609
|
*/
|
|
4860
|
-
function readMembers(bytes, kind, choose) {
|
|
5610
|
+
function* readMembers(bytes, kind, choose) {
|
|
4861
5611
|
if (kind === "zip") {
|
|
4862
5612
|
const names = [];
|
|
4863
5613
|
unzip(bytes, (file) => {
|
|
4864
5614
|
names.push(file.name);
|
|
4865
5615
|
return false;
|
|
4866
5616
|
});
|
|
4867
|
-
const
|
|
4868
|
-
|
|
4869
|
-
const files = unzip(bytes, (candidate) => chosen.has(candidate.name));
|
|
4870
|
-
return choices.map((choice) => {
|
|
4871
|
-
const file = getOwn(files, choice.path);
|
|
5617
|
+
for (const choice of choose(names)) {
|
|
5618
|
+
const file = getOwn(unzip(bytes, (candidate) => candidate.name === choice.path), choice.path);
|
|
4872
5619
|
if (file === void 0) throw new ArchiveError(`zip member ${choice.path} could not be read`);
|
|
4873
|
-
|
|
5620
|
+
yield {
|
|
4874
5621
|
...choice,
|
|
4875
5622
|
bytes: file
|
|
4876
5623
|
};
|
|
4877
|
-
}
|
|
5624
|
+
}
|
|
5625
|
+
return;
|
|
4878
5626
|
}
|
|
4879
5627
|
const members = readTar(untarred(bytes, kind));
|
|
4880
|
-
|
|
5628
|
+
for (const choice of choose(members.map((member) => member.path))) {
|
|
4881
5629
|
const found = members.find((member) => member.path === choice.path);
|
|
4882
5630
|
if (found === void 0) throw new ArchiveError(`tar member ${choice.path} could not be read`);
|
|
4883
|
-
|
|
5631
|
+
yield {
|
|
4884
5632
|
...choice,
|
|
4885
5633
|
bytes: found.data
|
|
4886
5634
|
};
|
|
4887
|
-
}
|
|
4888
|
-
}
|
|
4889
|
-
/**
|
|
4890
|
-
* Removes the compression a tar archive kind states, leaving the tar bytes.
|
|
4891
|
-
* @param bytes - The downloaded bytes.
|
|
4892
|
-
* @param kind - The tar archive kind: `tar` is uncompressed, `tar.gz` is gzip and `tar.bz2` is bzip2.
|
|
4893
|
-
* @returns The tar archive.
|
|
4894
|
-
*/
|
|
4895
|
-
function untarred(bytes, kind) {
|
|
4896
|
-
if (kind === "tar") return bytes;
|
|
4897
|
-
return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
|
|
5635
|
+
}
|
|
4898
5636
|
}
|
|
4899
5637
|
/**
|
|
4900
5638
|
* Reads one sheet of a spreadsheet as CSV text, by the spreadsheet's format.
|
|
@@ -4912,6 +5650,41 @@ function sheetToCsv(bytes, archive) {
|
|
|
4912
5650
|
return gridToCsv(read(bytes, archive.sheet));
|
|
4913
5651
|
}
|
|
4914
5652
|
/**
|
|
5653
|
+
* Unpacks downloaded bytes into every file a multi-file format's selector matches, gunzipping each when the layout's `inner` says so. The paths stay the archive's own, so a gunzipped `edge.csv.gz` keeps its name.
|
|
5654
|
+
* @param bytes - The downloaded bytes.
|
|
5655
|
+
* @param archive - The recipe's archive layout, which selects files with `files`.
|
|
5656
|
+
* @returns Each matching file, in path order.
|
|
5657
|
+
* @throws ArchiveError when the bytes, or a file `inner` gunzips, are not well-formed, or the selector matches no member.
|
|
5658
|
+
*/
|
|
5659
|
+
function unpackFiles(bytes, archive) {
|
|
5660
|
+
const files = [...unpackMembers(bytes, {
|
|
5661
|
+
kind: archive.kind,
|
|
5662
|
+
members: archive.files
|
|
5663
|
+
})];
|
|
5664
|
+
if (archive.inner === void 0) return files;
|
|
5665
|
+
return files.map((file) => {
|
|
5666
|
+
try {
|
|
5667
|
+
return {
|
|
5668
|
+
path: file.path,
|
|
5669
|
+
bytes: gunzip(file.bytes)
|
|
5670
|
+
};
|
|
5671
|
+
} catch (error) {
|
|
5672
|
+
throw new ArchiveError(`member ${file.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5673
|
+
}
|
|
5674
|
+
});
|
|
5675
|
+
}
|
|
5676
|
+
/**
|
|
5677
|
+
* Unpacks a tar archive's bytes, which may be compressed: `tar.gz` is gunzipped and `tar.bz2` bunzipped first, `tar` read as it is.
|
|
5678
|
+
* @param bytes - The archive's bytes.
|
|
5679
|
+
* @param kind - The archive kind.
|
|
5680
|
+
* @returns The tar's bytes.
|
|
5681
|
+
* @throws ArchiveError when the bytes are not valid gzip or bzip2 data, as `gunzip` and `bunzip2` report.
|
|
5682
|
+
*/
|
|
5683
|
+
function untarred(bytes, kind) {
|
|
5684
|
+
if (kind === "tar") return bytes;
|
|
5685
|
+
return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
|
|
5686
|
+
}
|
|
5687
|
+
/**
|
|
4915
5688
|
* Gunzips bytes, rethrowing zlib's failure in the package's taxonomy.
|
|
4916
5689
|
* @param bytes - Gzip data.
|
|
4917
5690
|
* @returns The decompressed bytes.
|
|
@@ -5177,15 +5950,25 @@ async function getBytes(port, url, signal) {
|
|
|
5177
5950
|
return new Uint8Array(await response.arrayBuffer());
|
|
5178
5951
|
}
|
|
5179
5952
|
/**
|
|
5180
|
-
* Reads a source's pages, each starting where the rows
|
|
5953
|
+
* Reads a source's pages in order, each starting where the rows read so far end, and checks each holds exactly the rows it should, so a response cut short is an error and not a table with rows missing. Only the page being read and the first page's header are held, so a caller that hands each page on holds no more of the source than that.
|
|
5181
5954
|
* @param read - The source read.
|
|
5182
|
-
* @
|
|
5183
|
-
* @
|
|
5184
|
-
* @returns Every page.
|
|
5185
|
-
* @throws KeepsetError when a page holds a different number of rows than the count says it should, or its header differs from the first page's.
|
|
5955
|
+
* @returns Each page, the first with its header record and every later one without it.
|
|
5956
|
+
* @throws KeepsetError when the source has no rows, a page holds a different number of rows than the count says it should, or its header differs from the first page's.
|
|
5186
5957
|
*/
|
|
5187
|
-
async function
|
|
5188
|
-
if (
|
|
5958
|
+
async function* resourcePages(read) {
|
|
5959
|
+
if (read.count === 0) throw new KeepsetError(`${read.resource} has no rows`);
|
|
5960
|
+
yield* pagesFrom(read, 0, void 0);
|
|
5961
|
+
}
|
|
5962
|
+
/**
|
|
5963
|
+
* Reads one page and then, by recursion, the pages after it.
|
|
5964
|
+
* @param read - The source read.
|
|
5965
|
+
* @param offset - The row the page starts at.
|
|
5966
|
+
* @param header - The first page's header record, or undefined when this is the first page.
|
|
5967
|
+
* @returns This page and every page after it.
|
|
5968
|
+
* @throws KeepsetError as `resourcePages` does.
|
|
5969
|
+
*/
|
|
5970
|
+
async function* pagesFrom(read, offset, header) {
|
|
5971
|
+
if (offset >= read.count) return;
|
|
5189
5972
|
const expected = Math.min(read.pageSize, read.count - offset);
|
|
5190
5973
|
const page = await withAttempts(async () => {
|
|
5191
5974
|
const bytes = await getBytes(read.port, read.pageUrl(offset), read.signal);
|
|
@@ -5193,10 +5976,15 @@ async function readResourcePages(read, held, offset) {
|
|
|
5193
5976
|
if (rows !== expected) throw new KeepsetError(`${read.resource} gave ${String(rows)} rows at offset ${String(offset)} where ${String(expected)} of its ${String(read.count)} were due, so a response was cut short or the table changed during the download`);
|
|
5194
5977
|
return bytes;
|
|
5195
5978
|
}, read.signal);
|
|
5196
|
-
const first = held[0];
|
|
5197
5979
|
const headerEnd = page.indexOf(LINE_FEED) + 1;
|
|
5198
|
-
if (
|
|
5199
|
-
|
|
5980
|
+
if (header === void 0) {
|
|
5981
|
+
yield page;
|
|
5982
|
+
yield* pagesFrom(read, offset + expected, page.slice(0, headerEnd));
|
|
5983
|
+
return;
|
|
5984
|
+
}
|
|
5985
|
+
if (headerEnd !== header.length || !page.subarray(0, headerEnd).every((byte, index) => byte === header[index])) throw new KeepsetError(`${read.resource} gave a different header at offset ${String(offset)} than at the start`);
|
|
5986
|
+
yield page.subarray(headerEnd);
|
|
5987
|
+
yield* pagesFrom(read, offset + expected, header);
|
|
5200
5988
|
}
|
|
5201
5989
|
/**
|
|
5202
5990
|
* Downloads a CSV source a page at a time and joins the pages under the first header.
|
|
@@ -5205,42 +5993,44 @@ async function readResourcePages(read, held, offset) {
|
|
|
5205
5993
|
* @throws KeepsetError when the source has no rows or a page is not what it should be.
|
|
5206
5994
|
*/
|
|
5207
5995
|
async function downloadPages(read) {
|
|
5208
|
-
const pages =
|
|
5209
|
-
|
|
5996
|
+
const pages = [];
|
|
5997
|
+
for await (const page of resourcePages(read)) pages.push(page);
|
|
5210
5998
|
return Buffer.concat(pages);
|
|
5211
5999
|
}
|
|
5212
6000
|
/**
|
|
5213
|
-
*
|
|
6001
|
+
* How a Socrata resource is read: its row count is read first, then pages of `pageSize` rows in the order of the rows' ids, with only the `select` columns when the recipe lists them, so the columns a parser drops are never transferred.
|
|
5214
6002
|
* @param resource - The resource's URL, such as `https://data.example.org/resource/abcd-1234.csv`, without a query.
|
|
5215
|
-
* @param
|
|
6003
|
+
* @param paging - The recipe's paging.
|
|
5216
6004
|
* @param port - The network port.
|
|
5217
6005
|
* @param signal - Aborts the download.
|
|
5218
|
-
* @returns The
|
|
5219
|
-
* @throws KeepsetError when the resource does not report a count
|
|
6006
|
+
* @returns The source read.
|
|
6007
|
+
* @throws KeepsetError when the resource does not report a count.
|
|
5220
6008
|
*/
|
|
5221
|
-
async function
|
|
6009
|
+
async function socrataRead(resource, paging, port, signal) {
|
|
5222
6010
|
const reported = /^"count"\r?\n"(\d+)"/.exec(new TextDecoder("utf-8").decode(await getBytes(port, `${resource}?%24select=count(*)`, signal)));
|
|
5223
6011
|
if (reported?.[1] === void 0) throw new KeepsetError(`${resource} did not report a row count`);
|
|
5224
|
-
|
|
6012
|
+
const { pageSize, select } = paging;
|
|
6013
|
+
const columns = select === void 0 ? "" : `%24select=${encodeURIComponent(select.join(","))}&`;
|
|
6014
|
+
return {
|
|
5225
6015
|
resource,
|
|
5226
|
-
pageUrl: (offset) => `${resource}
|
|
6016
|
+
pageUrl: (offset) => `${resource}?${columns}%24order=%3Aid&%24limit=${String(pageSize)}&%24offset=${String(offset)}`,
|
|
5227
6017
|
pageSize,
|
|
5228
6018
|
count: Number(reported[1]),
|
|
5229
6019
|
port,
|
|
5230
6020
|
signal
|
|
5231
|
-
}
|
|
6021
|
+
};
|
|
5232
6022
|
}
|
|
5233
6023
|
const CountResultSchema = z.object({ result: z.object({ total: z.int().min(0) }) });
|
|
5234
6024
|
/**
|
|
5235
|
-
*
|
|
6025
|
+
* How a CKAN datastore table is read: the row count is read from the portal's datastore API, then pages of `pageSize` rows of the table's CSV dump.
|
|
5236
6026
|
* @param dump - The dump's URL, such as `https://data.example.org/datastore/dump/<resource id>`, without a query; a portal that serves CKAN under a path prefix keeps it before `/datastore/dump/`.
|
|
5237
6027
|
* @param pageSize - How many rows a page holds.
|
|
5238
6028
|
* @param port - The network port.
|
|
5239
6029
|
* @param signal - Aborts the download.
|
|
5240
|
-
* @returns The
|
|
5241
|
-
* @throws KeepsetError when the URL is not a datastore dump
|
|
6030
|
+
* @returns The source read.
|
|
6031
|
+
* @throws KeepsetError when the URL is not a datastore dump or the table does not report a count.
|
|
5242
6032
|
*/
|
|
5243
|
-
async function
|
|
6033
|
+
async function ckanRead(dump, pageSize, port, signal) {
|
|
5244
6034
|
const parsed = /^(.*)\/datastore\/dump\/([^/?#]+)$/.exec(dump);
|
|
5245
6035
|
if (parsed?.[1] === void 0 || parsed[2] === void 0) throw new KeepsetError(`${dump} is not a CKAN datastore dump URL`);
|
|
5246
6036
|
const url = `${parsed[1]}/api/3/action/datastore_search?resource_id=${parsed[2]}&limit=0`;
|
|
@@ -5252,26 +6042,26 @@ async function downloadCkanDatastore(dump, pageSize, port, signal) {
|
|
|
5252
6042
|
}
|
|
5253
6043
|
const count = CountResultSchema.safeParse(body);
|
|
5254
6044
|
if (!count.success) throw new KeepsetError(`${dump} did not report a row count`);
|
|
5255
|
-
return
|
|
6045
|
+
return {
|
|
5256
6046
|
resource: dump,
|
|
5257
6047
|
pageUrl: (offset) => `${dump}?format=csv&limit=${String(pageSize)}&offset=${String(offset)}`,
|
|
5258
6048
|
pageSize,
|
|
5259
6049
|
count: count.data.result.total,
|
|
5260
6050
|
port,
|
|
5261
6051
|
signal
|
|
5262
|
-
}
|
|
6052
|
+
};
|
|
5263
6053
|
}
|
|
5264
6054
|
const OpenFemaCountSchema = z.object({ metadata: z.object({ count: z.int().min(0) }) });
|
|
5265
6055
|
/**
|
|
5266
|
-
*
|
|
6056
|
+
* How an OpenFEMA entity, or the rows of it the source URL selects, is read: the row count is read from the API's own metadata, then pages of `pageSize` rows in the order of the entity's id.
|
|
5267
6057
|
* @param entity - The entity's URL with any `$select` or `$filter` but no `$top`, `$skip`, `$orderby` or `$format`, such as `https://www.fema.gov/api/open/v3/NfipClaims?$select=id,state`.
|
|
5268
6058
|
* @param pageSize - How many rows a page holds.
|
|
5269
6059
|
* @param port - The network port.
|
|
5270
6060
|
* @param signal - Aborts the download.
|
|
5271
|
-
* @returns The
|
|
5272
|
-
* @throws KeepsetError when the source URL sets a paging parameter
|
|
6061
|
+
* @returns The source read.
|
|
6062
|
+
* @throws KeepsetError when the source URL sets a paging parameter or the entity does not report a count.
|
|
5273
6063
|
*/
|
|
5274
|
-
async function
|
|
6064
|
+
async function openFemaRead(entity, pageSize, port, signal) {
|
|
5275
6065
|
const base = new URL(entity);
|
|
5276
6066
|
for (const name of [
|
|
5277
6067
|
"$top",
|
|
@@ -5297,7 +6087,7 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
|
|
|
5297
6087
|
}
|
|
5298
6088
|
const count = OpenFemaCountSchema.safeParse(body);
|
|
5299
6089
|
if (!count.success) throw new KeepsetError(`${entity} did not report a row count`);
|
|
5300
|
-
return
|
|
6090
|
+
return {
|
|
5301
6091
|
resource: entity,
|
|
5302
6092
|
pageUrl: (offset) => withParameters({
|
|
5303
6093
|
$format: "csv",
|
|
@@ -5309,10 +6099,34 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
|
|
|
5309
6099
|
count: count.data.metadata.count,
|
|
5310
6100
|
port,
|
|
5311
6101
|
signal
|
|
5312
|
-
}
|
|
6102
|
+
};
|
|
5313
6103
|
}
|
|
5314
6104
|
/**
|
|
5315
|
-
*
|
|
6105
|
+
* How a CSV paged source is read, in the way its recipe's paging says.
|
|
6106
|
+
* @param url - The source's URL.
|
|
6107
|
+
* @param paging - How the source is paged.
|
|
6108
|
+
* @param port - The network port.
|
|
6109
|
+
* @param signal - Aborts the download.
|
|
6110
|
+
* @returns The source read.
|
|
6111
|
+
*/
|
|
6112
|
+
async function csvRead(url, paging, port, signal) {
|
|
6113
|
+
if (paging.kind === "socrata-resource") return socrataRead(url, paging, port, signal);
|
|
6114
|
+
return paging.kind === "ckan-datastore" ? ckanRead(url, paging.pageSize, port, signal) : openFemaRead(url, paging.pageSize, port, signal);
|
|
6115
|
+
}
|
|
6116
|
+
/**
|
|
6117
|
+
* Reads a CSV paged source a page at a time: the first page with its header and every later page without it, so the pages in order are the one CSV `downloadPaged` joins them into. Only the page being read is held.
|
|
6118
|
+
* @param url - The source's URL.
|
|
6119
|
+
* @param paging - How the source is paged.
|
|
6120
|
+
* @param port - The network port.
|
|
6121
|
+
* @param signal - Aborts the download.
|
|
6122
|
+
* @returns Each page's bytes, in order.
|
|
6123
|
+
* @throws KeepsetError when the source does not report a count, has no rows, or a page is not what it should be.
|
|
6124
|
+
*/
|
|
6125
|
+
async function* csvPages(url, paging, port, signal) {
|
|
6126
|
+
yield* resourcePages(await csvRead(url, paging, port, signal));
|
|
6127
|
+
}
|
|
6128
|
+
/**
|
|
6129
|
+
* Downloads a paged source as the one file its pages make, in the way its recipe's paging says: an ArcGIS layer as one GeoJSON feature collection, any other source as one CSV of its pages joined under the first page's header.
|
|
5316
6130
|
* @param url - The source's URL.
|
|
5317
6131
|
* @param paging - How the source is paged.
|
|
5318
6132
|
* @param port - The network port.
|
|
@@ -5322,17 +6136,16 @@ async function downloadOpenFema(entity, pageSize, port, signal) {
|
|
|
5322
6136
|
*/
|
|
5323
6137
|
async function downloadPaged(url, paging, port, signal) {
|
|
5324
6138
|
if (paging.kind === "arcgis-query") return downloadArcgisLayer(url, paging.orderBy, port, signal);
|
|
5325
|
-
|
|
5326
|
-
return paging.kind === "ckan-datastore" ? downloadCkanDatastore(url, paging.pageSize, port, signal) : downloadOpenFema(url, paging.pageSize, port, signal);
|
|
6139
|
+
return downloadPages(await csvRead(url, paging, port, signal));
|
|
5327
6140
|
}
|
|
5328
6141
|
//#endregion
|
|
5329
|
-
//#region src/node/
|
|
6142
|
+
//#region src/node/recipe-run.ts
|
|
5330
6143
|
/**
|
|
5331
|
-
* The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
|
|
6144
|
+
* The URL a recipe downloads a source from, as `downloadUrl` gives it. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
|
|
5332
6145
|
* @param source - The source.
|
|
5333
6146
|
* @returns The URL to request.
|
|
5334
6147
|
*/
|
|
5335
|
-
function
|
|
6148
|
+
function sourceUrl(source) {
|
|
5336
6149
|
if (source.status === "dead") {
|
|
5337
6150
|
if (source.snapshot === void 0) throw new KeepsetError(`dead source ${source.url} has no snapshot`);
|
|
5338
6151
|
return `https://web.archive.org/web/${source.snapshot.timestamp}id_/${source.url}`;
|
|
@@ -5340,6 +6153,18 @@ function downloadUrl(source) {
|
|
|
5340
6153
|
return source.url;
|
|
5341
6154
|
}
|
|
5342
6155
|
/**
|
|
6156
|
+
* The source record a provenance carries: the entry's source as recorded.
|
|
6157
|
+
* @param source - The source.
|
|
6158
|
+
* @returns Its URL, status and any snapshot.
|
|
6159
|
+
*/
|
|
6160
|
+
function sourceRecord(source) {
|
|
6161
|
+
return {
|
|
6162
|
+
url: source.url,
|
|
6163
|
+
status: source.status,
|
|
6164
|
+
...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
|
|
6165
|
+
};
|
|
6166
|
+
}
|
|
6167
|
+
/**
|
|
5343
6168
|
* Runs one step, rethrowing any failure as a RecipeError naming the entry and the step, with the original error as its cause.
|
|
5344
6169
|
* @param entryId - The entry.
|
|
5345
6170
|
* @param name - The step.
|
|
@@ -5366,6 +6191,373 @@ function metaOf(entry) {
|
|
|
5366
6191
|
...entry.properties
|
|
5367
6192
|
};
|
|
5368
6193
|
}
|
|
6194
|
+
//#endregion
|
|
6195
|
+
//#region src/node/id-set.ts
|
|
6196
|
+
/** The first table size, in slots: 2 to the 16th, a power of two, so a hash is reduced to a slot by masking (and every doubling keeps it one). */
|
|
6197
|
+
const INITIAL_SLOTS = 65536;
|
|
6198
|
+
/** The first arena size, in bytes: a mebibyte, which a stream of a few tens of thousands of short ids fills before the arena first doubles. */
|
|
6199
|
+
const INITIAL_ARENA_BYTES = 1048576;
|
|
6200
|
+
/** The table grows when more than this share of its slots hold an id, which keeps linear probing short. */
|
|
6201
|
+
const MAX_LOAD = .7;
|
|
6202
|
+
/** The bytes before each id in the arena: its length, as an unsigned 32-bit integer. */
|
|
6203
|
+
const LENGTH_BYTES = 4;
|
|
6204
|
+
/** The largest arena, 2 to the 31st less one: slots hold an arena offset plus one in a signed 32-bit integer. */
|
|
6205
|
+
const MAX_ARENA_BYTES = 2147483647;
|
|
6206
|
+
/** The FNV-1a 32-bit offset basis and prime (http://www.isthe.com/chongo/tech/comp/fnv/). */
|
|
6207
|
+
const FNV_OFFSET = 2166136261;
|
|
6208
|
+
const FNV_PRIME = 16777619;
|
|
6209
|
+
const encoder = new TextEncoder();
|
|
6210
|
+
/**
|
|
6211
|
+
* The FNV-1a hash of bytes.
|
|
6212
|
+
* @param bytes - The bytes.
|
|
6213
|
+
* @returns The 32-bit hash.
|
|
6214
|
+
*/
|
|
6215
|
+
function fnv1a(bytes) {
|
|
6216
|
+
let hash = FNV_OFFSET;
|
|
6217
|
+
for (const byte of bytes) hash = Math.imul(hash ^ byte, FNV_PRIME);
|
|
6218
|
+
return hash >>> 0;
|
|
6219
|
+
}
|
|
6220
|
+
/**
|
|
6221
|
+
* Reads a slot of a table.
|
|
6222
|
+
* @param slots - The table.
|
|
6223
|
+
* @param slot - A slot index, always masked into the table's range.
|
|
6224
|
+
* @returns What the slot holds.
|
|
6225
|
+
* @throws RangeError when the index is outside the table, which masking never gives.
|
|
6226
|
+
*/
|
|
6227
|
+
function slotOf(slots, slot) {
|
|
6228
|
+
const held = slots[slot];
|
|
6229
|
+
if (held === void 0) throw new RangeError(`slot ${String(slot)} is outside a table of ${String(slots.length)}`);
|
|
6230
|
+
return held;
|
|
6231
|
+
}
|
|
6232
|
+
/**
|
|
6233
|
+
* A set of ids that holds each id as its UTF-8 bytes in one growing arena, indexed by an open-addressed table of arena offsets, so a stream of millions of ids is checked for repeats exactly while holding a few bytes beyond each id's own and no JavaScript string or object per id. Membership compares the bytes themselves, so two different ids are never taken for one.
|
|
6234
|
+
*/
|
|
6235
|
+
var IdSet = class {
|
|
6236
|
+
arena = new Uint8Array(INITIAL_ARENA_BYTES);
|
|
6237
|
+
view = new DataView(this.arena.buffer);
|
|
6238
|
+
used = 0;
|
|
6239
|
+
/** Each slot holds an id's arena offset plus one, or zero when empty. */
|
|
6240
|
+
slots = new Int32Array(INITIAL_SLOTS);
|
|
6241
|
+
count = 0;
|
|
6242
|
+
/** @returns How many ids the set holds. */
|
|
6243
|
+
get size() {
|
|
6244
|
+
return this.count;
|
|
6245
|
+
}
|
|
6246
|
+
/**
|
|
6247
|
+
* Adds an id.
|
|
6248
|
+
* @param id - The id.
|
|
6249
|
+
* @returns True when the id was new, false when the set already held it.
|
|
6250
|
+
* @throws KeepsetError when the ids held would pass the largest arena.
|
|
6251
|
+
*/
|
|
6252
|
+
add(id) {
|
|
6253
|
+
const bytes = encoder.encode(id);
|
|
6254
|
+
const hash = fnv1a(bytes);
|
|
6255
|
+
const mask = this.slots.length - 1;
|
|
6256
|
+
for (let slot = hash & mask;; slot = slot + 1 & mask) {
|
|
6257
|
+
const held = slotOf(this.slots, slot);
|
|
6258
|
+
if (held === 0) {
|
|
6259
|
+
this.slots[slot] = this.store(bytes) + 1;
|
|
6260
|
+
this.count += 1;
|
|
6261
|
+
if (this.count > this.slots.length * MAX_LOAD) this.grow();
|
|
6262
|
+
return true;
|
|
6263
|
+
}
|
|
6264
|
+
if (this.equals(held - 1, bytes)) return false;
|
|
6265
|
+
}
|
|
6266
|
+
}
|
|
6267
|
+
/**
|
|
6268
|
+
* Whether the id stored at an offset is these bytes.
|
|
6269
|
+
* @param offset - The id's arena offset.
|
|
6270
|
+
* @param bytes - The bytes.
|
|
6271
|
+
* @returns True when they are the same.
|
|
6272
|
+
*/
|
|
6273
|
+
equals(offset, bytes) {
|
|
6274
|
+
if (this.lengthAt(offset) !== bytes.length) return false;
|
|
6275
|
+
const start = offset + LENGTH_BYTES;
|
|
6276
|
+
return bytes.every((byte, index) => this.arena[start + index] === byte);
|
|
6277
|
+
}
|
|
6278
|
+
/**
|
|
6279
|
+
* The length of the id stored at an offset.
|
|
6280
|
+
* @param offset - The id's arena offset.
|
|
6281
|
+
* @returns Its byte length.
|
|
6282
|
+
*/
|
|
6283
|
+
lengthAt(offset) {
|
|
6284
|
+
return this.view.getUint32(offset, true);
|
|
6285
|
+
}
|
|
6286
|
+
/**
|
|
6287
|
+
* Appends an id's bytes to the arena.
|
|
6288
|
+
* @param bytes - The bytes.
|
|
6289
|
+
* @returns Their offset.
|
|
6290
|
+
*/
|
|
6291
|
+
store(bytes) {
|
|
6292
|
+
const needed = this.used + LENGTH_BYTES + bytes.length;
|
|
6293
|
+
if (needed > MAX_ARENA_BYTES) throw new KeepsetError(`the ids of the stream pass ${String(MAX_ARENA_BYTES)} bytes, more than one id set holds`);
|
|
6294
|
+
if (needed > this.arena.length) {
|
|
6295
|
+
const grown = new Uint8Array(Math.min(MAX_ARENA_BYTES, Math.max(needed, this.arena.length * 2)));
|
|
6296
|
+
grown.set(this.arena.subarray(0, this.used));
|
|
6297
|
+
this.arena = grown;
|
|
6298
|
+
this.view = new DataView(grown.buffer);
|
|
6299
|
+
}
|
|
6300
|
+
const offset = this.used;
|
|
6301
|
+
this.view.setUint32(offset, bytes.length, true);
|
|
6302
|
+
this.arena.set(bytes, offset + LENGTH_BYTES);
|
|
6303
|
+
this.used = needed;
|
|
6304
|
+
return offset;
|
|
6305
|
+
}
|
|
6306
|
+
/** Doubles the table, placing every id again by its hash, which is recomputed from the arena so the table holds nothing but offsets. */
|
|
6307
|
+
grow() {
|
|
6308
|
+
const slots = new Int32Array(this.slots.length * 2);
|
|
6309
|
+
const mask = slots.length - 1;
|
|
6310
|
+
for (const held of this.slots) if (held !== 0) {
|
|
6311
|
+
const start = held - 1 + LENGTH_BYTES;
|
|
6312
|
+
let slot = fnv1a(this.arena.subarray(start, start + this.lengthAt(held - 1))) & mask;
|
|
6313
|
+
while (slotOf(slots, slot) !== 0) slot = slot + 1 & mask;
|
|
6314
|
+
slots[slot] = held;
|
|
6315
|
+
}
|
|
6316
|
+
this.slots = slots;
|
|
6317
|
+
}
|
|
6318
|
+
};
|
|
6319
|
+
//#endregion
|
|
6320
|
+
//#region src/node/stream-dataset.ts
|
|
6321
|
+
/**
|
|
6322
|
+
* The parts of a recipe a stream reads, checked to be a streamed recipe: the entry schema refuses a `stream` recipe that is not a CSV paged source read by the csv-table parser with no archive, so this only fails for an entry whose recipe does not stream.
|
|
6323
|
+
* @param entry - The entry.
|
|
6324
|
+
* @returns The recipe's source, paging and parser, and the category.
|
|
6325
|
+
* @throws RecipeError when the recipe is not fetchable, does not stream, or its category has no streamed form.
|
|
6326
|
+
*/
|
|
6327
|
+
function streamedRecipe(entry) {
|
|
6328
|
+
const { recipe, category } = entry;
|
|
6329
|
+
if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
|
|
6330
|
+
if ("files" in recipe || recipe.stream !== true) throw new RecipeError(entry.id, "recipe", "the recipe is not streamed, so fetchDataset reads it as one document");
|
|
6331
|
+
const { paging, parser } = recipe;
|
|
6332
|
+
if (paging === void 0 || paging.kind === "arcgis-query" || parser.format !== "csv-table" || !isStreamable(category)) throw new RecipeError(entry.id, "recipe", "a streamed recipe reads a CSV paged source through the csv-table parser, for a category with a streamed form");
|
|
6333
|
+
return {
|
|
6334
|
+
source: recipe.source,
|
|
6335
|
+
paging,
|
|
6336
|
+
parser,
|
|
6337
|
+
category
|
|
6338
|
+
};
|
|
6339
|
+
}
|
|
6340
|
+
/**
|
|
6341
|
+
* Whether a recipe is streamed, so `fetchDataset` refuses it and `streamDataset` reads it.
|
|
6342
|
+
* @param recipe - A recipe.
|
|
6343
|
+
* @returns True for a streamed recipe.
|
|
6344
|
+
*/
|
|
6345
|
+
function isStreamed(recipe) {
|
|
6346
|
+
return recipe.status === "fetchable" && !("files" in recipe) && recipe.stream === true;
|
|
6347
|
+
}
|
|
6348
|
+
/**
|
|
6349
|
+
* What a crosswalk path addresses, for the separability check: the document's meta, a field of every node, or anything else.
|
|
6350
|
+
* @param path - A crosswalk path.
|
|
6351
|
+
* @returns Its class.
|
|
6352
|
+
*/
|
|
6353
|
+
function pathClass(path) {
|
|
6354
|
+
if (path === "meta" || path.startsWith("meta.")) return "meta";
|
|
6355
|
+
return path === "nodes[]" || path.startsWith("nodes[].") ? "node" : "other";
|
|
6356
|
+
}
|
|
6357
|
+
/**
|
|
6358
|
+
* Why a crosswalk cannot be applied a page of nodes at a time, if it cannot: every rule must read and write the meta alone or one node's fields alone, with no `when`, so encoding a page's nodes under the document's meta gives exactly that page's part of encoding the whole document, and a round trip that holds on every page holds on the whole.
|
|
6359
|
+
* @param crosswalk - The crosswalk.
|
|
6360
|
+
* @returns The first rule that breaks this, described, or undefined when every rule keeps to it.
|
|
6361
|
+
*/
|
|
6362
|
+
function inseparableRule(crosswalk) {
|
|
6363
|
+
for (const [index, rule] of [...crosswalk.rules, ...crosswalk.oneWay].entries()) {
|
|
6364
|
+
if ("when" in rule && rule.when !== void 0) return `rule ${String(index)} (${rule.op}) has a when, which may read the whole document`;
|
|
6365
|
+
const paths = ["from" in rule ? rule.from : void 0, "to" in rule ? rule.to : void 0].filter((path) => path !== void 0);
|
|
6366
|
+
const classes = new Set(paths.map(pathClass));
|
|
6367
|
+
const [only] = classes;
|
|
6368
|
+
if (classes.size !== 1 || only === "other") return `rule ${String(index)} (${rule.op}) reads or writes ${paths.join(" and ")}, not the meta alone or one node's fields alone`;
|
|
6369
|
+
}
|
|
6370
|
+
}
|
|
6371
|
+
/**
|
|
6372
|
+
* Passes on what an async iterable yields, rethrowing its failure as a RecipeError naming the entry and the step, as `step` does for one call.
|
|
6373
|
+
* @param entryId - The entry.
|
|
6374
|
+
* @param name - The step.
|
|
6375
|
+
* @param source - What the step yields.
|
|
6376
|
+
* @returns The same items, in order.
|
|
6377
|
+
*/
|
|
6378
|
+
async function* inStep(entryId, name, source) {
|
|
6379
|
+
try {
|
|
6380
|
+
yield* source;
|
|
6381
|
+
} catch (error) {
|
|
6382
|
+
if (error instanceof RecipeError) throw error;
|
|
6383
|
+
throw new RecipeError(entryId, name, error instanceof Error ? error.message : String(error), { cause: error });
|
|
6384
|
+
}
|
|
6385
|
+
}
|
|
6386
|
+
/**
|
|
6387
|
+
* Runs a streamed recipe's download and parse steps a page at a time: each page's bytes are hashed, decoded and read into rows, and then let go.
|
|
6388
|
+
* @param dataset - The dataset.
|
|
6389
|
+
* @param recipe - Its streamed recipe.
|
|
6390
|
+
* @param options - The network port, abort signal and clock.
|
|
6391
|
+
* @returns The stream.
|
|
6392
|
+
*/
|
|
6393
|
+
function openNative(dataset, recipe, options) {
|
|
6394
|
+
const { entry } = dataset;
|
|
6395
|
+
const { fetch: port = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
|
|
6396
|
+
const source = entry.sources[recipe.source];
|
|
6397
|
+
if (source === void 0) throw new RecipeError(entry.id, "recipe", `the recipe names source ${String(recipe.source)}, which the entry does not have`);
|
|
6398
|
+
const fetchedUrl = sourceUrl(source);
|
|
6399
|
+
const retrievedAt = now().toISOString();
|
|
6400
|
+
const hash = createHash("sha256");
|
|
6401
|
+
let byteLength = 0;
|
|
6402
|
+
async function* pages() {
|
|
6403
|
+
const decoder = new PieceDecoder(recipe.parser);
|
|
6404
|
+
const reader = new TableReader(recipe.parser);
|
|
6405
|
+
for await (const page of inStep(entry.id, "download", csvPages(fetchedUrl, recipe.paging, port, signal))) {
|
|
6406
|
+
hash.update(page);
|
|
6407
|
+
byteLength += page.byteLength;
|
|
6408
|
+
const rows = await step(entry.id, "parse", () => reader.push(decoder.decode(page)));
|
|
6409
|
+
if (rows.length > 0) yield rows;
|
|
6410
|
+
}
|
|
6411
|
+
const rows = await step(entry.id, "parse", () => [...reader.push(decoder.end()), ...reader.end()]);
|
|
6412
|
+
if (rows.length > 0) yield rows;
|
|
6413
|
+
}
|
|
6414
|
+
return {
|
|
6415
|
+
meta: metaOf(entry),
|
|
6416
|
+
pages: pages(),
|
|
6417
|
+
provenance: () => ({
|
|
6418
|
+
entryId: entry.id,
|
|
6419
|
+
catalogue: dataset.catalogue,
|
|
6420
|
+
category: entry.category,
|
|
6421
|
+
source: sourceRecord(source),
|
|
6422
|
+
fetchedUrl,
|
|
6423
|
+
retrievedAt,
|
|
6424
|
+
sha256: hash.copy().digest("hex"),
|
|
6425
|
+
byteLength
|
|
6426
|
+
})
|
|
6427
|
+
};
|
|
6428
|
+
}
|
|
6429
|
+
/**
|
|
6430
|
+
* The nodes of an encoded page: the crosswalk writes only the meta and the nodes, so the result is an object with a `nodes` array.
|
|
6431
|
+
* @param encoded - The crosswalk's output for a page.
|
|
6432
|
+
* @returns Its nodes.
|
|
6433
|
+
* @throws KeepsetError when the output is not an object with a nodes array and nothing beyond meta.
|
|
6434
|
+
*/
|
|
6435
|
+
function nodesOf(encoded) {
|
|
6436
|
+
if (typeof encoded !== "object" || encoded === null || Array.isArray(encoded) || !("nodes" in encoded) || !Array.isArray(encoded.nodes)) throw new KeepsetError("the crosswalk's output is not an object with a nodes array");
|
|
6437
|
+
const extra = Object.keys(encoded).filter((key) => key !== "meta" && key !== "nodes");
|
|
6438
|
+
if (extra.length > 0) throw new KeepsetError(`the crosswalk's output has ${extra.join(", ")} beside meta and nodes`);
|
|
6439
|
+
return encoded.nodes;
|
|
6440
|
+
}
|
|
6441
|
+
/**
|
|
6442
|
+
* Checks a crosswalk can run a page at a time, or fails the crosswalk step.
|
|
6443
|
+
* @param entry - The entry.
|
|
6444
|
+
* @param crosswalk - Its crosswalk.
|
|
6445
|
+
* @throws RecipeError when a rule reaches beyond the meta or one node's fields.
|
|
6446
|
+
*/
|
|
6447
|
+
function assertSeparable(entry, crosswalk) {
|
|
6448
|
+
const reason = inseparableRule(crosswalk);
|
|
6449
|
+
if (reason !== void 0) throw new RecipeError(entry.id, "crosswalk", `a streamed crosswalk must apply to a page of nodes on its own: ${reason}`);
|
|
6450
|
+
}
|
|
6451
|
+
/**
|
|
6452
|
+
* Streams a table, as `streamEntry` describes.
|
|
6453
|
+
* @param dataset - A dataset whose recipe streams.
|
|
6454
|
+
* @param sink - Where the meta and the nodes go.
|
|
6455
|
+
* @param options - The network port, abort signal and clock.
|
|
6456
|
+
* @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
|
|
6457
|
+
* @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
|
|
6458
|
+
*/
|
|
6459
|
+
async function streamTable(dataset, sink, options) {
|
|
6460
|
+
const { entry, crosswalk } = dataset;
|
|
6461
|
+
const recipe = streamedRecipe(entry);
|
|
6462
|
+
assertSeparable(entry, crosswalk);
|
|
6463
|
+
const shape = getStreamedShape(recipe.category);
|
|
6464
|
+
const native = openNative(dataset, recipe, options);
|
|
6465
|
+
const meta = await step(entry.id, "validate", async () => {
|
|
6466
|
+
const encoded = await step(entry.id, "crosswalk", () => encode(crosswalk, {
|
|
6467
|
+
meta: native.meta,
|
|
6468
|
+
nodes: []
|
|
6469
|
+
}));
|
|
6470
|
+
const parsed = shape.meta.safeParse(typeof encoded === "object" && encoded !== null && !Array.isArray(encoded) && "meta" in encoded ? encoded.meta : void 0);
|
|
6471
|
+
if (!parsed.success) throw new KeepsetError(`the result's meta is not ${entry.category} meta: ${parsed.error.issues.map((issue) => `${["meta", ...issue.path.map(String)].join(".")}: ${issue.message}`).join("; ")}`);
|
|
6472
|
+
return parsed.data;
|
|
6473
|
+
});
|
|
6474
|
+
await sink.meta(meta);
|
|
6475
|
+
const ids = new IdSet();
|
|
6476
|
+
for await (const rows of native.pages) {
|
|
6477
|
+
const offset = ids.size;
|
|
6478
|
+
const encoded = await step(entry.id, "crosswalk", () => nodesOf(encode(crosswalk, {
|
|
6479
|
+
meta: native.meta,
|
|
6480
|
+
nodes: rows
|
|
6481
|
+
})));
|
|
6482
|
+
const nodes = await step(entry.id, "validate", () => encoded.map((candidate, index) => {
|
|
6483
|
+
const parsed = shape.node.safeParse(candidate);
|
|
6484
|
+
if (!parsed.success) throw new KeepsetError(`the result is not ${entry.category}: ${parsed.error.issues.map((issue) => `${[
|
|
6485
|
+
"nodes",
|
|
6486
|
+
String(offset + index),
|
|
6487
|
+
...issue.path.map(String)
|
|
6488
|
+
].join(".")}: ${issue.message}`).join("; ")}`);
|
|
6489
|
+
if (!ids.add(parsed.data.id)) throw new KeepsetError(`the result is not ${entry.category}: nodes.${String(offset + index)}.id: duplicate node id ${JSON.stringify(parsed.data.id)}`);
|
|
6490
|
+
return parsed.data;
|
|
6491
|
+
}));
|
|
6492
|
+
await sink.nodes(nodes);
|
|
6493
|
+
}
|
|
6494
|
+
const counts = /* @__PURE__ */ new Map([["nodes", ids.size]]);
|
|
6495
|
+
const warnings = Object.entries(entry.expectedCounts).filter(([key, expected]) => counts.get(key) !== expected).map(([key, expected]) => `${key}: the entry records ${String(expected)}, the fetched result has ${String(counts.get(key))}`);
|
|
6496
|
+
return {
|
|
6497
|
+
dataset,
|
|
6498
|
+
meta,
|
|
6499
|
+
provenance: native.provenance(),
|
|
6500
|
+
counts,
|
|
6501
|
+
warnings
|
|
6502
|
+
};
|
|
6503
|
+
}
|
|
6504
|
+
/**
|
|
6505
|
+
* Checks a streamed dataset's crosswalk on its real data a page at a time: crossmap's `assertRoundTrip` on the meta alone and then on each page's rows under the meta. The crosswalk applies to a page on its own, so the round trip holds on the whole table exactly when it holds on every page. It stops at the first page that differs and reports the path in the whole document.
|
|
6506
|
+
* @param dataset - A dataset whose recipe streams.
|
|
6507
|
+
* @param options - The network port, abort signal and clock.
|
|
6508
|
+
* @returns Whether the round trip holds, and where it first differs when it does not, with the provenance of the bytes it was checked on: every byte when it holds, those read up to the page that differs when it does not.
|
|
6509
|
+
* @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison or the crosswalk cannot run.
|
|
6510
|
+
*/
|
|
6511
|
+
async function roundTripTable(dataset, options) {
|
|
6512
|
+
const { entry, crosswalk } = dataset;
|
|
6513
|
+
const recipe = streamedRecipe(entry);
|
|
6514
|
+
assertSeparable(entry, crosswalk);
|
|
6515
|
+
const native = openNative(dataset, recipe, options);
|
|
6516
|
+
const check = (nodes, offset) => {
|
|
6517
|
+
try {
|
|
6518
|
+
assertRoundTrip(crosswalk, [{
|
|
6519
|
+
meta: native.meta,
|
|
6520
|
+
nodes
|
|
6521
|
+
}]);
|
|
6522
|
+
} catch (error) {
|
|
6523
|
+
if (error instanceof RoundTripMismatchError) {
|
|
6524
|
+
const path = error.path.replace(/^nodes\[(\d+)\]/, (_, index) => `nodes[${String(offset + Number(index))}]`);
|
|
6525
|
+
return {
|
|
6526
|
+
holds: false,
|
|
6527
|
+
path,
|
|
6528
|
+
message: error.message.replaceAll(error.path, path),
|
|
6529
|
+
provenance: native.provenance()
|
|
6530
|
+
};
|
|
6531
|
+
}
|
|
6532
|
+
throw new RecipeError(entry.id, "crosswalk", error instanceof Error ? error.message : String(error), { cause: error });
|
|
6533
|
+
}
|
|
6534
|
+
};
|
|
6535
|
+
const metaMismatch = check([], 0);
|
|
6536
|
+
if (metaMismatch !== void 0) return metaMismatch;
|
|
6537
|
+
let offset = 0;
|
|
6538
|
+
for await (const rows of native.pages) {
|
|
6539
|
+
const mismatch = check(rows, offset);
|
|
6540
|
+
if (mismatch !== void 0) {
|
|
6541
|
+
await native.pages.return();
|
|
6542
|
+
return mismatch;
|
|
6543
|
+
}
|
|
6544
|
+
offset += rows.length;
|
|
6545
|
+
}
|
|
6546
|
+
return {
|
|
6547
|
+
holds: true,
|
|
6548
|
+
provenance: native.provenance()
|
|
6549
|
+
};
|
|
6550
|
+
}
|
|
6551
|
+
//#endregion
|
|
6552
|
+
//#region src/node/fetch-dataset.ts
|
|
6553
|
+
/**
|
|
6554
|
+
* The URL a recipe downloads a source from. A dead source is fetched from its pinned snapshot in the Wayback Machine's `id_` form, which serves the archived bytes as they were captured rather than wrapped in the archive's page chrome.
|
|
6555
|
+
* @param source - The source.
|
|
6556
|
+
* @returns The URL to request.
|
|
6557
|
+
*/
|
|
6558
|
+
function downloadUrl(source) {
|
|
6559
|
+
return sourceUrl(source);
|
|
6560
|
+
}
|
|
5369
6561
|
/**
|
|
5370
6562
|
* Puts an instance's id beside the fields of its encoded document. A document that is not an object is left as it is, for the collection schema to refuse.
|
|
5371
6563
|
* @param document - The crosswalk's output for one member.
|
|
@@ -5379,6 +6571,139 @@ function withId(document, id) {
|
|
|
5379
6571
|
} : document;
|
|
5380
6572
|
}
|
|
5381
6573
|
/**
|
|
6574
|
+
* Downloads one URL's bytes through the network port.
|
|
6575
|
+
* @param port - The network port.
|
|
6576
|
+
* @param url - The URL.
|
|
6577
|
+
* @param signal - Aborts the download.
|
|
6578
|
+
* @returns The bytes.
|
|
6579
|
+
* @throws KeepsetError when the server does not answer successfully.
|
|
6580
|
+
*/
|
|
6581
|
+
async function downloadBytes(port, url, signal) {
|
|
6582
|
+
const response = await port(url, signal === void 0 ? {} : { signal });
|
|
6583
|
+
if (!response.ok) throw new KeepsetError(`${url} answered ${String(response.status)} ${response.statusText}`);
|
|
6584
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
6585
|
+
}
|
|
6586
|
+
/**
|
|
6587
|
+
* The record of one download: the source, the URL requested and a hash of exactly the bytes that arrived.
|
|
6588
|
+
* @param source - The entry's source.
|
|
6589
|
+
* @param fetchedUrl - The URL requested.
|
|
6590
|
+
* @param bytes - The bytes.
|
|
6591
|
+
* @returns The download record.
|
|
6592
|
+
*/
|
|
6593
|
+
function downloadOf(source, fetchedUrl, bytes) {
|
|
6594
|
+
return {
|
|
6595
|
+
source: sourceRecord(source),
|
|
6596
|
+
fetchedUrl,
|
|
6597
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
6598
|
+
byteLength: bytes.byteLength
|
|
6599
|
+
};
|
|
6600
|
+
}
|
|
6601
|
+
/**
|
|
6602
|
+
* Parses one file of a collection into its native document, naming the file when it fails.
|
|
6603
|
+
* @param id - The instance's id.
|
|
6604
|
+
* @param bytes - The file's bytes.
|
|
6605
|
+
* @param parser - The recipe's parser.
|
|
6606
|
+
* @returns The instance's native document, with only its own header values as `meta`.
|
|
6607
|
+
* @throws KeepsetError naming the instance when the file does not parse.
|
|
6608
|
+
*/
|
|
6609
|
+
function parseInstance(id, bytes, parser) {
|
|
6610
|
+
try {
|
|
6611
|
+
const parsed = parseNative(bytes, parser);
|
|
6612
|
+
return {
|
|
6613
|
+
id,
|
|
6614
|
+
native: {
|
|
6615
|
+
...parsed,
|
|
6616
|
+
meta: "meta" in parsed ? parsed.meta : {}
|
|
6617
|
+
}
|
|
6618
|
+
};
|
|
6619
|
+
} catch (error) {
|
|
6620
|
+
throw new KeepsetError(`member ${id}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
6621
|
+
}
|
|
6622
|
+
}
|
|
6623
|
+
/**
|
|
6624
|
+
* Unpacks and parses the bytes of one archive or file into instances, each member let go once it is parsed.
|
|
6625
|
+
* @param bytes - The bytes.
|
|
6626
|
+
* @param archive - Their archive layout.
|
|
6627
|
+
* @param parser - The recipe's parser.
|
|
6628
|
+
* @param prefix - The path the instance ids are filed under, joined to each member's path, or undefined to use the member's path alone.
|
|
6629
|
+
* @returns The instances, in member order.
|
|
6630
|
+
*/
|
|
6631
|
+
function instancesOf(bytes, archive, parser, prefix) {
|
|
6632
|
+
const named = (member) => {
|
|
6633
|
+
if (prefix === void 0) {
|
|
6634
|
+
if (member === void 0) throw new KeepsetError("an instance of a collection has no path");
|
|
6635
|
+
return member;
|
|
6636
|
+
}
|
|
6637
|
+
return member === void 0 ? prefix : `${prefix}/${member}`;
|
|
6638
|
+
};
|
|
6639
|
+
if ("members" in archive) return Array.from(unpackMembers(bytes, archive), (member) => parseInstance(named(member.path), member.bytes, parser));
|
|
6640
|
+
if ("parts" in archive) throw new ArchiveError(`an instance is read from one file or from matching members, not from the named parts ${archive.parts.map((part) => part.name).join(", ")}`);
|
|
6641
|
+
if ("files" in archive) throw new ArchiveError("an instance is read from one file or from matching members, not from every file a multi-file format's files selector matches");
|
|
6642
|
+
const unpacked = unpack(bytes, archive);
|
|
6643
|
+
return [parseInstance(named(unpacked.member), unpacked.bytes, parser)];
|
|
6644
|
+
}
|
|
6645
|
+
/**
|
|
6646
|
+
* Unpacks and parses instances as `instancesOf` does, failing at the extract step when the archive cannot be read or its selector matches nothing, and at the parse step when a member does not parse: the members are unpacked one at a time as they are parsed, so the two steps interleave.
|
|
6647
|
+
* @param entryId - The entry.
|
|
6648
|
+
* @param run - The unpacking and parsing.
|
|
6649
|
+
* @returns The instances.
|
|
6650
|
+
* @throws RecipeError naming the entry and the step that failed.
|
|
6651
|
+
*/
|
|
6652
|
+
function extractAndParse(entryId, run) {
|
|
6653
|
+
try {
|
|
6654
|
+
return run();
|
|
6655
|
+
} catch (error) {
|
|
6656
|
+
throw new RecipeError(entryId, error instanceof ArchiveError ? "extract" : "parse", error instanceof Error ? error.message : String(error), { cause: error });
|
|
6657
|
+
}
|
|
6658
|
+
}
|
|
6659
|
+
/**
|
|
6660
|
+
* Runs a files recipe: downloads each file in turn (a dead file from its own pinned snapshot), unpacks it and parses it into its instances, so only one file's bytes are held at a time.
|
|
6661
|
+
* @param dataset - The dataset.
|
|
6662
|
+
* @param recipe - Its files recipe.
|
|
6663
|
+
* @param options - The network port, abort signal and clock.
|
|
6664
|
+
* @returns The instances and the provenance of every file.
|
|
6665
|
+
*/
|
|
6666
|
+
async function runFiles(dataset, recipe, options) {
|
|
6667
|
+
const { entry } = dataset;
|
|
6668
|
+
const { fetch: port = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
|
|
6669
|
+
const retrievedAt = now().toISOString();
|
|
6670
|
+
const read = async (index) => {
|
|
6671
|
+
const file = recipe.files[index];
|
|
6672
|
+
if (file === void 0) return [];
|
|
6673
|
+
const source = entry.sources[file.source];
|
|
6674
|
+
if (source === void 0) throw new RecipeError(entry.id, "recipe", `file ${file.path} names source ${String(file.source)}, which the entry does not have`);
|
|
6675
|
+
const fetchedUrl = sourceUrl(source);
|
|
6676
|
+
const bytes = await step(entry.id, "download", async () => downloadBytes(port, fetchedUrl, signal));
|
|
6677
|
+
const documents = extractAndParse(entry.id, () => instancesOf(bytes, file.archive, recipe.parser, file.path));
|
|
6678
|
+
return [{
|
|
6679
|
+
documents,
|
|
6680
|
+
download: {
|
|
6681
|
+
...downloadOf(source, fetchedUrl, bytes),
|
|
6682
|
+
path: file.path,
|
|
6683
|
+
instances: documents.map((document) => document.id)
|
|
6684
|
+
}
|
|
6685
|
+
}, ...await read(index + 1)];
|
|
6686
|
+
};
|
|
6687
|
+
const files = await read(0);
|
|
6688
|
+
const documents = files.flatMap((file) => file.documents);
|
|
6689
|
+
const ids = /* @__PURE__ */ new Set();
|
|
6690
|
+
for (const { id } of documents) {
|
|
6691
|
+
if (ids.has(id)) throw new RecipeError(entry.id, "extract", `two files give the instance ${id}`);
|
|
6692
|
+
ids.add(id);
|
|
6693
|
+
}
|
|
6694
|
+
return {
|
|
6695
|
+
collection: true,
|
|
6696
|
+
documents,
|
|
6697
|
+
provenance: {
|
|
6698
|
+
entryId: entry.id,
|
|
6699
|
+
catalogue: dataset.catalogue,
|
|
6700
|
+
category: entry.category,
|
|
6701
|
+
retrievedAt,
|
|
6702
|
+
files: files.map((file) => file.download)
|
|
6703
|
+
}
|
|
6704
|
+
};
|
|
6705
|
+
}
|
|
6706
|
+
/**
|
|
5382
6707
|
* Runs a recipe's download, extract and parse steps, and builds the native documents the crosswalk receives.
|
|
5383
6708
|
* @param dataset - The dataset.
|
|
5384
6709
|
* @param options - The network port, abort signal and clock.
|
|
@@ -5389,51 +6714,28 @@ async function runToNative(dataset, options) {
|
|
|
5389
6714
|
const { fetch: fetchPort = downloadRaw, signal, now = () => /* @__PURE__ */ new Date() } = options;
|
|
5390
6715
|
const { recipe } = entry;
|
|
5391
6716
|
if (recipe.status === "not-fetchable") throw new RecipeError(entry.id, "recipe", `the entry has no fetchable recipe: ${recipe.reason}`);
|
|
6717
|
+
if ("files" in recipe) return runFiles(dataset, recipe, options);
|
|
6718
|
+
if (recipe.stream === true) throw new RecipeError(entry.id, "recipe", "the table is too large to hold as one document, so its recipe streams: read it with streamDataset, which hands its nodes on a page at a time");
|
|
5392
6719
|
const source = entry.sources[recipe.source];
|
|
5393
6720
|
if (source === void 0) throw new RecipeError(entry.id, "recipe", `the recipe names source ${String(recipe.source)}, which the entry does not have`);
|
|
5394
|
-
const fetchedUrl =
|
|
6721
|
+
const fetchedUrl = sourceUrl(source);
|
|
5395
6722
|
const retrievedAt = now().toISOString();
|
|
5396
|
-
const bytes = await step(entry.id, "download", async () =>
|
|
5397
|
-
if (recipe.paging !== void 0) return downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal);
|
|
5398
|
-
const response = await fetchPort(fetchedUrl, signal === void 0 ? {} : { signal });
|
|
5399
|
-
if (!response.ok) throw new KeepsetError(`${fetchedUrl} answered ${String(response.status)} ${response.statusText}`);
|
|
5400
|
-
return new Uint8Array(await response.arrayBuffer());
|
|
5401
|
-
});
|
|
6723
|
+
const bytes = await step(entry.id, "download", async () => recipe.paging === void 0 ? downloadBytes(fetchPort, fetchedUrl, signal) : downloadPaged(fetchedUrl, recipe.paging, fetchPort, signal));
|
|
5402
6724
|
const provenanceOf = (extra) => ({
|
|
5403
6725
|
entryId: entry.id,
|
|
5404
6726
|
catalogue: dataset.catalogue,
|
|
5405
6727
|
category: entry.category,
|
|
5406
|
-
source: {
|
|
5407
|
-
url: source.url,
|
|
5408
|
-
status: source.status,
|
|
5409
|
-
...source.snapshot === void 0 ? {} : { snapshot: source.snapshot }
|
|
5410
|
-
},
|
|
5411
|
-
fetchedUrl,
|
|
5412
6728
|
retrievedAt,
|
|
5413
|
-
|
|
5414
|
-
byteLength: bytes.byteLength,
|
|
6729
|
+
...downloadOf(source, fetchedUrl, bytes),
|
|
5415
6730
|
...extra
|
|
5416
6731
|
});
|
|
5417
6732
|
const { archive } = recipe;
|
|
5418
6733
|
if ("members" in archive) {
|
|
5419
|
-
const
|
|
6734
|
+
const documents = extractAndParse(entry.id, () => instancesOf(bytes, archive, recipe.parser, void 0));
|
|
5420
6735
|
return {
|
|
5421
6736
|
collection: true,
|
|
5422
|
-
documents
|
|
5423
|
-
|
|
5424
|
-
const parsed = parseNative(file.bytes, recipe.parser);
|
|
5425
|
-
return {
|
|
5426
|
-
id: file.path,
|
|
5427
|
-
native: {
|
|
5428
|
-
...parsed,
|
|
5429
|
-
meta: "meta" in parsed ? parsed.meta : {}
|
|
5430
|
-
}
|
|
5431
|
-
};
|
|
5432
|
-
} catch (error) {
|
|
5433
|
-
throw new KeepsetError(`member ${file.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5434
|
-
}
|
|
5435
|
-
})),
|
|
5436
|
-
provenance: provenanceOf({ members: files.map((file) => file.path) })
|
|
6737
|
+
documents,
|
|
6738
|
+
provenance: provenanceOf({ members: documents.map((document) => document.id) })
|
|
5437
6739
|
};
|
|
5438
6740
|
}
|
|
5439
6741
|
if ("parts" in archive) {
|
|
@@ -5450,6 +6752,20 @@ async function runToNative(dataset, options) {
|
|
|
5450
6752
|
provenance: provenanceOf({ members: parts.map((part) => part.path) })
|
|
5451
6753
|
};
|
|
5452
6754
|
}
|
|
6755
|
+
if ("files" in archive) {
|
|
6756
|
+
const files = await step(entry.id, "extract", () => unpackFiles(bytes, archive));
|
|
6757
|
+
return {
|
|
6758
|
+
collection: false,
|
|
6759
|
+
documents: [{
|
|
6760
|
+
id: void 0,
|
|
6761
|
+
native: {
|
|
6762
|
+
...await step(entry.id, "parse", () => parseNativeFiles(files, recipe.parser)),
|
|
6763
|
+
meta: metaOf(entry)
|
|
6764
|
+
}
|
|
6765
|
+
}],
|
|
6766
|
+
provenance: provenanceOf({ members: files.map((file) => file.path) })
|
|
6767
|
+
};
|
|
6768
|
+
}
|
|
5453
6769
|
const unpacked = await step(entry.id, "extract", () => unpack(bytes, archive));
|
|
5454
6770
|
const parsed = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
|
|
5455
6771
|
return {
|
|
@@ -5472,7 +6788,7 @@ async function runToNative(dataset, options) {
|
|
|
5472
6788
|
* @param dataset - A dataset from any catalogue.
|
|
5473
6789
|
* @param options - The network port, abort signal and clock.
|
|
5474
6790
|
* @returns The resolved target with its provenance and any count warnings.
|
|
5475
|
-
* @throws RecipeError naming the entry and the failing step; nothing partial or default-shaped is ever returned.
|
|
6791
|
+
* @throws RecipeError naming the entry and the failing step, including for a recipe that streams, which `streamEntry` reads; nothing partial or default-shaped is ever returned.
|
|
5476
6792
|
*/
|
|
5477
6793
|
async function fetchEntry(dataset, options = {}) {
|
|
5478
6794
|
const { entry, crosswalk } = dataset;
|
|
@@ -5513,13 +6829,14 @@ async function fetchDataset(id, options = {}) {
|
|
|
5513
6829
|
return fetchEntry(getDataset(id), options);
|
|
5514
6830
|
}
|
|
5515
6831
|
/**
|
|
5516
|
-
* Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve.
|
|
6832
|
+
* Checks a dataset's crosswalk on its real data: runs the recipe to the native document, then crossmap's `assertRoundTrip`, which encodes it, decodes the result and compares the two over every path the crosswalk promises to preserve. A streamed recipe is checked a page at a time, with the round trip holding on the whole table exactly when it holds on every page, so the table is never held.
|
|
5517
6833
|
* @param dataset - A dataset from any catalogue.
|
|
5518
6834
|
* @param options - The network port, abort signal and clock.
|
|
5519
6835
|
* @returns Whether the round trip holds, and where it first differs when it does not.
|
|
5520
6836
|
* @throws RecipeError naming the entry and the failing step when the recipe fails before the comparison, or when the crosswalk cannot run.
|
|
5521
6837
|
*/
|
|
5522
6838
|
async function roundTripEntry(dataset, options = {}) {
|
|
6839
|
+
if (isStreamed(dataset.entry.recipe)) return roundTripTable(dataset, options);
|
|
5523
6840
|
const { documents, provenance } = await runToNative(dataset, options);
|
|
5524
6841
|
try {
|
|
5525
6842
|
assertRoundTrip(dataset.crosswalk, documents.map(({ native }) => native));
|
|
@@ -5538,6 +6855,28 @@ async function roundTripEntry(dataset, options = {}) {
|
|
|
5538
6855
|
};
|
|
5539
6856
|
}
|
|
5540
6857
|
/**
|
|
6858
|
+
* Streams a dataset of the bundled catalogue by id, as `streamEntry` does: for a table too large to hold as one document, whose recipe streams.
|
|
6859
|
+
* @param id - The entry id.
|
|
6860
|
+
* @param sink - Where the meta and the nodes go.
|
|
6861
|
+
* @param options - The network port, abort signal and clock.
|
|
6862
|
+
* @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
|
|
6863
|
+
* @throws DatasetNotFoundError when no entry has that id, and RecipeError as `streamEntry` does.
|
|
6864
|
+
*/
|
|
6865
|
+
async function streamDataset(id, sink, options = {}) {
|
|
6866
|
+
return streamEntry(getDataset(id), sink, options);
|
|
6867
|
+
}
|
|
6868
|
+
/**
|
|
6869
|
+
* Streams a dataset whose table is too large to hold as one document: reads its paged source a page at a time, applies the crosswalk to each page's rows under the document's meta, checks the meta once and each node against the category's node shape and for an id no earlier node had, and hands the nodes on to the sink before reading the next page. What it holds at once is one page's bytes and rows and the ids seen so far (as bytes, not strings), never the table, so the result is the category target `{ meta, nodes }` delivered in order rather than held: the same nodes `fetchEntry` would give, checked by the same rules.
|
|
6870
|
+
* @param dataset - A dataset whose recipe streams.
|
|
6871
|
+
* @param sink - Where the meta and the nodes go.
|
|
6872
|
+
* @param options - The network port, abort signal and clock.
|
|
6873
|
+
* @returns The meta, the provenance, the node count and any count warnings, once every node has been handed on.
|
|
6874
|
+
* @throws RecipeError naming the entry and the failing step, after which the sink has had the nodes of the pages before the failure and no more; a recipe that does not stream, or a crosswalk that cannot run a page at a time, fails before anything is downloaded.
|
|
6875
|
+
*/
|
|
6876
|
+
async function streamEntry(dataset, sink, options = {}) {
|
|
6877
|
+
return streamTable(dataset, sink, options);
|
|
6878
|
+
}
|
|
6879
|
+
/**
|
|
5541
6880
|
* Runs `roundTripEntry` on a dataset of the bundled catalogue by id.
|
|
5542
6881
|
* @param id - The entry id.
|
|
5543
6882
|
* @param options - The network port, abort signal and clock.
|
|
@@ -5548,4 +6887,4 @@ async function roundTripDataset(id, options = {}) {
|
|
|
5548
6887
|
return roundTripEntry(getDataset(id), options);
|
|
5549
6888
|
}
|
|
5550
6889
|
//#endregion
|
|
5551
|
-
export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry };
|
|
6890
|
+
export { downloadUrl, fetchDataset, fetchEntry, roundTripDataset, roundTripEntry, streamDataset, streamEntry };
|