keepset 0.37.1 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{bundled-CRr7GPD3.js → bundled-6dYCPVD-.js} +509 -80
- package/dist/{categories-Djnk3zwg.d.ts → categories-DijttH_M.d.ts} +286 -15
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/node/fetch-dataset.d.ts +2 -2
- package/dist/node/fetch-dataset.js +1217 -118
- package/dist-cli/keepset.mjs +2 -2
- package/package.json +1 -1
- package/schemas/entry.schema.json +1 -1
- package/schemas/manifest.schema.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as ArchiveError, H as getCollectionTarget, L as getOwn, U as getTarget, at as RecipeError, it as NativeParseError, rt as KeepsetError, t as getDataset, z as countTarget } from "../bundled-
|
|
1
|
+
import { $ as ArchiveError, H as getCollectionTarget, L as getOwn, U as getTarget, at as RecipeError, it as NativeParseError, rt as KeepsetError, t as getDataset, z as countTarget } from "../bundled-6dYCPVD-.js";
|
|
2
2
|
import { RoundTripMismatchError, assertRoundTrip, encode } from "crossmap";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
@@ -127,6 +127,15 @@ function endpoint$1(fields, column, lineNumber) {
|
|
|
127
127
|
return value.trim();
|
|
128
128
|
}
|
|
129
129
|
/**
|
|
130
|
+
* Whether a line meets one of the recipe's `where` conditions.
|
|
131
|
+
* @param fields - The line's fields, at least as many as the highest column the recipe names.
|
|
132
|
+
* @param condition - The condition.
|
|
133
|
+
* @returns True when the condition's column, trimmed, equals its value.
|
|
134
|
+
*/
|
|
135
|
+
function meets(fields, condition) {
|
|
136
|
+
return fields[condition.column]?.trim() === condition.equals;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
130
139
|
* Resolves `detect` to a fixed delimiter from the first data line: tab when it holds a tab, comma when it holds a comma, whitespace otherwise. The rule is a function of the file's bytes, so a recipe that detects still reads the same way every time.
|
|
131
140
|
* @param dataLine - The first line that is neither skipped, blank nor a comment.
|
|
132
141
|
* @returns The fixed delimiter.
|
|
@@ -136,34 +145,76 @@ function detect(dataLine) {
|
|
|
136
145
|
return dataLine.includes(",") ? "comma" : "whitespace";
|
|
137
146
|
}
|
|
138
147
|
/**
|
|
139
|
-
*
|
|
140
|
-
* @param
|
|
148
|
+
* Reads a line's multiplicity.
|
|
149
|
+
* @param fields - The line's fields.
|
|
150
|
+
* @param column - The multiplicity column.
|
|
151
|
+
* @param lineNumber - The 1-based line, for the error.
|
|
152
|
+
* @returns How many edges the line stands for.
|
|
153
|
+
* @throws NativeParseError when the field is not a whole number.
|
|
154
|
+
*/
|
|
155
|
+
function multiplicity(fields, column, lineNumber) {
|
|
156
|
+
const value = fields[column]?.trim() ?? "";
|
|
157
|
+
if (!/^(?:0|[1-9]\d*)$/.test(value)) throw new NativeParseError("edge-list", lineNumber, `column ${String(column)} holds ${JSON.stringify(value)}, which is not a whole number of edges`);
|
|
158
|
+
return Number(value);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Parses a delimited edge list: one edge per line, source and target in the configured columns, after `skipLines` header lines, with blank lines and lines whose first non-blank characters are a comment prefix skipped. A `detect` delimiter is fixed from the first data line. A recipe that names a `keep` column reads only the lines whose field there is one of its values, and one that states `where` reads only the lines whose listed columns, trimmed, equal their values, so a file that records several kinds of row (an interaction file listing genetic beside physical interactions) is read for the rows the dataset is made of; a line that is not read is still checked for the fields it needs. A recipe that names a `multiplicityColumn` emits as many edges for a line as the field says, none for a zero; one that says `mirrored` emits each pair of mirror-image lines once. Nodes are the endpoints of the lines read, in order of first appearance. The text is read line by line, so a file longer than the engine's longest string is read too.
|
|
162
|
+
* @param text - The file text, whole or in pieces.
|
|
141
163
|
* @param options - The recipe's edge-list parser.
|
|
142
164
|
* @returns The native graph.
|
|
143
|
-
* @throws NativeParseError naming the line when a data line has too few fields
|
|
165
|
+
* @throws NativeParseError naming the line when a data line has too few fields, an empty endpoint or a multiplicity that is not a whole number, when a mirrored table has a line without its mirror, or when no line is an edge.
|
|
144
166
|
*/
|
|
145
167
|
function parseEdgeList(text, options) {
|
|
146
168
|
let split = options.delimiter === "detect" ? void 0 : splitters[options.delimiter];
|
|
147
|
-
const
|
|
169
|
+
const { where } = options;
|
|
170
|
+
const needed = Math.max(options.sourceColumn, options.targetColumn, options.keep?.column ?? 0, options.multiplicityColumn ?? 0, ...where === void 0 ? [] : where.map((condition) => condition.column)) + 1;
|
|
148
171
|
const order = new NodeOrder();
|
|
149
172
|
const edges = [];
|
|
150
|
-
|
|
151
|
-
|
|
173
|
+
const awaiting = /* @__PURE__ */ new Map();
|
|
174
|
+
let lastLine = 0;
|
|
175
|
+
forEachLine(typeof text === "string" ? [text] : text, (line, lineNumber) => {
|
|
176
|
+
lastLine = lineNumber;
|
|
152
177
|
const content = line.trimStart();
|
|
153
|
-
if (
|
|
178
|
+
if (lineNumber <= options.skipLines || content === "" || options.commentPrefixes.some((prefix) => content.startsWith(prefix))) return;
|
|
154
179
|
split ??= splitters[detect(line)];
|
|
155
180
|
const fields = split(line);
|
|
156
181
|
if (fields.length < needed) throw new NativeParseError("edge-list", lineNumber, `expected at least ${String(needed)} fields, found ${String(fields.length)}`);
|
|
182
|
+
if (options.keep !== void 0 && !options.keep.values.includes(fields[options.keep.column]?.trim() ?? "")) return;
|
|
183
|
+
if (where !== void 0 && !where.every((condition) => meets(fields, condition))) return;
|
|
157
184
|
const source = endpoint$1(fields, options.sourceColumn, lineNumber);
|
|
158
185
|
const target = endpoint$1(fields, options.targetColumn, lineNumber);
|
|
186
|
+
const count = options.multiplicityColumn === void 0 ? 1 : multiplicity(fields, options.multiplicityColumn, lineNumber);
|
|
187
|
+
if (count === 0) return;
|
|
159
188
|
order.add(source);
|
|
160
189
|
order.add(target);
|
|
161
|
-
|
|
190
|
+
if (options.mirrored === true && source !== target) {
|
|
191
|
+
const key = JSON.stringify([
|
|
192
|
+
source,
|
|
193
|
+
target,
|
|
194
|
+
count
|
|
195
|
+
]);
|
|
196
|
+
const waiting = awaiting.get(key);
|
|
197
|
+
if (waiting !== void 0 && waiting.length > 0) {
|
|
198
|
+
waiting.shift();
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
const mirror = JSON.stringify([
|
|
202
|
+
target,
|
|
203
|
+
source,
|
|
204
|
+
count
|
|
205
|
+
]);
|
|
206
|
+
awaiting.set(mirror, [...awaiting.get(mirror) ?? [], lineNumber]);
|
|
207
|
+
}
|
|
208
|
+
for (let copy = 0; copy < count; copy += 1) edges.push({
|
|
162
209
|
source,
|
|
163
210
|
target
|
|
164
211
|
});
|
|
212
|
+
});
|
|
213
|
+
for (const [key, waiting] of awaiting) {
|
|
214
|
+
const [first] = waiting;
|
|
215
|
+
if (first !== void 0) throw new NativeParseError("edge-list", first, `the table is mirrored but this line has no mirror image (${key} is never listed)`);
|
|
165
216
|
}
|
|
166
|
-
if (edges.length === 0) throw new NativeParseError("edge-list",
|
|
217
|
+
if (edges.length === 0) throw new NativeParseError("edge-list", lastLine, where === void 0 ? "the file holds no edges" : "no line of the file meets the recipe's where conditions");
|
|
167
218
|
return {
|
|
168
219
|
nodes: order.nodes(),
|
|
169
220
|
edges
|
|
@@ -171,7 +222,7 @@ function parseEdgeList(text, options) {
|
|
|
171
222
|
}
|
|
172
223
|
//#endregion
|
|
173
224
|
//#region src/parsers/gd-contest.ts
|
|
174
|
-
const FORMAT$
|
|
225
|
+
const FORMAT$15 = "gd-contest";
|
|
175
226
|
const NUMBER$9 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
|
|
176
227
|
/**
|
|
177
228
|
* Reads a vertex number, which the format numbers from 0 to N - 1.
|
|
@@ -180,7 +231,7 @@ const NUMBER$9 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[Ee][+-]?\d+)?$/;
|
|
|
180
231
|
* @returns The vertex id as text.
|
|
181
232
|
*/
|
|
182
233
|
function vertex$1(token, count) {
|
|
183
|
-
if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$
|
|
234
|
+
if (!/^(?:0|[1-9]\d*)$/.test(token.text) || Number(token.text) >= count) throw new NativeParseError(FORMAT$15, token.line, `expected a vertex number from 0 to ${String(count - 1)}, found ${JSON.stringify(token.text)}`);
|
|
184
235
|
return token.text;
|
|
185
236
|
}
|
|
186
237
|
/**
|
|
@@ -200,17 +251,17 @@ function parseGdContest(text) {
|
|
|
200
251
|
});
|
|
201
252
|
}
|
|
202
253
|
const [first] = tokens;
|
|
203
|
-
if (first === void 0) throw new NativeParseError(FORMAT$
|
|
204
|
-
if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$
|
|
254
|
+
if (first === void 0) throw new NativeParseError(FORMAT$15, 1, "the file holds no vertex count");
|
|
255
|
+
if (!/^(?:0|[1-9]\d*)$/.test(first.text)) throw new NativeParseError(FORMAT$15, first.line, `the vertex count must be a non-negative integer, found ${JSON.stringify(first.text)}`);
|
|
205
256
|
const count = Number(first.text);
|
|
206
257
|
const coordinates = tokens.slice(1, 1 + 2 * count);
|
|
207
|
-
if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$
|
|
208
|
-
for (const coordinate of coordinates) if (!NUMBER$9.test(coordinate.text)) throw new NativeParseError(FORMAT$
|
|
258
|
+
if (coordinates.length < 2 * count) throw new NativeParseError(FORMAT$15, lines(text).length, `expected ${String(2 * count)} coordinates for ${String(count)} vertices, found ${String(coordinates.length)}`);
|
|
259
|
+
for (const coordinate of coordinates) if (!NUMBER$9.test(coordinate.text)) throw new NativeParseError(FORMAT$15, coordinate.line, `coordinate ${JSON.stringify(coordinate.text)} is not a number`);
|
|
209
260
|
const ends = tokens.slice(1 + 2 * count);
|
|
210
261
|
const edges = [];
|
|
211
262
|
for (let source = ends.shift(); source !== void 0; source = ends.shift()) {
|
|
212
263
|
const target = ends.shift();
|
|
213
|
-
if (target === void 0) throw new NativeParseError(FORMAT$
|
|
264
|
+
if (target === void 0) throw new NativeParseError(FORMAT$15, source.line, `edge from vertex ${source.text} has no second vertex`);
|
|
214
265
|
edges.push({
|
|
215
266
|
source: vertex$1(source, count),
|
|
216
267
|
target: vertex$1(target, count)
|
|
@@ -380,11 +431,11 @@ function parseGml(text) {
|
|
|
380
431
|
}
|
|
381
432
|
//#endregion
|
|
382
433
|
//#region src/parsers/hgb.ts
|
|
383
|
-
const FORMAT$
|
|
434
|
+
const FORMAT$14 = "hgb";
|
|
384
435
|
/** A type code or node id as the HGB files write them: a non-negative decimal integer with no sign or leading zero. */
|
|
385
436
|
const CODE = /^(?:0|[1-9]\d*)$/;
|
|
386
437
|
/** A number as a decimal literal with an optional sign and exponent, so `Number` cannot read hex, an empty string or infinity. */
|
|
387
|
-
const DECIMAL$
|
|
438
|
+
const DECIMAL$5 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/;
|
|
388
439
|
/** `node.dat` lines are the id, the name and the type, then the attributes on node types that have them. */
|
|
389
440
|
const NODE_FIELDS = 3;
|
|
390
441
|
const NODE_FIELDS_WITH_ATTRIBUTES = 4;
|
|
@@ -400,7 +451,7 @@ const LINK_FIELDS = 4;
|
|
|
400
451
|
*/
|
|
401
452
|
function resolve(field, types, where) {
|
|
402
453
|
const type = types[CODE.test(field) ? Number(field) : -1];
|
|
403
|
-
if (type === void 0) throw new NativeParseError(FORMAT$
|
|
454
|
+
if (type === void 0) throw new NativeParseError(FORMAT$14, where.line, `${where.file}: the ${where.kind} type ${JSON.stringify(field)} is not one of the ${String(types.length)} the recipe names`);
|
|
404
455
|
return type;
|
|
405
456
|
}
|
|
406
457
|
/**
|
|
@@ -418,9 +469,9 @@ function parseHgb(parts, options) {
|
|
|
418
469
|
forEachLine(parts.nodes, (line, number) => {
|
|
419
470
|
const fields = line.split(" ");
|
|
420
471
|
const [id, , type] = fields;
|
|
421
|
-
if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$
|
|
422
|
-
if (!CODE.test(id)) throw new NativeParseError(FORMAT$
|
|
423
|
-
if (typeOf.has(id)) throw new NativeParseError(FORMAT$
|
|
472
|
+
if (fields.length !== NODE_FIELDS && fields.length !== NODE_FIELDS_WITH_ATTRIBUTES || id === void 0 || type === void 0) throw new NativeParseError(FORMAT$14, number, `node.dat: expected ${String(NODE_FIELDS)} or ${String(NODE_FIELDS_WITH_ATTRIBUTES)} fields separated by tabs, found ${String(fields.length)}`);
|
|
473
|
+
if (!CODE.test(id)) throw new NativeParseError(FORMAT$14, number, `node.dat: the node id ${JSON.stringify(id)} is not a non-negative integer`);
|
|
474
|
+
if (typeOf.has(id)) throw new NativeParseError(FORMAT$14, number, `node.dat: the node id ${id} is repeated`);
|
|
424
475
|
const nodeType = resolve(type, options.nodeTypes, {
|
|
425
476
|
file: "node.dat",
|
|
426
477
|
line: number,
|
|
@@ -432,12 +483,12 @@ function parseHgb(parts, options) {
|
|
|
432
483
|
type: nodeType
|
|
433
484
|
});
|
|
434
485
|
});
|
|
435
|
-
if (nodes.length === 0) throw new NativeParseError(FORMAT$
|
|
486
|
+
if (nodes.length === 0) throw new NativeParseError(FORMAT$14, 1, "node.dat: the file holds no nodes");
|
|
436
487
|
const edges = [];
|
|
437
488
|
forEachLine(parts.links, (line, number) => {
|
|
438
489
|
const fields = line.split(" ");
|
|
439
490
|
const [source, target, type, weight] = fields;
|
|
440
|
-
if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$
|
|
491
|
+
if (fields.length !== LINK_FIELDS || source === void 0 || target === void 0 || type === void 0 || weight === void 0) throw new NativeParseError(FORMAT$14, number, `link.dat: expected ${String(LINK_FIELDS)} fields separated by tabs, found ${String(fields.length)}`);
|
|
441
492
|
const linkType = resolve(type, options.linkTypes, {
|
|
442
493
|
file: "link.dat",
|
|
443
494
|
line: number,
|
|
@@ -453,17 +504,17 @@ function parseHgb(parts, options) {
|
|
|
453
504
|
linkType.target
|
|
454
505
|
]]) {
|
|
455
506
|
const actual = typeOf.get(id);
|
|
456
|
-
if (actual === void 0) throw new NativeParseError(FORMAT$
|
|
457
|
-
if (actual !== expected) throw new NativeParseError(FORMAT$
|
|
507
|
+
if (actual === void 0) throw new NativeParseError(FORMAT$14, number, `link.dat: the ${end} ${JSON.stringify(id)} is not a node of node.dat`);
|
|
508
|
+
if (actual !== expected) throw new NativeParseError(FORMAT$14, number, `link.dat: a ${linkType.name} link's ${end} must be a ${expected} node, and ${id} is a ${actual} node`);
|
|
458
509
|
}
|
|
459
|
-
if (!DECIMAL$
|
|
510
|
+
if (!DECIMAL$5.test(weight) || Number(weight) !== 1) throw new NativeParseError(FORMAT$14, number, `link.dat: the weight ${JSON.stringify(weight)} is not 1; a weighted HGB file would lose its weights`);
|
|
460
511
|
edges.push({
|
|
461
512
|
source,
|
|
462
513
|
target,
|
|
463
514
|
type: linkType.name
|
|
464
515
|
});
|
|
465
516
|
});
|
|
466
|
-
if (edges.length === 0) throw new NativeParseError(FORMAT$
|
|
517
|
+
if (edges.length === 0) throw new NativeParseError(FORMAT$14, 1, "link.dat: the file holds no links");
|
|
467
518
|
return {
|
|
468
519
|
nodes,
|
|
469
520
|
edges
|
|
@@ -471,7 +522,7 @@ function parseHgb(parts, options) {
|
|
|
471
522
|
}
|
|
472
523
|
//#endregion
|
|
473
524
|
//#region src/parsers/kg-triples.ts
|
|
474
|
-
const FORMAT$
|
|
525
|
+
const FORMAT$13 = "kg-triples";
|
|
475
526
|
/** The character each delimiter option splits a line on. A line is split on exactly that character, so a field cannot hide a second delimiter. */
|
|
476
527
|
const DELIMITERS$1 = {
|
|
477
528
|
tab: " ",
|
|
@@ -493,8 +544,8 @@ const QUOTE$2 = "\"";
|
|
|
493
544
|
*/
|
|
494
545
|
function value(field, quoted, where) {
|
|
495
546
|
const inner = quoted ? field.slice(1, -1) : field;
|
|
496
|
-
if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$
|
|
497
|
-
if (inner === "") throw new NativeParseError(FORMAT$
|
|
547
|
+
if (quoted && (field.length < 2 || !field.startsWith(QUOTE$2) || !field.endsWith(QUOTE$2) || inner.includes(QUOTE$2))) throw new NativeParseError(FORMAT$13, where.line, `${where.part}the field ${JSON.stringify(field)} is not one value wrapped in double quotes`);
|
|
548
|
+
if (inner === "") throw new NativeParseError(FORMAT$13, where.line, `${where.part}a field is empty`);
|
|
498
549
|
return inner;
|
|
499
550
|
}
|
|
500
551
|
/**
|
|
@@ -518,16 +569,16 @@ function parseKgTriples(parts, options) {
|
|
|
518
569
|
let lastLine = 0;
|
|
519
570
|
forEachLine(part.pieces, (line, number) => {
|
|
520
571
|
lastLine = number;
|
|
521
|
-
if (line.trim() === "") throw new NativeParseError(FORMAT$
|
|
572
|
+
if (line.trim() === "") throw new NativeParseError(FORMAT$13, number, `${label}the line is blank`);
|
|
522
573
|
if (number === 1 && options.header === "names") return;
|
|
523
574
|
if (number === 1 && options.header === "count") {
|
|
524
|
-
if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$
|
|
575
|
+
if (!COUNT$1.test(line)) throw new NativeParseError(FORMAT$13, number, `${label}expected the number of triples on the first line, found ${JSON.stringify(line)}`);
|
|
525
576
|
declared = Number(line);
|
|
526
577
|
return;
|
|
527
578
|
}
|
|
528
579
|
const fields = line.split(delimiter);
|
|
529
580
|
const [head, middle, last] = fields;
|
|
530
|
-
if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$
|
|
581
|
+
if (fields.length !== TRIPLE_FIELDS || head === void 0 || middle === void 0 || last === void 0) throw new NativeParseError(FORMAT$13, number, `${label}expected ${String(TRIPLE_FIELDS)} fields separated by ${options.delimiter}, found ${String(fields.length)}`);
|
|
531
582
|
const where = {
|
|
532
583
|
line: number,
|
|
533
584
|
part: label
|
|
@@ -553,8 +604,8 @@ function parseKgTriples(parts, options) {
|
|
|
553
604
|
});
|
|
554
605
|
triples += 1;
|
|
555
606
|
});
|
|
556
|
-
if (triples === 0) throw new NativeParseError(FORMAT$
|
|
557
|
-
if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$
|
|
607
|
+
if (triples === 0) throw new NativeParseError(FORMAT$13, lastLine, `${label}the file holds no triples`);
|
|
608
|
+
if (declared !== void 0 && declared !== triples) throw new NativeParseError(FORMAT$13, lastLine, `${label}the first line declares ${String(declared)} triples, the file holds ${String(triples)}`);
|
|
558
609
|
}
|
|
559
610
|
return {
|
|
560
611
|
nodes: order.nodes(),
|
|
@@ -563,7 +614,7 @@ function parseKgTriples(parts, options) {
|
|
|
563
614
|
}
|
|
564
615
|
//#endregion
|
|
565
616
|
//#region src/parsers/konect.ts
|
|
566
|
-
const FORMAT$
|
|
617
|
+
const FORMAT$12 = "konect";
|
|
567
618
|
/** The network formats of KONECT's first line (Kunegis, "The KONECT Handbook", table 1): undirected, directed and bipartite. */
|
|
568
619
|
const NETWORK_FORMATS = /* @__PURE__ */ new Set([
|
|
569
620
|
"sym",
|
|
@@ -588,7 +639,7 @@ const COUNT_LINE = /^%\s+(\d+)\s+(\d+)\s+(\d+)\s*$/;
|
|
|
588
639
|
/** A node id: KONECT numbers nodes from 1. */
|
|
589
640
|
const NODE_ID = /^[1-9]\d*$/;
|
|
590
641
|
/** A number as a decimal literal with an optional sign and exponent, so `Number` cannot read hex, an empty string or infinity. */
|
|
591
|
-
const DECIMAL$
|
|
642
|
+
const DECIMAL$4 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/;
|
|
592
643
|
/** The columns of an edge line (handbook section 9.1): the two nodes, then optionally the weight, and the time only when the weight is also given. */
|
|
593
644
|
const ENDPOINT_COLUMNS = 2;
|
|
594
645
|
const WEIGHTED_COLUMNS = 3;
|
|
@@ -638,9 +689,9 @@ function parseKonect(pieces, options) {
|
|
|
638
689
|
}));
|
|
639
690
|
});
|
|
640
691
|
const { header } = state;
|
|
641
|
-
if (header === void 0) throw new NativeParseError(FORMAT$
|
|
642
|
-
if (edges.length === 0) throw new NativeParseError(FORMAT$
|
|
643
|
-
if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$
|
|
692
|
+
if (header === void 0) throw new NativeParseError(FORMAT$12, lastLine, "the file has no first line naming its format and weights");
|
|
693
|
+
if (edges.length === 0) throw new NativeParseError(FORMAT$12, lastLine, "the file holds no edges");
|
|
694
|
+
if (header.counts !== void 0 && header.counts.edges !== edges.length) throw new NativeParseError(FORMAT$12, lastLine, `the second line declares ${String(header.counts.edges)} edges, the file holds ${String(edges.length)}`);
|
|
644
695
|
return {
|
|
645
696
|
nodes: order.nodes(),
|
|
646
697
|
edges
|
|
@@ -658,9 +709,9 @@ function readHeader(content, number, options) {
|
|
|
658
709
|
const match = FORMAT_LINE.exec(content);
|
|
659
710
|
const network = match?.[1];
|
|
660
711
|
const weights = match?.[2];
|
|
661
|
-
if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$
|
|
662
|
-
if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$
|
|
663
|
-
if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$
|
|
712
|
+
if (network === void 0 || weights === void 0 || !NETWORK_FORMATS.has(network)) throw new NativeParseError(FORMAT$12, number, `expected a first line "% FORMAT WEIGHTS" with FORMAT one of ${[...NETWORK_FORMATS].join(", ")}, found ${JSON.stringify(content)}`);
|
|
713
|
+
if (!WEIGHTED_TYPES.has(weights) && !UNWEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$12, number, `the weight type ${JSON.stringify(weights)} is not one this parser reads (its third column is not an edge weight or a repetition count)`);
|
|
714
|
+
if (options.weighted !== WEIGHTED_TYPES.has(weights)) throw new NativeParseError(FORMAT$12, number, `the file's weight type is ${weights}, so a recipe with weighted ${String(WEIGHTED_TYPES.has(weights))} reads it, not one with weighted ${String(options.weighted)}`);
|
|
664
715
|
return {
|
|
665
716
|
line: number,
|
|
666
717
|
network
|
|
@@ -680,15 +731,15 @@ function readEdge(content, number, context) {
|
|
|
680
731
|
const [from, to, third, time] = fields;
|
|
681
732
|
const expected = options.timestamped ? TIMESTAMPED_COLUMNS : options.weighted ? WEIGHTED_COLUMNS : void 0;
|
|
682
733
|
const fits = expected === void 0 ? fields.length === ENDPOINT_COLUMNS || fields.length === WEIGHTED_COLUMNS : fields.length === expected;
|
|
683
|
-
if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$
|
|
734
|
+
if (from === void 0 || to === void 0 || !fits) throw new NativeParseError(FORMAT$12, number, `expected ${expected === void 0 ? `${String(ENDPOINT_COLUMNS)} or ${String(WEIGHTED_COLUMNS)}` : String(expected)} columns for this recipe, found ${String(fields.length)}`);
|
|
684
735
|
const bipartite = header.network === "bip";
|
|
685
736
|
const source = endpoint(from, bipartite ? LEFT_PREFIX : "", header.counts?.left, number);
|
|
686
737
|
const target = endpoint(to, bipartite ? RIGHT_PREFIX : "", header.counts?.right, number);
|
|
687
738
|
order.add(source);
|
|
688
739
|
order.add(target);
|
|
689
|
-
if (third !== void 0 && !options.weighted && (!DECIMAL$
|
|
690
|
-
if (third !== void 0 && options.weighted && !DECIMAL$
|
|
691
|
-
if (time !== void 0 && !DECIMAL$
|
|
740
|
+
if (third !== void 0 && !options.weighted && (!DECIMAL$4.test(third) || Number(third) !== 1)) throw new NativeParseError(FORMAT$12, number, `the third column ${JSON.stringify(third)} is not 1; an edge that stands for several would be read as one`);
|
|
741
|
+
if (third !== void 0 && options.weighted && !DECIMAL$4.test(third)) throw new NativeParseError(FORMAT$12, number, `the weight ${JSON.stringify(third)} is not a number`);
|
|
742
|
+
if (time !== void 0 && !DECIMAL$4.test(time)) throw new NativeParseError(FORMAT$12, number, `the time ${JSON.stringify(time)} is not a number`);
|
|
692
743
|
const edge = {
|
|
693
744
|
source,
|
|
694
745
|
target
|
|
@@ -707,10 +758,215 @@ function readEdge(content, number, context) {
|
|
|
707
758
|
* @throws NativeParseError when the token is not a positive integer or exceeds the set's size.
|
|
708
759
|
*/
|
|
709
760
|
function endpoint(token, prefix, size, number) {
|
|
710
|
-
if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$
|
|
761
|
+
if (!NODE_ID.test(token) || size !== void 0 && Number(token) > size) throw new NativeParseError(FORMAT$12, number, `${JSON.stringify(token)} is not a node id from 1${size === void 0 ? "" : ` to ${String(size)}`}`);
|
|
711
762
|
return `${prefix}${token}`;
|
|
712
763
|
}
|
|
713
764
|
//#endregion
|
|
765
|
+
//#region src/parsers/ogb.ts
|
|
766
|
+
const FORMAT$11 = "ogb";
|
|
767
|
+
/** The files of an OGB raw directory this reader takes, by their path under the dataset's root directory. A relation's files sit in `raw/relations/<head>___<relation>___<tail>/`. */
|
|
768
|
+
const RAW = /^(?<root>(?:[^/]+\/)*?)raw\/(?<name>num-node-list|num-node-dict|edge|num-edge-list|triplet-type-list|relations\/[^/]+\/(?:edge|num-edge-list))\.csv\.gz$/u;
|
|
769
|
+
const INDEX = /^(?:0|[1-9]\d*)$/;
|
|
770
|
+
/** The fields of a line of `triplet-type-list.csv.gz`: a head type, a relation and a tail type. */
|
|
771
|
+
const TRIPLET_PARTS = 3;
|
|
772
|
+
/**
|
|
773
|
+
* Orders two names by code point, the order Python's `sorted` gives the ASCII type and relation names OGB uses.
|
|
774
|
+
* @param left - A name.
|
|
775
|
+
* @param right - Another.
|
|
776
|
+
* @returns Negative, zero or positive.
|
|
777
|
+
*/
|
|
778
|
+
function compare(left, right) {
|
|
779
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* Visits the non-blank lines of a file, read a piece at a time.
|
|
783
|
+
* @param file - The file.
|
|
784
|
+
* @param visit - Called with each non-blank line and its number.
|
|
785
|
+
*/
|
|
786
|
+
function forEachRow(file, visit) {
|
|
787
|
+
forEachLine(file.text(), (text, line) => {
|
|
788
|
+
if (text.trim() !== "") visit({
|
|
789
|
+
text,
|
|
790
|
+
line
|
|
791
|
+
});
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
/**
|
|
795
|
+
* The non-blank lines of a small file, such as a count or the triplet list.
|
|
796
|
+
* @param file - The file.
|
|
797
|
+
* @returns Each non-blank line and its number.
|
|
798
|
+
*/
|
|
799
|
+
function rows(file) {
|
|
800
|
+
const all = [];
|
|
801
|
+
forEachRow(file, (row) => {
|
|
802
|
+
all.push(row);
|
|
803
|
+
});
|
|
804
|
+
return all;
|
|
805
|
+
}
|
|
806
|
+
/**
|
|
807
|
+
* Reads a file that holds one line, such as a count.
|
|
808
|
+
* @param file - The file.
|
|
809
|
+
* @returns The line's text.
|
|
810
|
+
* @throws NativeParseError when the file holds no line or more than one.
|
|
811
|
+
*/
|
|
812
|
+
function onlyLine(file) {
|
|
813
|
+
const all = rows(file);
|
|
814
|
+
const [only] = all;
|
|
815
|
+
if (all.length !== 1 || only === void 0) throw new NativeParseError(FORMAT$11, all[1]?.line ?? 1, `${file.path} must hold one line, found ${String(all.length)}`);
|
|
816
|
+
return only.text.trim();
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Reads a non-negative integer.
|
|
820
|
+
* @param text - The field.
|
|
821
|
+
* @param file - The file, for the error.
|
|
822
|
+
* @param line - The 1-based line, for the error.
|
|
823
|
+
* @returns The integer.
|
|
824
|
+
* @throws NativeParseError when the field is not a non-negative integer.
|
|
825
|
+
*/
|
|
826
|
+
function index(text, file, line) {
|
|
827
|
+
const trimmed = text.trim();
|
|
828
|
+
if (!INDEX.test(trimmed)) throw new NativeParseError(FORMAT$11, line, `${file.path}: expected a non-negative integer, found ${JSON.stringify(text)}`);
|
|
829
|
+
return Number(trimmed);
|
|
830
|
+
}
|
|
831
|
+
/**
|
|
832
|
+
* Reads an `edge.csv.gz` file, one `<source>,<target>` pair of node indices per line, checking each index against its node type's count and the line count against the relation's `num-edge-list.csv.gz`.
|
|
833
|
+
* @param edges - The edge file.
|
|
834
|
+
* @param count - The relation's `num-edge-list.csv.gz`.
|
|
835
|
+
* @param ends - The source and target node types' counts and how each index is named.
|
|
836
|
+
* @param emit - Called with each edge, in file order, so a large file is never held twice.
|
|
837
|
+
* @throws NativeParseError naming the file and line for a line that is not a pair of indices, an index outside its type's nodes, or a line count other than the recorded one.
|
|
838
|
+
*/
|
|
839
|
+
function readEdges(edges, count, ends, emit) {
|
|
840
|
+
const recorded = index(onlyLine(count), count, 1);
|
|
841
|
+
let read = 0;
|
|
842
|
+
forEachRow(edges, ({ text, line }) => {
|
|
843
|
+
const fields = text.split(",");
|
|
844
|
+
const [first, second] = fields;
|
|
845
|
+
if (fields.length !== 2 || first === void 0 || second === void 0) throw new NativeParseError(FORMAT$11, line, `${edges.path}: expected a source and a target index, found ${JSON.stringify(text)}`);
|
|
846
|
+
const source = index(first, edges, line);
|
|
847
|
+
const target = index(second, edges, line);
|
|
848
|
+
if (source >= ends.sources || target >= ends.targets) throw new NativeParseError(FORMAT$11, line, `${edges.path}: edge ${text} names a node outside the ${String(ends.sources)} sources and ${String(ends.targets)} targets the dataset records`);
|
|
849
|
+
emit({
|
|
850
|
+
source: ends.source(source),
|
|
851
|
+
target: ends.target(target)
|
|
852
|
+
});
|
|
853
|
+
read += 1;
|
|
854
|
+
});
|
|
855
|
+
if (read !== recorded) throw new NativeParseError(FORMAT$11, read, `${edges.path} holds ${String(read)} edges, but ${count.path} records ${String(recorded)}`);
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Takes one file of the dataset directory by its name under `raw/`.
|
|
859
|
+
* @param files - The files by name.
|
|
860
|
+
* @param name - The name, such as `edge` or `relations/paper___cites___paper/edge`.
|
|
861
|
+
* @param root - The dataset's root directory, for the error.
|
|
862
|
+
* @returns The file.
|
|
863
|
+
* @throws NativeParseError when the archive's selection holds no such file.
|
|
864
|
+
*/
|
|
865
|
+
function required$1(files, name, root) {
|
|
866
|
+
const file = files.get(name);
|
|
867
|
+
if (file === void 0) throw new NativeParseError(FORMAT$11, 1, `the dataset directory ${root} has no raw/${name}.csv.gz among the files the recipe selects`);
|
|
868
|
+
return file;
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* Parses an Open Graph Benchmark dataset from the files of its raw directory, as OGB's own `read_csv_graph_raw` and `read_csv_heterograph_raw` read them (ogb/io/read_graph_raw.py in snap-stanford/ogb). A homogeneous dataset has `raw/num-node-list.csv.gz` (its one graph's node count), `raw/edge.csv.gz` (one `<source>,<target>` pair of node indices per line, no header) and `raw/num-edge-list.csv.gz`: its nodes are `0` to `N - 1`, isolated ones included, and its edges the pairs in file order. A heterogeneous dataset has `raw/num-node-dict.csv.gz` (a header of node types over one row of their counts), `raw/triplet-type-list.csv.gz` (one `<head>,<relation>,<tail>` type triplet per line) and, per triplet, `raw/relations/<head>___<relation>___<tail>/edge.csv.gz` and `num-edge-list.csv.gz`, whose indices count within the head and tail types: its nodes are `<type>:<index>` for each type in sorted order and each index from 0, and its edges are each relation's pairs in file order, relations in the sorted order of their triplets, which is the order OGB reads them in. Edges keep the file's direction. Relation names, node features, labels and splits are not carried, as no graph parser in this version carries more than topology.
|
|
872
|
+
* @param files - The files the recipe's archive selects, gunzipped and decoded: every one must be a file of one dataset's raw directory named above.
|
|
873
|
+
* @returns The native graph.
|
|
874
|
+
* @throws NativeParseError naming the file and line for a file outside the raw layout, files of two datasets, a dataset with neither or both node-count files, a missing file, a graph count other than one, a malformed count or triplet, a relation directory the triplet list does not name, an edge that is not a pair of indices of the dataset's nodes, or an edge file whose line count differs from its recorded count.
|
|
875
|
+
*/
|
|
876
|
+
function parseOgb(files) {
|
|
877
|
+
const byName = /* @__PURE__ */ new Map();
|
|
878
|
+
const roots = /* @__PURE__ */ new Set();
|
|
879
|
+
for (const file of files) {
|
|
880
|
+
const match = RAW.exec(file.path);
|
|
881
|
+
const name = match?.groups?.["name"];
|
|
882
|
+
const root = match?.groups?.["root"];
|
|
883
|
+
if (name === void 0 || root === void 0) throw new NativeParseError(FORMAT$11, 1, `${file.path} is not a file of an OGB raw directory this reader takes`);
|
|
884
|
+
roots.add(root);
|
|
885
|
+
byName.set(name, file);
|
|
886
|
+
}
|
|
887
|
+
const [root] = roots;
|
|
888
|
+
if (roots.size !== 1 || root === void 0) throw new NativeParseError(FORMAT$11, 1, `the recipe must select the raw files of one dataset, found ${roots.size === 0 ? "none" : `the directories ${[...roots].join(", ")}`}`);
|
|
889
|
+
const homogeneous = byName.get("num-node-list");
|
|
890
|
+
const heterogeneous = byName.get("num-node-dict");
|
|
891
|
+
const neitherOrBoth = `the dataset directory ${root} must have exactly one of raw/num-node-list.csv.gz (one node type) and raw/num-node-dict.csv.gz (several)`;
|
|
892
|
+
const edges = [];
|
|
893
|
+
if (homogeneous !== void 0) {
|
|
894
|
+
if (heterogeneous !== void 0) throw new NativeParseError(FORMAT$11, 1, neitherOrBoth);
|
|
895
|
+
const [relation] = [...byName.keys()].filter((name) => name.startsWith("relations/"));
|
|
896
|
+
if (relation !== void 0 || byName.has("triplet-type-list")) throw new NativeParseError(FORMAT$11, 1, `the dataset directory ${root} has raw/num-node-list.csv.gz, so it has one node type, but the recipe selects ${relation === void 0 ? "raw/triplet-type-list.csv.gz" : `raw/${relation}.csv.gz`} as well`);
|
|
897
|
+
const count = index(onlyLine(homogeneous), homogeneous, 1);
|
|
898
|
+
const name = (node) => String(node);
|
|
899
|
+
readEdges(required$1(byName, "edge", root), required$1(byName, "num-edge-list", root), {
|
|
900
|
+
sources: count,
|
|
901
|
+
targets: count,
|
|
902
|
+
source: name,
|
|
903
|
+
target: name
|
|
904
|
+
}, (edge) => {
|
|
905
|
+
edges.push(edge);
|
|
906
|
+
});
|
|
907
|
+
return {
|
|
908
|
+
nodes: Array.from({ length: count }, (_, node) => ({ id: String(node) })),
|
|
909
|
+
edges
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
if (heterogeneous === void 0) throw new NativeParseError(FORMAT$11, 1, neitherOrBoth);
|
|
913
|
+
for (const stray of ["edge", "num-edge-list"]) if (byName.has(stray)) throw new NativeParseError(FORMAT$11, 1, `the dataset directory ${root} has raw/num-node-dict.csv.gz, so its edges are in raw/relations/, but the recipe selects raw/${stray}.csv.gz as well`);
|
|
914
|
+
const header = rows(heterogeneous);
|
|
915
|
+
const [types, counts] = header;
|
|
916
|
+
if (header.length !== 2 || types === void 0 || counts === void 0) throw new NativeParseError(FORMAT$11, header[2]?.line ?? 1, `${heterogeneous.path} must hold a header of node types and one row of their counts, found ${String(header.length)} lines`);
|
|
917
|
+
const typeNames = types.text.split(",").map((type) => type.trim());
|
|
918
|
+
const typeCounts = counts.text.split(",");
|
|
919
|
+
if (typeNames.length !== typeCounts.length || typeNames.some((type) => type === "") || new Set(typeNames).size !== typeNames.length) throw new NativeParseError(FORMAT$11, counts.line, `${heterogeneous.path} must name each node type once, over one count per type`);
|
|
920
|
+
const nodeTypes = typeNames.map((type, position) => {
|
|
921
|
+
const count = typeCounts[position];
|
|
922
|
+
if (count === void 0) throw new NativeParseError(FORMAT$11, counts.line, `${heterogeneous.path} gives node type ${type} no count`);
|
|
923
|
+
return {
|
|
924
|
+
type,
|
|
925
|
+
count: index(count, heterogeneous, counts.line)
|
|
926
|
+
};
|
|
927
|
+
});
|
|
928
|
+
nodeTypes.sort((left, right) => compare(left.type, right.type));
|
|
929
|
+
const nodeCounts = new Map(nodeTypes.map(({ type, count }) => [type, count]));
|
|
930
|
+
const triplets = required$1(byName, "triplet-type-list", root);
|
|
931
|
+
const relations = [];
|
|
932
|
+
for (const { text, line } of rows(triplets)) {
|
|
933
|
+
const parts = text.split(",").map((part) => part.trim());
|
|
934
|
+
const [head, relation, tail] = parts;
|
|
935
|
+
if (parts.length !== TRIPLET_PARTS || head === void 0 || relation === void 0 || tail === void 0 || relation === "") throw new NativeParseError(FORMAT$11, line, `${triplets.path}: expected a head type, a relation and a tail type, found ${JSON.stringify(text)}`);
|
|
936
|
+
const sources = nodeCounts.get(head);
|
|
937
|
+
const targets = nodeCounts.get(tail);
|
|
938
|
+
if (sources === void 0 || targets === void 0) throw new NativeParseError(FORMAT$11, line, `${triplets.path}: ${text} names a node type that ${heterogeneous.path} does not count`);
|
|
939
|
+
relations.push({
|
|
940
|
+
head,
|
|
941
|
+
relation,
|
|
942
|
+
tail,
|
|
943
|
+
sources,
|
|
944
|
+
targets
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
relations.sort((left, right) => compare(left.head, right.head) || compare(left.relation, right.relation) || compare(left.tail, right.tail));
|
|
948
|
+
const listed = new Set(relations.map(({ head, relation, tail }) => `relations/${head}___${relation}___${tail}`));
|
|
949
|
+
for (const name of byName.keys()) {
|
|
950
|
+
const directory = name.slice(0, name.lastIndexOf("/"));
|
|
951
|
+
if (name.startsWith("relations/") && !listed.has(directory)) throw new NativeParseError(FORMAT$11, 1, `raw/${directory}/ is not one of the relations ${triplets.path} lists`);
|
|
952
|
+
}
|
|
953
|
+
for (const { head, relation, tail, sources, targets } of relations) {
|
|
954
|
+
const directory = `relations/${head}___${relation}___${tail}`;
|
|
955
|
+
readEdges(required$1(byName, `${directory}/edge`, root), required$1(byName, `${directory}/num-edge-list`, root), {
|
|
956
|
+
sources,
|
|
957
|
+
targets,
|
|
958
|
+
source: (node) => `${head}:${String(node)}`,
|
|
959
|
+
target: (node) => `${tail}:${String(node)}`
|
|
960
|
+
}, (edge) => {
|
|
961
|
+
edges.push(edge);
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
return {
|
|
965
|
+
nodes: nodeTypes.flatMap(({ type, count }) => Array.from({ length: count }, (_, node) => ({ id: `${type}:${String(node)}` }))),
|
|
966
|
+
edges
|
|
967
|
+
};
|
|
968
|
+
}
|
|
969
|
+
//#endregion
|
|
714
970
|
//#region src/parsers/matrix-market.ts
|
|
715
971
|
/** How many tokens an entry line of each field type holds: the coordinates, then the value's parts (none for a pattern, a real and an imaginary part for a complex value). */
|
|
716
972
|
const tokensPerEntry = /* @__PURE__ */ new Map([
|
|
@@ -808,7 +1064,7 @@ const DELIMITERS = {
|
|
|
808
1064
|
semicolon: ";"
|
|
809
1065
|
};
|
|
810
1066
|
/** A number as a decimal literal, with an optional sign and exponent. `Number` alone would also accept hex, binary, `Infinity` and the empty string. */
|
|
811
|
-
const DECIMAL$
|
|
1067
|
+
const DECIMAL$3 = /^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/;
|
|
812
1068
|
/** The byte order mark some exporters put before the first header. */
|
|
813
1069
|
const BYTE_ORDER_MARK = "";
|
|
814
1070
|
/**
|
|
@@ -952,7 +1208,7 @@ var RecordScanner = class {
|
|
|
952
1208
|
* @returns The records in order, blank lines included as a record of one empty field.
|
|
953
1209
|
* @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
|
|
954
1210
|
*/
|
|
955
|
-
function* records(pieces, delimiter, firstLine) {
|
|
1211
|
+
function* records$1(pieces, delimiter, firstLine) {
|
|
956
1212
|
const scanner = new RecordScanner(delimiter.charCodeAt(0), firstLine);
|
|
957
1213
|
let carry = "";
|
|
958
1214
|
for (const piece of pieces) {
|
|
@@ -971,7 +1227,7 @@ function* records(pieces, delimiter, firstLine) {
|
|
|
971
1227
|
* @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
|
|
972
1228
|
*/
|
|
973
1229
|
function csvGrid(text) {
|
|
974
|
-
return [...records([text], ",", 1)].map((record) => record.fields);
|
|
1230
|
+
return [...records$1([text], ",", 1)].map((record) => record.fields);
|
|
975
1231
|
}
|
|
976
1232
|
/**
|
|
977
1233
|
* Drops the first physical lines of a file given in pieces.
|
|
@@ -1020,7 +1276,7 @@ function* withoutByteOrderMark(text) {
|
|
|
1020
1276
|
*/
|
|
1021
1277
|
function numeric$1(cell, factor, column, line) {
|
|
1022
1278
|
if (cell === "") return null;
|
|
1023
|
-
if (!DECIMAL$
|
|
1279
|
+
if (!DECIMAL$3.test(cell)) throw new NativeParseError("csv-table", line, `column ${column} is declared numeric but holds ${JSON.stringify(cell)}`);
|
|
1024
1280
|
return Number(cell) * factor;
|
|
1025
1281
|
}
|
|
1026
1282
|
/**
|
|
@@ -1032,7 +1288,7 @@ function numeric$1(cell, factor, column, line) {
|
|
|
1032
1288
|
*/
|
|
1033
1289
|
function* tableRecords(text, options) {
|
|
1034
1290
|
const body = afterLines(withoutByteOrderMark(text), options.skipLines);
|
|
1035
|
-
for (const record of records(body, DELIMITERS[options.delimiter], options.skipLines + 1)) if (record.fields.length > 1 || record.fields[0]?.trim() !== "") yield record;
|
|
1291
|
+
for (const record of records$1(body, DELIMITERS[options.delimiter], options.skipLines + 1)) if (record.fields.length > 1 || record.fields[0]?.trim() !== "") yield record;
|
|
1036
1292
|
}
|
|
1037
1293
|
/**
|
|
1038
1294
|
* Parses a delimited table into rows (a cell whose text is one of the recipe's `nullValues` is empty, so null): the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
|
|
@@ -2543,8 +2799,8 @@ const PREDEFINED = {
|
|
|
2543
2799
|
apos: "'"
|
|
2544
2800
|
};
|
|
2545
2801
|
const CODE_POINT = /^#(?:x([0-9A-Fa-f]+)|(\d+))$/;
|
|
2546
|
-
const HEXADECIMAL$
|
|
2547
|
-
const DECIMAL$
|
|
2802
|
+
const HEXADECIMAL$2 = 16;
|
|
2803
|
+
const DECIMAL$2 = 10;
|
|
2548
2804
|
/** The markup that runs from an opening marker to a closing one and holds no elements: comments, CDATA sections and processing instructions. */
|
|
2549
2805
|
const COMMENT = {
|
|
2550
2806
|
open: "<!--",
|
|
@@ -2591,7 +2847,7 @@ var XmlReader = class {
|
|
|
2591
2847
|
if (predefined !== void 0) return predefined;
|
|
2592
2848
|
const code = CODE_POINT.exec(reference);
|
|
2593
2849
|
if (code === null) return this.#fail(index, `the entity ${whole} is not one of the five predefined ones`);
|
|
2594
|
-
return String.fromCodePoint(code[1] === void 0 ? Number.parseInt(code[2] ?? "", DECIMAL$
|
|
2850
|
+
return String.fromCodePoint(code[1] === void 0 ? Number.parseInt(code[2] ?? "", DECIMAL$2) : Number.parseInt(code[1], HEXADECIMAL$2));
|
|
2595
2851
|
});
|
|
2596
2852
|
}
|
|
2597
2853
|
/**
|
|
@@ -2839,6 +3095,47 @@ function parseVrpRep(text) {
|
|
|
2839
3095
|
};
|
|
2840
3096
|
}
|
|
2841
3097
|
//#endregion
|
|
3098
|
+
//#region src/parsers/witvliet-synapses.ts
|
|
3099
|
+
/** One chemical synapse of the file: the presynaptic neuron and every neuron it contacts. The file's other fields (CATMAID and VAST ids, the synapse's size and its partners' shares of it) are not read. */
|
|
3100
|
+
const SynapseSchema = z.object({
|
|
3101
|
+
pre: z.string().min(1),
|
|
3102
|
+
post: z.array(z.string().min(1)).min(1)
|
|
3103
|
+
});
|
|
3104
|
+
/**
|
|
3105
|
+
* Parses a Witvliet synapse file: a JSON array of chemical synapses, each a `pre` neuron and its `post` partners. A synapse is polyadic, one presynaptic site facing several partners, and each partner is one edge from `pre` to it, so a synapse with three partners is three edges and a partner listed twice is two. Nodes are the neurons in order of first appearance, `pre` before its partners.
|
|
3106
|
+
* @param text - The file text.
|
|
3107
|
+
* @returns The native graph.
|
|
3108
|
+
* @throws NativeParseError, numbering the synapse (from 1) in place of a line, when the text is not JSON, not an array, a synapse lacks its `pre` or `post` or names an empty neuron, or the file holds no synapse.
|
|
3109
|
+
*/
|
|
3110
|
+
function parseWitvlietSynapses(text) {
|
|
3111
|
+
let document;
|
|
3112
|
+
try {
|
|
3113
|
+
document = JSON.parse(text);
|
|
3114
|
+
} catch (error) {
|
|
3115
|
+
throw new NativeParseError("witvliet-synapses", 1, `the file is not JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
3116
|
+
}
|
|
3117
|
+
if (!Array.isArray(document)) throw new NativeParseError("witvliet-synapses", 1, "the file is not a JSON array of synapses");
|
|
3118
|
+
const order = new NodeOrder();
|
|
3119
|
+
const edges = [];
|
|
3120
|
+
for (const [index, item] of document.entries()) {
|
|
3121
|
+
const synapse = SynapseSchema.safeParse(item);
|
|
3122
|
+
if (!synapse.success) throw new NativeParseError("witvliet-synapses", index + 1, `synapse ${String(index + 1)} is not a pre neuron with a non-empty list of post neurons: ${synapse.error.issues.map((issue) => `${issue.path.join(".")} ${issue.message}`).join("; ")}`);
|
|
3123
|
+
order.add(synapse.data.pre);
|
|
3124
|
+
for (const post of synapse.data.post) {
|
|
3125
|
+
order.add(post);
|
|
3126
|
+
edges.push({
|
|
3127
|
+
source: synapse.data.pre,
|
|
3128
|
+
target: post
|
|
3129
|
+
});
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
if (edges.length === 0) throw new NativeParseError("witvliet-synapses", 1, "the file holds no synapses");
|
|
3133
|
+
return {
|
|
3134
|
+
nodes: order.nodes(),
|
|
3135
|
+
edges
|
|
3136
|
+
};
|
|
3137
|
+
}
|
|
3138
|
+
//#endregion
|
|
2842
3139
|
//#region src/parsers/parse-native.ts
|
|
2843
3140
|
/** Bytes converted per String.fromCharCode call: spreading a whole file would exceed the engine's argument-count limit, and this many stays well inside every engine's. */
|
|
2844
3141
|
const LATIN1_CHUNK = 32768;
|
|
@@ -2897,13 +3194,14 @@ function parseNative(bytes, parser) {
|
|
|
2897
3194
|
pieces: decodePieces(bytes, parser)
|
|
2898
3195
|
}], parser);
|
|
2899
3196
|
if (parser.format === "hgb") throw new NativeParseError(parser.format, 1, "an HGB dataset is two files, nodes and links, which a recipe reads from a parts archive");
|
|
3197
|
+
if (parser.format === "edge-list") return parseEdgeList(decodePieces(bytes, parser), parser);
|
|
2900
3198
|
const text = decode(bytes, parser);
|
|
2901
3199
|
if (parser.format === "geojson") return parseGeojson(text, parser);
|
|
2902
|
-
if (parser.format === "edge-list") return parseEdgeList(text, parser);
|
|
2903
3200
|
if (parser.format === "matrix-market") return parseMatrixMarket(text);
|
|
2904
3201
|
if (parser.format === "gml") return parseGml(text);
|
|
2905
3202
|
if (parser.format === "ucinet-dl") return parseUcinetDl(text, parser);
|
|
2906
3203
|
if (parser.format === "gd-contest") return parseGdContest(text);
|
|
3204
|
+
if (parser.format === "witvliet-synapses") return parseWitvlietSynapses(text);
|
|
2907
3205
|
if (parser.format === "customer-points") return parseCustomerPoints(text);
|
|
2908
3206
|
if (parser.format === "solomon") return parseSolomon(text);
|
|
2909
3207
|
if (parser.format === "top") return parseTop(text);
|
|
@@ -2912,6 +3210,7 @@ function parseNative(bytes, parser) {
|
|
|
2912
3210
|
if (parser.format === "verolog-2019") return parseVerolog2019(text);
|
|
2913
3211
|
if (parser.format === "vrp-rep") return parseVrpRep(text);
|
|
2914
3212
|
if (parser.format === "hhcrsp") return parseHhcrsp(text);
|
|
3213
|
+
if (parser.format === "ogb") throw new NativeParseError(parser.format, 1, "the format reads the several files a files archive selects, not one file");
|
|
2915
3214
|
return parsePajek(text);
|
|
2916
3215
|
}
|
|
2917
3216
|
/**
|
|
@@ -2937,6 +3236,20 @@ function parseNativeParts(parts, parser) {
|
|
|
2937
3236
|
}
|
|
2938
3237
|
throw new NativeParseError(parser.format, 1, `the ${parser.format} parser reads one file, not named parts`);
|
|
2939
3238
|
}
|
|
3239
|
+
/**
|
|
3240
|
+
* Reads the several files of a multi-file format into the one native document they make together.
|
|
3241
|
+
* @param files - The files a `files` archive selects, each with its path in the archive, in path order.
|
|
3242
|
+
* @param parser - The recipe's parser, which must be a multi-file format.
|
|
3243
|
+
* @returns The native graph.
|
|
3244
|
+
* @throws NativeParseError when a file is not valid in the stated encoding or the files are not in the stated format, or when the parser reads one file and not several.
|
|
3245
|
+
*/
|
|
3246
|
+
function parseNativeFiles(files, parser) {
|
|
3247
|
+
if (parser.format !== "ogb") throw new NativeParseError(parser.format, 1, "the format reads one file, not the several a files archive selects");
|
|
3248
|
+
return parseOgb(files.map((file) => ({
|
|
3249
|
+
path: file.path,
|
|
3250
|
+
text: () => decodePieces(file.bytes, parser)
|
|
3251
|
+
})));
|
|
3252
|
+
}
|
|
2940
3253
|
//#endregion
|
|
2941
3254
|
//#region src/node/bzip2.ts
|
|
2942
3255
|
/** The four bytes every bzip2 stream starts with, as ASCII: `BZh` and the block size digit, `1` to `9`. */
|
|
@@ -3564,23 +3877,10 @@ function readTar(bytes) {
|
|
|
3564
3877
|
return members;
|
|
3565
3878
|
}
|
|
3566
3879
|
//#endregion
|
|
3567
|
-
//#region src/node/
|
|
3568
|
-
|
|
3569
|
-
const
|
|
3570
|
-
const
|
|
3571
|
-
const FIRST_LETTER = "A".charCodeAt(0);
|
|
3572
|
-
const SECONDS_PER_DAY = 86400;
|
|
3573
|
-
const MILLISECONDS = 1e3;
|
|
3574
|
-
/** The serial number Excel gives its 29th of February 1900, a day that never existed: serials above it count from 30 December 1899 and serials below it from 31 December 1899. */
|
|
3575
|
-
const LEAP_BUG_SERIAL = 60;
|
|
3576
|
-
/** The day a serial number counts from: 1 January 1904 in a workbook that says so, otherwise 31 December 1899 for the serials before Excel's false 29th of February 1900 and 30 December 1899 for those after it. */
|
|
3577
|
-
const EPOCH_1904 = "1904-01-01T00:00:00Z";
|
|
3578
|
-
const EPOCH_BEFORE_LEAP_BUG = "1899-12-31T00:00:00Z";
|
|
3579
|
-
const EPOCH_AFTER_LEAP_BUG = "1899-12-30T00:00:00Z";
|
|
3580
|
-
const DATE_LENGTH = 10;
|
|
3581
|
-
const DATE_TIME_LENGTH = 19;
|
|
3582
|
-
const SECONDS_PER_MINUTE = 60;
|
|
3583
|
-
const MINUTES_PER_HOUR = 60;
|
|
3880
|
+
//#region src/node/xml-part.ts
|
|
3881
|
+
/** Helpers shared by the readers of spreadsheet formats that store their parts as XML (xlsx and OpenDocument): they read the flat, namespace-prefixed markup those parts are written in, without building a tree. */
|
|
3882
|
+
const HEXADECIMAL$1 = 16;
|
|
3883
|
+
const DECIMAL$1 = 10;
|
|
3584
3884
|
/**
|
|
3585
3885
|
* Replaces the XML entities and character references of a text.
|
|
3586
3886
|
* @param text - Escaped text.
|
|
@@ -3588,8 +3888,8 @@ const MINUTES_PER_HOUR = 60;
|
|
|
3588
3888
|
*/
|
|
3589
3889
|
function unescape(text) {
|
|
3590
3890
|
return text.replaceAll(/&(#x[0-9A-Fa-f]+|#[0-9]+|amp|lt|gt|quot|apos);/g, (_whole, entity) => {
|
|
3591
|
-
if (entity.startsWith("#x")) return String.fromCodePoint(Number.parseInt(entity.slice(2), HEXADECIMAL));
|
|
3592
|
-
if (entity.startsWith("#")) return String.fromCodePoint(Number.parseInt(entity.slice(1), DECIMAL));
|
|
3891
|
+
if (entity.startsWith("#x")) return String.fromCodePoint(Number.parseInt(entity.slice(2), HEXADECIMAL$1));
|
|
3892
|
+
if (entity.startsWith("#")) return String.fromCodePoint(Number.parseInt(entity.slice(1), DECIMAL$1));
|
|
3593
3893
|
return {
|
|
3594
3894
|
amp: "&",
|
|
3595
3895
|
lt: "<",
|
|
@@ -3618,16 +3918,156 @@ function attributesOf(tag) {
|
|
|
3618
3918
|
function elements(xml, name) {
|
|
3619
3919
|
return [...xml.matchAll(new RegExp(`<${name}(?=[\\s/>])([^>]*)>`, "g"))].map((match) => attributesOf(match[1] ?? ""));
|
|
3620
3920
|
}
|
|
3921
|
+
//#endregion
|
|
3922
|
+
//#region src/node/ods.ts
|
|
3923
|
+
/** The part of an OpenDocument spreadsheet that holds its tables ([ODF 1.3] part 2, 3.1.3). */
|
|
3924
|
+
const CONTENT_PART = "content.xml";
|
|
3925
|
+
const DECIMAL = 10;
|
|
3926
|
+
const SECONDS_PER_MINUTE$1 = 60;
|
|
3927
|
+
const MINUTES_PER_HOUR$1 = 60;
|
|
3928
|
+
/** An `office:time-value`, an ISO 8601 duration of hours, minutes and seconds such as `PT13H05M00S` (the form spreadsheet applications write for a time of day, [ODF 1.3] part 3, 19.389). */
|
|
3929
|
+
const TIME_VALUE = /^PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?$/;
|
|
3930
|
+
/** The value types whose value is the number in `office:value` ([ODF 1.3] part 3, 19.387). */
|
|
3931
|
+
const NUMERIC_TYPES = /* @__PURE__ */ new Set([
|
|
3932
|
+
"float",
|
|
3933
|
+
"percentage",
|
|
3934
|
+
"currency"
|
|
3935
|
+
]);
|
|
3621
3936
|
/**
|
|
3622
|
-
* The
|
|
3623
|
-
* @param
|
|
3624
|
-
* @
|
|
3937
|
+
* The text of a cell's paragraphs, as a spreadsheet shows it: paragraphs joined by line breaks, `text:s` as its count of spaces, `text:tab` as a tab and `text:line-break` as a line break, the text of spans and links kept and annotations left out.
|
|
3938
|
+
* @param body - The XML between the cell's start and end tags.
|
|
3939
|
+
* @param row - The 1-based row, for the error.
|
|
3940
|
+
* @returns The text.
|
|
3941
|
+
* @throws ArchiveError when a run of spaces states a count that is not a positive whole number.
|
|
3625
3942
|
*/
|
|
3626
|
-
function
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3943
|
+
function cellText(body, row) {
|
|
3944
|
+
const paragraphs = [];
|
|
3945
|
+
for (const paragraph of body.replaceAll(/<office:annotation[\s>][\s\S]*?<\/office:annotation>/g, "").matchAll(/<text:(?:p|h)(?=[\s/>])[^>]*?(?:\/>|>([\s\S]*?)<\/text:(?:p|h)>)/g)) {
|
|
3946
|
+
const inner = (paragraph[1] ?? "").replaceAll(/<text:s(?=[\s/>])([^>]*?)\/?>/g, (_whole, attributes) => " ".repeat(repeats(attributesOf(attributes), "text:c", row))).replaceAll(/<text:tab(?=[\s/>])[^>]*?\/?>/g, " ").replaceAll(/<text:line-break(?=[\s/>])[^>]*?\/?>/g, "\n").replaceAll(/<[^>]*>/g, "");
|
|
3947
|
+
paragraphs.push(unescape(inner));
|
|
3948
|
+
}
|
|
3949
|
+
return paragraphs.join("\n");
|
|
3630
3950
|
}
|
|
3951
|
+
/**
|
|
3952
|
+
* A time value as the clock text the xlsx reader gives a time alone, `hh:mm:ss`, with the seconds rounded to whole seconds and an hour count past a day kept as it stands.
|
|
3953
|
+
* @param duration - The `office:time-value`.
|
|
3954
|
+
* @param row - The 1-based row, for the error.
|
|
3955
|
+
* @returns The text.
|
|
3956
|
+
* @throws ArchiveError when the value is not an hours, minutes and seconds duration.
|
|
3957
|
+
*/
|
|
3958
|
+
function timeText(duration, row) {
|
|
3959
|
+
const match = TIME_VALUE.exec(duration);
|
|
3960
|
+
if (match === null) throw new ArchiveError(`row ${String(row)} has a time cell holding ${duration}, which is not a duration of hours, minutes and seconds`);
|
|
3961
|
+
const total = Math.round(Number.parseInt(match[1] ?? "0", DECIMAL) * MINUTES_PER_HOUR$1 * SECONDS_PER_MINUTE$1 + Number.parseInt(match[2] ?? "0", DECIMAL) * SECONDS_PER_MINUTE$1 + Number(match[3] ?? "0"));
|
|
3962
|
+
return [
|
|
3963
|
+
Math.floor(total / 3600),
|
|
3964
|
+
Math.floor(total / SECONDS_PER_MINUTE$1) % MINUTES_PER_HOUR$1,
|
|
3965
|
+
total % SECONDS_PER_MINUTE$1
|
|
3966
|
+
].map((part) => String(part).padStart(2, "0")).join(":");
|
|
3967
|
+
}
|
|
3968
|
+
/**
|
|
3969
|
+
* The text a cell gives the CSV, by its value type: the stored number for a number, percentage or currency, the stored ISO 8601 date (or date and time) for a date, `hh:mm:ss` for a time, TRUE or FALSE for a boolean, the shown text for a string or an error, and nothing for a cell with no value.
|
|
3970
|
+
* @param attributes - The cell's attributes.
|
|
3971
|
+
* @param body - The XML between its tags.
|
|
3972
|
+
* @param row - The 1-based row, for errors.
|
|
3973
|
+
* @returns The text, empty for no value.
|
|
3974
|
+
* @throws ArchiveError when a typed cell lacks the attribute its type stores its value in.
|
|
3975
|
+
*/
|
|
3976
|
+
function valueText(attributes, body, row) {
|
|
3977
|
+
const type = attributes["office:value-type"];
|
|
3978
|
+
const stored = (name) => {
|
|
3979
|
+
const value = attributes[name];
|
|
3980
|
+
if (value === void 0) throw new ArchiveError(`row ${String(row)} has a ${type ?? ""} cell with no ${name}`);
|
|
3981
|
+
return value;
|
|
3982
|
+
};
|
|
3983
|
+
if (type === void 0 || type === "string" || attributes["calcext:value-type"] === "error") return cellText(body, row);
|
|
3984
|
+
if (NUMERIC_TYPES.has(type)) return stored("office:value");
|
|
3985
|
+
if (type === "date") return stored("office:date-value");
|
|
3986
|
+
if (type === "time") return timeText(stored("office:time-value"), row);
|
|
3987
|
+
if (type === "boolean") return stored("office:boolean-value") === "true" ? "TRUE" : "FALSE";
|
|
3988
|
+
throw new ArchiveError(`row ${String(row)} has a cell of value type ${type}, which OpenDocument does not define`);
|
|
3989
|
+
}
|
|
3990
|
+
/**
|
|
3991
|
+
* How many times a row or cell repeats.
|
|
3992
|
+
* @param attributes - Its attributes.
|
|
3993
|
+
* @param name - The repeat attribute, which is absent for an element that appears once.
|
|
3994
|
+
* @param row - The 1-based row, for the error.
|
|
3995
|
+
* @returns The count, at least one.
|
|
3996
|
+
* @throws ArchiveError when the attribute is not a positive whole number.
|
|
3997
|
+
*/
|
|
3998
|
+
function repeats(attributes, name, row) {
|
|
3999
|
+
const stated = attributes[name];
|
|
4000
|
+
if (stated === void 0) return 1;
|
|
4001
|
+
if (!/^[1-9]\d*$/.test(stated)) throw new ArchiveError(`row ${String(row)} states ${name} as ${stated}, which is not a positive whole number`);
|
|
4002
|
+
return Number(stated);
|
|
4003
|
+
}
|
|
4004
|
+
/**
|
|
4005
|
+
* Reads one table of an OpenDocument spreadsheet (`.ods`, a zip of XML parts) as the same text the xlsx reader gives (see `valueText`). A cell or row repeated by `table:number-columns-repeated` or `table:number-rows-repeated` is written as often as it repeats when it holds a value; an empty one only moves the position on, so the filler an application writes to the end of the sheet costs nothing. A covered cell (the hidden part of a merged range) is an empty cell.
|
|
4006
|
+
* @param bytes - The spreadsheet.
|
|
4007
|
+
* @param sheet - The table's name.
|
|
4008
|
+
* @returns The table's cells.
|
|
4009
|
+
* @throws ArchiveError when the bytes are not an OpenDocument spreadsheet, the table is missing, or a cell is malformed.
|
|
4010
|
+
*/
|
|
4011
|
+
function odsGrid(bytes, sheet) {
|
|
4012
|
+
let content;
|
|
4013
|
+
try {
|
|
4014
|
+
content = getOwn(unzipSync(bytes, { filter: (file) => file.name === CONTENT_PART }), CONTENT_PART);
|
|
4015
|
+
} catch (error) {
|
|
4016
|
+
throw new ArchiveError(`not a valid OpenDocument spreadsheet: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
4017
|
+
}
|
|
4018
|
+
if (content === void 0) throw new ArchiveError(`the file is not an OpenDocument spreadsheet: it has no ${CONTENT_PART}`);
|
|
4019
|
+
const xml = new TextDecoder("utf-8", { fatal: true }).decode(content);
|
|
4020
|
+
const names = [];
|
|
4021
|
+
let table;
|
|
4022
|
+
for (const match of xml.matchAll(/<table:table(?=[\s>])([^>]*)>([\s\S]*?)<\/table:table>/g)) {
|
|
4023
|
+
const name = attributesOf(match[1] ?? "")["table:name"] ?? "";
|
|
4024
|
+
names.push(name);
|
|
4025
|
+
if (name === sheet) {
|
|
4026
|
+
table = match[2] ?? "";
|
|
4027
|
+
break;
|
|
4028
|
+
}
|
|
4029
|
+
}
|
|
4030
|
+
if (table === void 0) throw new ArchiveError(`the spreadsheet has no table ${sheet}; it has ${names.join(", ")}`);
|
|
4031
|
+
const rows = /* @__PURE__ */ new Map();
|
|
4032
|
+
let width = 0;
|
|
4033
|
+
let next = 1;
|
|
4034
|
+
for (const row of table.matchAll(/<table:table-row(?=[\s/>])([^>]*?)(?:\/>|>([\s\S]*?)<\/table:table-row>)/g)) {
|
|
4035
|
+
const repeated = repeats(attributesOf(row[1] ?? ""), "table:number-rows-repeated", next);
|
|
4036
|
+
const cells = [];
|
|
4037
|
+
let column = 0;
|
|
4038
|
+
for (const cell of (row[2] ?? "").matchAll(/<table:(?:covered-)?table-cell(?=[\s/>])([^>]*?)(?:\/>|>([\s\S]*?)<\/table:(?:covered-)?table-cell>)/g)) {
|
|
4039
|
+
const attributes = attributesOf(cell[1] ?? "");
|
|
4040
|
+
const span = repeats(attributes, "table:number-columns-repeated", next);
|
|
4041
|
+
const text = valueText(attributes, cell[2] ?? "", next);
|
|
4042
|
+
if (text !== "") {
|
|
4043
|
+
for (let copy = 0; copy < span; copy += 1) cells[column + copy] = text;
|
|
4044
|
+
width = Math.max(width, column + span);
|
|
4045
|
+
}
|
|
4046
|
+
column += span;
|
|
4047
|
+
}
|
|
4048
|
+
if (cells.length > 0) for (let copy = 0; copy < repeated; copy += 1) rows.set(next + copy, cells);
|
|
4049
|
+
next += repeated;
|
|
4050
|
+
}
|
|
4051
|
+
return {
|
|
4052
|
+
rows,
|
|
4053
|
+
width
|
|
4054
|
+
};
|
|
4055
|
+
}
|
|
4056
|
+
//#endregion
|
|
4057
|
+
//#region src/node/sheet.ts
|
|
4058
|
+
/** What the spreadsheet readers (`xlsx`, `xls` and `ods`) share: the date arithmetic of the Excel serial number, the test for a number format that shows a date, and the CSV a sheet is written as, so the three formats give the table parser the same text for the same cells. */
|
|
4059
|
+
const SECONDS_PER_DAY = 86400;
|
|
4060
|
+
const MILLISECONDS = 1e3;
|
|
4061
|
+
/** The serial number Excel gives its 29th of February 1900, a day that never existed: serials above it count from 30 December 1899 and serials below it from 31 December 1899. */
|
|
4062
|
+
const LEAP_BUG_SERIAL = 60;
|
|
4063
|
+
/** The day a serial number counts from: 1 January 1904 in a workbook that says so, otherwise 31 December 1899 for the serials before Excel's false 29th of February 1900 and 30 December 1899 for those after it. */
|
|
4064
|
+
const EPOCH_1904 = "1904-01-01T00:00:00Z";
|
|
4065
|
+
const EPOCH_BEFORE_LEAP_BUG = "1899-12-31T00:00:00Z";
|
|
4066
|
+
const EPOCH_AFTER_LEAP_BUG = "1899-12-30T00:00:00Z";
|
|
4067
|
+
const DATE_LENGTH = 10;
|
|
4068
|
+
const DATE_TIME_LENGTH = 19;
|
|
4069
|
+
const SECONDS_PER_MINUTE = 60;
|
|
4070
|
+
const MINUTES_PER_HOUR = 60;
|
|
3631
4071
|
/** The built-in number format ids that show a date or a time, as inclusive ranges (ECMA-376, 18.8.30): the short and long dates and times, the locale dates, and the elapsed and minute times. */
|
|
3632
4072
|
const BUILT_IN_DATE_FORMATS = [
|
|
3633
4073
|
{
|
|
@@ -3656,24 +4096,6 @@ function isDateFormat(code) {
|
|
|
3656
4096
|
return /[ymdhs]/i.test(code.replaceAll(/"[^"]*"|\\.|\[[^\]]*\]/g, ""));
|
|
3657
4097
|
}
|
|
3658
4098
|
/**
|
|
3659
|
-
* Which cell styles show dates, by the style's index.
|
|
3660
|
-
* @param xml - The `styles.xml` text, or undefined when the workbook has none.
|
|
3661
|
-
* @returns The style indexes whose number format is a date or a time.
|
|
3662
|
-
*/
|
|
3663
|
-
function dateStyles(xml) {
|
|
3664
|
-
if (xml === void 0) return /* @__PURE__ */ new Set();
|
|
3665
|
-
const custom = /* @__PURE__ */ new Map();
|
|
3666
|
-
for (const format of elements(xml, "numFmt")) if (format["numFmtId"] !== void 0 && format["formatCode"] !== void 0) custom.set(Number(format["numFmtId"]), format["formatCode"]);
|
|
3667
|
-
const cellFormats = /<cellXfs(?:\s[^>]*)?>([\s\S]*?)<\/cellXfs>/.exec(xml)?.[1] ?? "";
|
|
3668
|
-
const styles = /* @__PURE__ */ new Set();
|
|
3669
|
-
for (const [index, format] of elements(cellFormats, "xf").entries()) {
|
|
3670
|
-
const id = Number(format["numFmtId"] ?? "0");
|
|
3671
|
-
const code = custom.get(id);
|
|
3672
|
-
if (code === void 0 ? BUILT_IN_DATE_FORMATS.some((range) => id >= range.first && id <= range.last) : isDateFormat(code)) styles.add(index);
|
|
3673
|
-
}
|
|
3674
|
-
return styles;
|
|
3675
|
-
}
|
|
3676
|
-
/**
|
|
3677
4099
|
* A date serial number as ISO 8601 text: a date when it has no time of day, a date and time when it has, and a time alone for a serial below one day in a 1900 workbook.
|
|
3678
4100
|
* @param serial - The serial number, with the time of day as its fraction.
|
|
3679
4101
|
* @param date1904 - Whether the workbook counts days from 1 January 1904.
|
|
@@ -3692,6 +4114,630 @@ function isoOf(serial, date1904) {
|
|
|
3692
4114
|
return new Date(Date.parse(date1904 ? EPOCH_1904 : whole < LEAP_BUG_SERIAL ? EPOCH_BEFORE_LEAP_BUG : EPOCH_AFTER_LEAP_BUG) + (whole * SECONDS_PER_DAY + seconds) * MILLISECONDS).toISOString().slice(0, seconds === 0 ? DATE_LENGTH : DATE_TIME_LENGTH);
|
|
3693
4115
|
}
|
|
3694
4116
|
/**
|
|
4117
|
+
* Writes a sheet's cells as CSV text, so a recipe reads every spreadsheet format with the table parser: one line for every row number from the first to the last row that holds a value (a row with no value before it is an empty line, so the lines above the header are the rows an editor shows, and the formatted but empty rows a sheet carries after its data are left out), every line the width of the widest row.
|
|
4118
|
+
* @param grid - The sheet's cells.
|
|
4119
|
+
* @returns The CSV, in UTF-8.
|
|
4120
|
+
*/
|
|
4121
|
+
function gridToCsv(grid) {
|
|
4122
|
+
let last = 0;
|
|
4123
|
+
for (const [number, cells] of grid.rows) if (cells.length > 0) last = Math.max(last, number);
|
|
4124
|
+
const lines = [];
|
|
4125
|
+
for (let number = 1; number <= last; number += 1) {
|
|
4126
|
+
const cells = grid.rows.get(number) ?? [];
|
|
4127
|
+
lines.push(Array.from({ length: grid.width }, (_empty, column) => csvField(cells[column] ?? "")).join(","));
|
|
4128
|
+
}
|
|
4129
|
+
return new TextEncoder().encode(lines.length === 0 ? "" : `${lines.join("\n")}\n`);
|
|
4130
|
+
}
|
|
4131
|
+
//#endregion
|
|
4132
|
+
//#region src/node/cfb.ts
|
|
4133
|
+
/** The signature every Compound File Binary file opens with ([MS-CFB] 2.2), the bytes D0 CF 11 E0 A1 B1 1A E1 read as one little-endian 64-bit number. */
|
|
4134
|
+
const SIGNATURE = 16220472316735377360n;
|
|
4135
|
+
const HEADER_BYTES = 512;
|
|
4136
|
+
/** The sector numbers that mark the end of a chain and a free sector; every regular sector number is below the first of them ([MS-CFB] 2.1). */
|
|
4137
|
+
const MAX_REGULAR_SECTOR = 4294967290;
|
|
4138
|
+
const END_OF_CHAIN = 4294967294;
|
|
4139
|
+
const FREE_SECTOR = 4294967295;
|
|
4140
|
+
/** The sibling or child id that names no directory entry. */
|
|
4141
|
+
const NO_STREAM = 4294967295;
|
|
4142
|
+
/** The DIFAT entries the header holds before the first DIFAT sector. */
|
|
4143
|
+
const HEADER_DIFAT_ENTRIES = 109;
|
|
4144
|
+
const HEADER_DIFAT_OFFSET = 76;
|
|
4145
|
+
const DIRECTORY_ENTRY_BYTES = 128;
|
|
4146
|
+
const DIRECTORY_NAME_BYTES = 64;
|
|
4147
|
+
const ENTRY_BYTES = 4;
|
|
4148
|
+
const STREAM_OBJECT = 2;
|
|
4149
|
+
const ROOT_OBJECT = 5;
|
|
4150
|
+
/** The header fields this reader uses, by byte offset ([MS-CFB] 2.2). */
|
|
4151
|
+
const FIELD = {
|
|
4152
|
+
sectorShift: 30,
|
|
4153
|
+
miniSectorShift: 32,
|
|
4154
|
+
fatSectors: 44,
|
|
4155
|
+
firstDirectorySector: 48,
|
|
4156
|
+
miniStreamCutoff: 56,
|
|
4157
|
+
firstMiniFatSector: 60,
|
|
4158
|
+
miniFatSectors: 64,
|
|
4159
|
+
firstDifatSector: 68,
|
|
4160
|
+
difatSectors: 72
|
|
4161
|
+
};
|
|
4162
|
+
/** The directory entry fields this reader uses, by byte offset within the entry ([MS-CFB] 2.6.1). */
|
|
4163
|
+
const ENTRY = {
|
|
4164
|
+
nameLength: 64,
|
|
4165
|
+
type: 66,
|
|
4166
|
+
left: 68,
|
|
4167
|
+
right: 72,
|
|
4168
|
+
child: 76,
|
|
4169
|
+
start: 116,
|
|
4170
|
+
size: 120
|
|
4171
|
+
};
|
|
4172
|
+
/** The sector sizes the format allows, as their power of two: 512 bytes for a version 3 file, 4096 for version 4. */
|
|
4173
|
+
const SECTOR_SHIFTS = {
|
|
4174
|
+
version3: 9,
|
|
4175
|
+
version4: 12
|
|
4176
|
+
};
|
|
4177
|
+
const MINI_SECTOR_SHIFT = 6;
|
|
4178
|
+
/**
|
|
4179
|
+
* Reads a stream that lives at the root of a Compound File Binary file (the container of the legacy Office formats, an xls workbook among them), following its sector chain in the file allocation table, or in the mini stream when it is shorter than the file's cutoff.
|
|
4180
|
+
* @param bytes - The file.
|
|
4181
|
+
* @param name - The stream's name, compared without regard to case as the format compares names.
|
|
4182
|
+
* @returns The stream's bytes, or undefined when the root storage holds no stream of that name.
|
|
4183
|
+
* @throws ArchiveError when the bytes are not a compound file, or a header field, a sector chain or a directory entry is malformed.
|
|
4184
|
+
*/
|
|
4185
|
+
function readRootStream(bytes, name) {
|
|
4186
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
4187
|
+
if (bytes.length < HEADER_BYTES || view.getBigUint64(0, true) !== SIGNATURE) throw new ArchiveError("the file is not a compound file: it does not open with the compound file signature");
|
|
4188
|
+
const u32 = (offset) => view.getUint32(offset, true);
|
|
4189
|
+
const sectorShift = view.getUint16(FIELD.sectorShift, true);
|
|
4190
|
+
if (sectorShift !== SECTOR_SHIFTS.version3 && sectorShift !== SECTOR_SHIFTS.version4 || view.getUint16(FIELD.miniSectorShift, true) !== MINI_SECTOR_SHIFT) throw new ArchiveError(`the compound file states a sector size of 2^${String(sectorShift)} bytes, which the format does not allow`);
|
|
4191
|
+
const sectorBytes = 2 ** sectorShift;
|
|
4192
|
+
const sectorCount = Math.floor((bytes.length - sectorBytes) / sectorBytes) + 1;
|
|
4193
|
+
const sector = (index) => {
|
|
4194
|
+
if (index >= MAX_REGULAR_SECTOR || index >= sectorCount) throw new ArchiveError(`the compound file names sector ${String(index)}, which it does not have`);
|
|
4195
|
+
const start = (index + 1) * sectorBytes;
|
|
4196
|
+
return bytes.subarray(start, Math.min(start + sectorBytes, bytes.length));
|
|
4197
|
+
};
|
|
4198
|
+
const fatSectors = [];
|
|
4199
|
+
for (let index = 0; index < HEADER_DIFAT_ENTRIES; index += 1) fatSectors.push(u32(HEADER_DIFAT_OFFSET + index * ENTRY_BYTES));
|
|
4200
|
+
const perDifatSector = sectorBytes / ENTRY_BYTES - 1;
|
|
4201
|
+
let difat = u32(FIELD.firstDifatSector);
|
|
4202
|
+
for (let read = 0; read < u32(FIELD.difatSectors); read += 1) {
|
|
4203
|
+
const difatSector = sector(difat);
|
|
4204
|
+
const entries = new DataView(difatSector.buffer, difatSector.byteOffset, difatSector.byteLength);
|
|
4205
|
+
for (let index = 0; index < perDifatSector; index += 1) fatSectors.push(entries.getUint32(index * ENTRY_BYTES, true));
|
|
4206
|
+
difat = entries.getUint32(perDifatSector * ENTRY_BYTES, true);
|
|
4207
|
+
}
|
|
4208
|
+
const fat = concatenateEntries(fatSectors.slice(0, u32(FIELD.fatSectors)).map(sector));
|
|
4209
|
+
const fatLimit = Math.min(fat.length, sectorCount);
|
|
4210
|
+
const chain = (start, table, limit) => {
|
|
4211
|
+
const sectors = [];
|
|
4212
|
+
let current = start;
|
|
4213
|
+
while (current !== END_OF_CHAIN) {
|
|
4214
|
+
const next = table[current];
|
|
4215
|
+
if (current >= limit || next === void 0 || sectors.length >= limit) throw new ArchiveError(`a sector chain of the compound file runs to ${current === FREE_SECTOR ? "a free sector" : `sector ${String(current)}`} instead of ending`);
|
|
4216
|
+
sectors.push(current);
|
|
4217
|
+
current = next;
|
|
4218
|
+
}
|
|
4219
|
+
return sectors;
|
|
4220
|
+
};
|
|
4221
|
+
const regularStream = (start, size) => {
|
|
4222
|
+
const joined = concatenate(chain(start, fat, fatLimit).map(sector));
|
|
4223
|
+
if (joined.length < size) throw new ArchiveError(`a stream of the compound file states ${String(size)} bytes but its sectors hold ${String(joined.length)}`);
|
|
4224
|
+
return joined.subarray(0, size);
|
|
4225
|
+
};
|
|
4226
|
+
const directory = concatenate(chain(u32(FIELD.firstDirectorySector), fat, fatLimit).map(sector));
|
|
4227
|
+
const entry = (id) => {
|
|
4228
|
+
const offset = id * DIRECTORY_ENTRY_BYTES;
|
|
4229
|
+
if (offset + DIRECTORY_ENTRY_BYTES > directory.length) throw new ArchiveError(`the compound file names directory entry ${String(id)}, which it does not have`);
|
|
4230
|
+
const fields = new DataView(directory.buffer, directory.byteOffset + offset, DIRECTORY_ENTRY_BYTES);
|
|
4231
|
+
const nameLength = fields.getUint16(ENTRY.nameLength, true);
|
|
4232
|
+
if (nameLength > DIRECTORY_NAME_BYTES || nameLength % 2 !== 0) throw new ArchiveError(`directory entry ${String(id)} of the compound file states a name of ${String(nameLength)} bytes`);
|
|
4233
|
+
return {
|
|
4234
|
+
name: new TextDecoder("utf-16le").decode(directory.subarray(offset, offset + Math.max(0, nameLength - 2))),
|
|
4235
|
+
type: fields.getUint8(ENTRY.type),
|
|
4236
|
+
left: fields.getUint32(ENTRY.left, true),
|
|
4237
|
+
right: fields.getUint32(ENTRY.right, true),
|
|
4238
|
+
child: fields.getUint32(ENTRY.child, true),
|
|
4239
|
+
start: fields.getUint32(ENTRY.start, true),
|
|
4240
|
+
size: fields.getUint32(ENTRY.size, true)
|
|
4241
|
+
};
|
|
4242
|
+
};
|
|
4243
|
+
const root = entry(0);
|
|
4244
|
+
if (root.type !== ROOT_OBJECT) throw new ArchiveError("the first directory entry of the compound file is not its root storage");
|
|
4245
|
+
const wanted = name.toUpperCase();
|
|
4246
|
+
const pending = root.child === NO_STREAM ? [] : [root.child];
|
|
4247
|
+
const visited = /* @__PURE__ */ new Set();
|
|
4248
|
+
let found;
|
|
4249
|
+
while (pending.length > 0 && found === void 0) {
|
|
4250
|
+
const id = pending.pop();
|
|
4251
|
+
if (id === void 0 || visited.has(id)) throw new ArchiveError("the directory tree of the compound file has a cycle");
|
|
4252
|
+
visited.add(id);
|
|
4253
|
+
const candidate = entry(id);
|
|
4254
|
+
if (candidate.type === STREAM_OBJECT && candidate.name.toUpperCase() === wanted) found = candidate;
|
|
4255
|
+
for (const sibling of [candidate.left, candidate.right]) if (sibling !== NO_STREAM) pending.push(sibling);
|
|
4256
|
+
}
|
|
4257
|
+
if (found === void 0) return;
|
|
4258
|
+
if (found.size >= u32(FIELD.miniStreamCutoff)) return regularStream(found.start, found.size);
|
|
4259
|
+
const miniStream = regularStream(root.start, root.size);
|
|
4260
|
+
const miniFat = u32(FIELD.miniFatSectors) === 0 ? /* @__PURE__ */ new Uint32Array() : concatenateEntries(chain(u32(FIELD.firstMiniFatSector), fat, fatLimit).map(sector));
|
|
4261
|
+
const miniBytes = 2 ** MINI_SECTOR_SHIFT;
|
|
4262
|
+
const joined = concatenate(chain(found.start, miniFat, Math.min(miniFat.length, Math.floor(miniStream.length / miniBytes))).map((index) => miniStream.subarray(index * miniBytes, (index + 1) * miniBytes)));
|
|
4263
|
+
if (joined.length < found.size) throw new ArchiveError(`a stream of the compound file states ${String(found.size)} bytes but its mini sectors hold ${String(joined.length)}`);
|
|
4264
|
+
return joined.subarray(0, found.size);
|
|
4265
|
+
}
|
|
4266
|
+
/**
|
|
4267
|
+
* Joins byte runs.
|
|
4268
|
+
* @param parts - The runs, in order.
|
|
4269
|
+
* @returns One array holding them all.
|
|
4270
|
+
*/
|
|
4271
|
+
function concatenate(parts) {
|
|
4272
|
+
const joined = new Uint8Array(parts.reduce((total, part) => total + part.length, 0));
|
|
4273
|
+
let offset = 0;
|
|
4274
|
+
for (const part of parts) {
|
|
4275
|
+
joined.set(part, offset);
|
|
4276
|
+
offset += part.length;
|
|
4277
|
+
}
|
|
4278
|
+
return joined;
|
|
4279
|
+
}
|
|
4280
|
+
/**
|
|
4281
|
+
* Reads sectors of an allocation table as one list of little-endian 32-bit entries.
|
|
4282
|
+
* @param sectors - The table's sectors, in order.
|
|
4283
|
+
* @returns The entries.
|
|
4284
|
+
*/
|
|
4285
|
+
function concatenateEntries(sectors) {
|
|
4286
|
+
const joined = concatenate(sectors);
|
|
4287
|
+
const view = new DataView(joined.buffer, joined.byteOffset, joined.byteLength);
|
|
4288
|
+
return Uint32Array.from({ length: Math.floor(joined.length / ENTRY_BYTES) }, (_unused, index) => view.getUint32(index * ENTRY_BYTES, true));
|
|
4289
|
+
}
|
|
4290
|
+
//#endregion
|
|
4291
|
+
//#region src/node/xls.ts
|
|
4292
|
+
/** The BIFF record types this reader acts on ([MS-XLS] 2.3), by name. */
|
|
4293
|
+
const RECORD = {
|
|
4294
|
+
bof: 2057,
|
|
4295
|
+
eof: 10,
|
|
4296
|
+
filePass: 47,
|
|
4297
|
+
dateMode: 34,
|
|
4298
|
+
format: 1054,
|
|
4299
|
+
xf: 224,
|
|
4300
|
+
boundSheet: 133,
|
|
4301
|
+
sst: 252,
|
|
4302
|
+
continue: 60,
|
|
4303
|
+
labelSst: 253,
|
|
4304
|
+
label: 516,
|
|
4305
|
+
richString: 214,
|
|
4306
|
+
number: 515,
|
|
4307
|
+
rk: 638,
|
|
4308
|
+
mulRk: 189,
|
|
4309
|
+
boolErr: 517,
|
|
4310
|
+
formula: 6,
|
|
4311
|
+
string: 519,
|
|
4312
|
+
array: 545,
|
|
4313
|
+
sharedFormula: 1212,
|
|
4314
|
+
table: 566
|
|
4315
|
+
};
|
|
4316
|
+
/** The records that may stand between a formula and the STRING record holding its text result: the array, shared formula or data table the formula belongs to ([MS-XLS] 2.1.7.20.6, the CELL rule). */
|
|
4317
|
+
const FORMULA_COMPANIONS = /* @__PURE__ */ new Set([
|
|
4318
|
+
RECORD.array,
|
|
4319
|
+
RECORD.sharedFormula,
|
|
4320
|
+
RECORD.table
|
|
4321
|
+
]);
|
|
4322
|
+
/** The BOF version of a BIFF8 stream, the format of Excel 97 to 2003; earlier BIFF versions store text in a code page and lay their records out differently. */
|
|
4323
|
+
const BIFF8 = 1536;
|
|
4324
|
+
const RECORD_HEADER_BYTES = 4;
|
|
4325
|
+
/** The widths of the unsigned integers a string's header holds. */
|
|
4326
|
+
const U8 = 1;
|
|
4327
|
+
const U16 = 2;
|
|
4328
|
+
const U32 = 4;
|
|
4329
|
+
/** The flag bits of a BIFF8 Unicode string ([MS-XLS] 2.5.293 and 2.5.294): two-byte characters, phonetic data after the runs, and formatting runs. */
|
|
4330
|
+
const HIGH_BYTE = 1;
|
|
4331
|
+
const EXTENDED = 4;
|
|
4332
|
+
const RICH = 8;
|
|
4333
|
+
const FORMATTING_RUN_BYTES = 4;
|
|
4334
|
+
/** A FORMULA record's cached result is not a number when its last two bytes are all ones; its first byte then says what it is ([MS-XLS] 2.5.133). */
|
|
4335
|
+
const NON_NUMERIC_RESULT = 65535;
|
|
4336
|
+
const FORMULA_RESULT = {
|
|
4337
|
+
string: 0,
|
|
4338
|
+
boolean: 1,
|
|
4339
|
+
error: 2,
|
|
4340
|
+
empty: 3
|
|
4341
|
+
};
|
|
4342
|
+
/** The error values a cell can hold, with the code BIFF stores for each ([MS-XLS] 2.5.97.2). */
|
|
4343
|
+
const ERRORS = [
|
|
4344
|
+
{
|
|
4345
|
+
code: 0,
|
|
4346
|
+
text: "#NULL!"
|
|
4347
|
+
},
|
|
4348
|
+
{
|
|
4349
|
+
code: 7,
|
|
4350
|
+
text: "#DIV/0!"
|
|
4351
|
+
},
|
|
4352
|
+
{
|
|
4353
|
+
code: 15,
|
|
4354
|
+
text: "#VALUE!"
|
|
4355
|
+
},
|
|
4356
|
+
{
|
|
4357
|
+
code: 23,
|
|
4358
|
+
text: "#REF!"
|
|
4359
|
+
},
|
|
4360
|
+
{
|
|
4361
|
+
code: 29,
|
|
4362
|
+
text: "#NAME?"
|
|
4363
|
+
},
|
|
4364
|
+
{
|
|
4365
|
+
code: 36,
|
|
4366
|
+
text: "#NUM!"
|
|
4367
|
+
},
|
|
4368
|
+
{
|
|
4369
|
+
code: 42,
|
|
4370
|
+
text: "#N/A"
|
|
4371
|
+
},
|
|
4372
|
+
{
|
|
4373
|
+
code: 43,
|
|
4374
|
+
text: "#GETTING_DATA"
|
|
4375
|
+
}
|
|
4376
|
+
];
|
|
4377
|
+
const HEXADECIMAL = 16;
|
|
4378
|
+
/** The byte offsets of the fields this reader uses: a cell record's style after its row and column, the unique string count in the SST header, and the high half of a double, little-endian. */
|
|
4379
|
+
const STYLE_OFFSET = 4;
|
|
4380
|
+
const SST_UNIQUE_OFFSET = 4;
|
|
4381
|
+
const DOUBLE_HIGH_HALF = 4;
|
|
4382
|
+
/** An RK number's flags ([MS-XLS] 2.5.217): its value is a hundredth of what is stored, and what is stored is a signed integer rather than the high bits of a double. */
|
|
4383
|
+
const RK_HUNDREDTHS = 1;
|
|
4384
|
+
const RK_INTEGER = 2;
|
|
4385
|
+
const HUNDRED = 100;
|
|
4386
|
+
const RK_INTEGER_SHIFT = 2;
|
|
4387
|
+
const RK_DOUBLE_MASK = 4294967292;
|
|
4388
|
+
const MULRK_ENTRY_BYTES = 6;
|
|
4389
|
+
/** A MULRK's row and first column before its values, and its last column after them. */
|
|
4390
|
+
const MULRK_FIRST_ENTRY = 4;
|
|
4391
|
+
const MULRK_FIXED_BYTES = 6;
|
|
4392
|
+
/** A cell record's row, column and cell style, the first six bytes of every cell record. */
|
|
4393
|
+
const CELL_HEADER_BYTES = 6;
|
|
4394
|
+
const DOUBLE_BYTES = 8;
|
|
4395
|
+
const BOOLERR_VALUE_OFFSET = 6;
|
|
4396
|
+
const BOOLERR_KIND_OFFSET = 7;
|
|
4397
|
+
const SHEET_NAME_OFFSET = 6;
|
|
4398
|
+
const SST_STRINGS_OFFSET = 8;
|
|
4399
|
+
/** The records that hold a cell's value; a BLANK or MULBLANK holds only a style, so an empty cell, and is not among them. */
|
|
4400
|
+
const CELL_RECORDS = /* @__PURE__ */ new Set([
|
|
4401
|
+
RECORD.labelSst,
|
|
4402
|
+
RECORD.label,
|
|
4403
|
+
RECORD.richString,
|
|
4404
|
+
RECORD.number,
|
|
4405
|
+
RECORD.rk,
|
|
4406
|
+
RECORD.mulRk,
|
|
4407
|
+
RECORD.boolErr,
|
|
4408
|
+
RECORD.formula
|
|
4409
|
+
]);
|
|
4410
|
+
/**
|
|
4411
|
+
* A boolean cell's text.
|
|
4412
|
+
* @param value - The stored byte: zero for false.
|
|
4413
|
+
* @returns TRUE or FALSE, as the xlsx reader writes them.
|
|
4414
|
+
*/
|
|
4415
|
+
function booleanText(value) {
|
|
4416
|
+
return value === 0 ? "FALSE" : "TRUE";
|
|
4417
|
+
}
|
|
4418
|
+
/**
|
|
4419
|
+
* An error cell's text.
|
|
4420
|
+
* @param code - The stored error code.
|
|
4421
|
+
* @param row - The 0-based row, for the error.
|
|
4422
|
+
* @returns The error as a spreadsheet shows it, such as `#N/A`.
|
|
4423
|
+
* @throws ArchiveError for a code BIFF does not define.
|
|
4424
|
+
*/
|
|
4425
|
+
function errorText(code, row) {
|
|
4426
|
+
const text = ERRORS.find((error) => error.code === code)?.text;
|
|
4427
|
+
if (text === void 0) throw new ArchiveError(`row ${String(row + 1)} holds the error code 0x${code.toString(HEXADECIMAL)}, which BIFF does not define`);
|
|
4428
|
+
return text;
|
|
4429
|
+
}
|
|
4430
|
+
/**
|
|
4431
|
+
* Splits a BIFF stream into records.
|
|
4432
|
+
* @param stream - The stream, or the part of it from a substream's BOF.
|
|
4433
|
+
* @returns The records, in order.
|
|
4434
|
+
* @throws ArchiveError when a record runs past the end of the stream.
|
|
4435
|
+
*/
|
|
4436
|
+
function* records(stream) {
|
|
4437
|
+
const view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength);
|
|
4438
|
+
for (let offset = 0; offset + RECORD_HEADER_BYTES <= stream.length;) {
|
|
4439
|
+
const type = view.getUint16(offset, true);
|
|
4440
|
+
const length = view.getUint16(offset + 2, true);
|
|
4441
|
+
const start = offset + RECORD_HEADER_BYTES;
|
|
4442
|
+
if (start + length > stream.length) throw new ArchiveError(`the workbook's record of type 0x${type.toString(HEXADECIMAL)} at byte ${String(offset)} runs past the end of its stream`);
|
|
4443
|
+
const body = stream.subarray(start, start + length);
|
|
4444
|
+
yield {
|
|
4445
|
+
type,
|
|
4446
|
+
body,
|
|
4447
|
+
view: new DataView(body.buffer, body.byteOffset, body.byteLength)
|
|
4448
|
+
};
|
|
4449
|
+
offset = start + length;
|
|
4450
|
+
}
|
|
4451
|
+
}
|
|
4452
|
+
/** Reads the characters of BIFF8 strings across a record and the CONTINUE records after it: a string whose characters cross into a CONTINUE record starts that record with a fresh flag byte saying how wide the rest of its characters are, while the other parts of a string cross without one ([MS-XLS] 2.5.293). */
|
|
4453
|
+
var StringReader = class {
|
|
4454
|
+
parts;
|
|
4455
|
+
part = 0;
|
|
4456
|
+
offset;
|
|
4457
|
+
constructor(parts, start) {
|
|
4458
|
+
this.parts = parts;
|
|
4459
|
+
this.offset = start;
|
|
4460
|
+
}
|
|
4461
|
+
/** @returns The current part, moving to the next one when the current one is spent. */
|
|
4462
|
+
current() {
|
|
4463
|
+
let body = this.parts[this.part];
|
|
4464
|
+
while (body !== void 0 && this.offset >= body.length) {
|
|
4465
|
+
this.part += 1;
|
|
4466
|
+
this.offset = 0;
|
|
4467
|
+
body = this.parts[this.part];
|
|
4468
|
+
}
|
|
4469
|
+
if (body === void 0) throw new ArchiveError("a string of the workbook runs past the end of its records");
|
|
4470
|
+
return body;
|
|
4471
|
+
}
|
|
4472
|
+
/**
|
|
4473
|
+
* Reads bytes that are not characters, which may cross from one part to the next.
|
|
4474
|
+
* @param count - How many.
|
|
4475
|
+
* @returns The bytes.
|
|
4476
|
+
*/
|
|
4477
|
+
bytes(count) {
|
|
4478
|
+
const out = new Uint8Array(count);
|
|
4479
|
+
for (let filled = 0; filled < count;) {
|
|
4480
|
+
const body = this.current();
|
|
4481
|
+
const taken = Math.min(count - filled, body.length - this.offset);
|
|
4482
|
+
out.set(body.subarray(this.offset, this.offset + taken), filled);
|
|
4483
|
+
filled += taken;
|
|
4484
|
+
this.offset += taken;
|
|
4485
|
+
}
|
|
4486
|
+
return out;
|
|
4487
|
+
}
|
|
4488
|
+
/**
|
|
4489
|
+
* Reads a little-endian unsigned integer.
|
|
4490
|
+
* @param width - Its width in bytes.
|
|
4491
|
+
* @returns The integer.
|
|
4492
|
+
*/
|
|
4493
|
+
uint(width) {
|
|
4494
|
+
const raw = this.bytes(width);
|
|
4495
|
+
const view = new DataView(raw.buffer);
|
|
4496
|
+
return width === U8 ? view.getUint8(0) : width === U16 ? view.getUint16(0, true) : view.getUint32(0, true);
|
|
4497
|
+
}
|
|
4498
|
+
/**
|
|
4499
|
+
* Reads characters, taking a fresh flag byte at the start of each later part they cross into.
|
|
4500
|
+
* @param count - How many characters.
|
|
4501
|
+
* @param high - Whether the characters start as two bytes each.
|
|
4502
|
+
* @returns The text.
|
|
4503
|
+
*/
|
|
4504
|
+
characters(count, high) {
|
|
4505
|
+
let text = "";
|
|
4506
|
+
let wide = high;
|
|
4507
|
+
for (let left = count; left > 0;) {
|
|
4508
|
+
let body = this.parts[this.part];
|
|
4509
|
+
if (body !== void 0 && this.offset >= body.length) {
|
|
4510
|
+
this.part += 1;
|
|
4511
|
+
this.offset = 0;
|
|
4512
|
+
wide = (this.uint(U8) & HIGH_BYTE) !== 0;
|
|
4513
|
+
}
|
|
4514
|
+
body = this.current();
|
|
4515
|
+
const width = wide ? 2 : 1;
|
|
4516
|
+
const taken = Math.min(left, Math.floor((body.length - this.offset) / width));
|
|
4517
|
+
if (taken === 0) throw new ArchiveError("a character of a workbook string is split across two records");
|
|
4518
|
+
const run = body.subarray(this.offset, this.offset + taken * width);
|
|
4519
|
+
text += wide ? new TextDecoder("utf-16le").decode(run) : String.fromCharCode(...run);
|
|
4520
|
+
this.offset += taken * width;
|
|
4521
|
+
left -= taken;
|
|
4522
|
+
}
|
|
4523
|
+
return text;
|
|
4524
|
+
}
|
|
4525
|
+
/**
|
|
4526
|
+
* Reads one string: its character count (one or two bytes wide), its flags, the counts of any formatting runs and phonetic data, its characters, and then skips the runs and the phonetic data.
|
|
4527
|
+
* @param countWidth - The width of its character count.
|
|
4528
|
+
* @returns The text.
|
|
4529
|
+
*/
|
|
4530
|
+
string(countWidth) {
|
|
4531
|
+
this.current();
|
|
4532
|
+
const count = this.uint(countWidth);
|
|
4533
|
+
const flags = this.uint(U8);
|
|
4534
|
+
const runs = (flags & RICH) === 0 ? 0 : this.uint(U16);
|
|
4535
|
+
const extended = (flags & EXTENDED) === 0 ? 0 : this.uint(U32);
|
|
4536
|
+
const text = this.characters(count, (flags & HIGH_BYTE) !== 0);
|
|
4537
|
+
this.bytes(runs * FORMATTING_RUN_BYTES + extended);
|
|
4538
|
+
return text;
|
|
4539
|
+
}
|
|
4540
|
+
};
|
|
4541
|
+
/**
|
|
4542
|
+
* Reads the workbook globals substream, up to its EOF.
|
|
4543
|
+
* @param stream - The Workbook stream.
|
|
4544
|
+
* @returns The globals.
|
|
4545
|
+
* @throws ArchiveError when the stream is not BIFF8, is encrypted or its shared string table is malformed.
|
|
4546
|
+
*/
|
|
4547
|
+
function readGlobals(stream) {
|
|
4548
|
+
const iterator = records(stream);
|
|
4549
|
+
const first = iterator.next();
|
|
4550
|
+
if (first.done === true || first.value.type !== RECORD.bof || first.value.body.length < 2) throw new ArchiveError("the workbook stream does not open with a BOF record");
|
|
4551
|
+
const version = first.value.view.getUint16(0, true);
|
|
4552
|
+
if (version !== BIFF8) throw new ArchiveError(`the workbook is BIFF version 0x${version.toString(HEXADECIMAL)}; only BIFF8 (Excel 97 to 2003) is read`);
|
|
4553
|
+
const formats = /* @__PURE__ */ new Map();
|
|
4554
|
+
const styleFormats = [];
|
|
4555
|
+
const sheets = /* @__PURE__ */ new Map();
|
|
4556
|
+
let strings = [];
|
|
4557
|
+
let date1904 = false;
|
|
4558
|
+
let sst;
|
|
4559
|
+
const finishSst = () => {
|
|
4560
|
+
if (sst === void 0) return;
|
|
4561
|
+
const [head] = sst;
|
|
4562
|
+
if (head === void 0 || head.length < SST_STRINGS_OFFSET) throw new ArchiveError("the workbook's shared string table is shorter than its header");
|
|
4563
|
+
const unique = new DataView(head.buffer, head.byteOffset, head.byteLength).getUint32(SST_UNIQUE_OFFSET, true);
|
|
4564
|
+
const reader = new StringReader(sst, SST_STRINGS_OFFSET);
|
|
4565
|
+
strings = Array.from({ length: unique }, () => reader.string(U16));
|
|
4566
|
+
sst = void 0;
|
|
4567
|
+
};
|
|
4568
|
+
for (const record of iterator) {
|
|
4569
|
+
if (record.type !== RECORD.continue) finishSst();
|
|
4570
|
+
if (record.type === RECORD.eof) break;
|
|
4571
|
+
if (record.type === RECORD.filePass) throw new ArchiveError("the workbook is encrypted");
|
|
4572
|
+
if (record.type === RECORD.dateMode) date1904 = record.view.getUint16(0, true) === 1;
|
|
4573
|
+
else if (record.type === RECORD.format) formats.set(record.view.getUint16(0, true), new StringReader([record.body], 2).string(U16));
|
|
4574
|
+
else if (record.type === RECORD.xf) styleFormats.push(record.view.getUint16(2, true));
|
|
4575
|
+
else if (record.type === RECORD.boundSheet) sheets.set(new StringReader([record.body], SHEET_NAME_OFFSET).string(U8), record.view.getUint32(0, true));
|
|
4576
|
+
else if (record.type === RECORD.sst) sst = [record.body];
|
|
4577
|
+
else if (record.type === RECORD.continue && sst !== void 0) sst.push(record.body);
|
|
4578
|
+
}
|
|
4579
|
+
finishSst();
|
|
4580
|
+
const dateStyles = /* @__PURE__ */ new Set();
|
|
4581
|
+
for (const [index, id] of styleFormats.entries()) {
|
|
4582
|
+
const code = formats.get(id);
|
|
4583
|
+
if (code === void 0 ? BUILT_IN_DATE_FORMATS.some((range) => id >= range.first && id <= range.last) : isDateFormat(code)) dateStyles.add(index);
|
|
4584
|
+
}
|
|
4585
|
+
return {
|
|
4586
|
+
strings,
|
|
4587
|
+
dateStyles,
|
|
4588
|
+
date1904,
|
|
4589
|
+
sheets
|
|
4590
|
+
};
|
|
4591
|
+
}
|
|
4592
|
+
/**
|
|
4593
|
+
* Decodes an RK number ([MS-XLS] 2.5.217): a signed 30-bit integer or the high 30 bits of a double, either of them possibly a hundredth of the value.
|
|
4594
|
+
* @param rk - The stored 32 bits.
|
|
4595
|
+
* @returns The number.
|
|
4596
|
+
*/
|
|
4597
|
+
function rkValue(rk) {
|
|
4598
|
+
let value;
|
|
4599
|
+
if ((rk & RK_INTEGER) === 0) {
|
|
4600
|
+
const bits = /* @__PURE__ */ new DataView(/* @__PURE__ */ new ArrayBuffer(DOUBLE_BYTES));
|
|
4601
|
+
bits.setUint32(DOUBLE_HIGH_HALF, (rk & RK_DOUBLE_MASK) >>> 0, true);
|
|
4602
|
+
value = bits.getFloat64(0, true);
|
|
4603
|
+
} else value = rk >> RK_INTEGER_SHIFT;
|
|
4604
|
+
return (rk & RK_HUNDREDTHS) === 0 ? value : value / HUNDRED;
|
|
4605
|
+
}
|
|
4606
|
+
/**
|
|
4607
|
+
* Reads one sheet of a legacy binary Excel workbook (`.xls`, BIFF8 in a compound file) as the same text the xlsx reader gives: a date cell as ISO 8601 text (recognised by its cell style's number format, in either date system), a boolean as TRUE or FALSE, an error as its text, a number as the shortest decimal that is the stored double, and a formula as its cached result.
|
|
4608
|
+
* @param bytes - The workbook.
|
|
4609
|
+
* @param sheet - The sheet's name.
|
|
4610
|
+
* @returns The sheet's cells.
|
|
4611
|
+
* @throws ArchiveError when the bytes are not a BIFF8 workbook, it is encrypted, the sheet is missing, or a record is malformed.
|
|
4612
|
+
*/
|
|
4613
|
+
function xlsGrid(bytes, sheet) {
|
|
4614
|
+
const stream = readRootStream(bytes, "Workbook");
|
|
4615
|
+
if (stream === void 0) throw new ArchiveError(readRootStream(bytes, "Book") === void 0 ? "the compound file holds no Workbook stream, so it is not an Excel workbook" : "the workbook is BIFF5 or earlier (Excel 95), which stores text in a code page; only BIFF8 (Excel 97 to 2003) is read");
|
|
4616
|
+
const globals = readGlobals(stream);
|
|
4617
|
+
const start = globals.sheets.get(sheet);
|
|
4618
|
+
if (start === void 0) throw new ArchiveError(`the workbook has no sheet ${sheet}; it has ${[...globals.sheets.keys()].join(", ")}`);
|
|
4619
|
+
if (start >= stream.length) throw new ArchiveError(`the workbook places the sheet ${sheet} at byte ${String(start)}, past the end of its stream`);
|
|
4620
|
+
const rows = /* @__PURE__ */ new Map();
|
|
4621
|
+
let width = 0;
|
|
4622
|
+
const put = (row, column, text) => {
|
|
4623
|
+
if (text === "") return;
|
|
4624
|
+
const cells = rows.get(row + 1) ?? [];
|
|
4625
|
+
cells[column] = text;
|
|
4626
|
+
rows.set(row + 1, cells);
|
|
4627
|
+
width = Math.max(width, column + 1);
|
|
4628
|
+
};
|
|
4629
|
+
const numberText = (value, style, row) => {
|
|
4630
|
+
if (!globals.dateStyles.has(style)) return String(value);
|
|
4631
|
+
if (!Number.isFinite(value) || value < 0) throw new ArchiveError(`row ${String(row + 1)} has a date cell holding ${String(value)}`);
|
|
4632
|
+
return isoOf(value, globals.date1904);
|
|
4633
|
+
};
|
|
4634
|
+
let depth = 0;
|
|
4635
|
+
let pendingString;
|
|
4636
|
+
const finishString = () => {
|
|
4637
|
+
if (pendingString !== void 0) {
|
|
4638
|
+
put(pendingString.row, pendingString.column, new StringReader(pendingString.parts, 0).string(U16));
|
|
4639
|
+
pendingString = void 0;
|
|
4640
|
+
}
|
|
4641
|
+
};
|
|
4642
|
+
for (const record of records(stream.subarray(start))) {
|
|
4643
|
+
if (pendingString !== void 0) {
|
|
4644
|
+
if (pendingString.parts.length === 0) {
|
|
4645
|
+
if (FORMULA_COMPANIONS.has(record.type)) continue;
|
|
4646
|
+
if (record.type !== RECORD.string) throw new ArchiveError(`row ${String(pendingString.row + 1)} has a formula with a text result but no STRING record after it`);
|
|
4647
|
+
pendingString.parts.push(record.body);
|
|
4648
|
+
continue;
|
|
4649
|
+
}
|
|
4650
|
+
if (record.type === RECORD.continue) {
|
|
4651
|
+
pendingString.parts.push(record.body);
|
|
4652
|
+
continue;
|
|
4653
|
+
}
|
|
4654
|
+
finishString();
|
|
4655
|
+
}
|
|
4656
|
+
if (record.type === RECORD.bof) {
|
|
4657
|
+
depth += 1;
|
|
4658
|
+
continue;
|
|
4659
|
+
}
|
|
4660
|
+
if (record.type === RECORD.eof) {
|
|
4661
|
+
depth -= 1;
|
|
4662
|
+
if (depth === 0) break;
|
|
4663
|
+
continue;
|
|
4664
|
+
}
|
|
4665
|
+
if (record.type === RECORD.string && depth === 1) throw new ArchiveError("the sheet has a STRING record that follows no formula");
|
|
4666
|
+
if (depth !== 1 || !CELL_RECORDS.has(record.type)) continue;
|
|
4667
|
+
if (record.body.length < CELL_HEADER_BYTES) throw new ArchiveError(`the sheet has a cell record of type 0x${record.type.toString(HEXADECIMAL)} shorter than a cell's row, column and style`);
|
|
4668
|
+
const row = record.view.getUint16(0, true);
|
|
4669
|
+
const column = record.view.getUint16(2, true);
|
|
4670
|
+
const style = record.view.getUint16(STYLE_OFFSET, true);
|
|
4671
|
+
if (record.type === RECORD.labelSst) {
|
|
4672
|
+
const index = record.view.getUint32(CELL_HEADER_BYTES, true);
|
|
4673
|
+
const text = globals.strings[index];
|
|
4674
|
+
if (text === void 0) throw new ArchiveError(`row ${String(row + 1)} names the shared string ${String(index)}, which the workbook does not have`);
|
|
4675
|
+
put(row, column, text);
|
|
4676
|
+
} else if (record.type === RECORD.label || record.type === RECORD.richString) put(row, column, new StringReader([record.body], CELL_HEADER_BYTES).string(U16));
|
|
4677
|
+
else if (record.type === RECORD.number) put(row, column, numberText(record.view.getFloat64(CELL_HEADER_BYTES, true), style, row));
|
|
4678
|
+
else if (record.type === RECORD.rk) put(row, column, numberText(rkValue(record.view.getUint32(CELL_HEADER_BYTES, true)), style, row));
|
|
4679
|
+
else if (record.type === RECORD.mulRk) {
|
|
4680
|
+
const count = (record.body.length - MULRK_FIXED_BYTES) / MULRK_ENTRY_BYTES;
|
|
4681
|
+
if (!Number.isInteger(count)) throw new ArchiveError(`row ${String(row + 1)} has a MULRK record whose length is not a whole number of values`);
|
|
4682
|
+
for (let index = 0; index < count; index += 1) {
|
|
4683
|
+
const offset = MULRK_FIRST_ENTRY + index * MULRK_ENTRY_BYTES;
|
|
4684
|
+
put(row, column + index, numberText(rkValue(record.view.getUint32(offset + 2, true)), record.view.getUint16(offset, true), row));
|
|
4685
|
+
}
|
|
4686
|
+
} else if (record.type === RECORD.boolErr) {
|
|
4687
|
+
const value = record.view.getUint8(BOOLERR_VALUE_OFFSET);
|
|
4688
|
+
put(row, column, record.view.getUint8(BOOLERR_KIND_OFFSET) === 0 ? booleanText(value) : errorText(value, row));
|
|
4689
|
+
} else if (record.view.getUint16(12, true) === NON_NUMERIC_RESULT) {
|
|
4690
|
+
const kind = record.view.getUint8(CELL_HEADER_BYTES);
|
|
4691
|
+
const value = record.view.getUint8(8);
|
|
4692
|
+
if (kind === FORMULA_RESULT.string) pendingString = {
|
|
4693
|
+
row,
|
|
4694
|
+
column,
|
|
4695
|
+
parts: []
|
|
4696
|
+
};
|
|
4697
|
+
else if (kind === FORMULA_RESULT.boolean) put(row, column, booleanText(value));
|
|
4698
|
+
else if (kind === FORMULA_RESULT.error) put(row, column, errorText(value, row));
|
|
4699
|
+
else if (kind !== FORMULA_RESULT.empty) throw new ArchiveError(`row ${String(row + 1)} has a formula whose cached result is of unknown kind ${String(kind)}`);
|
|
4700
|
+
} else put(row, column, numberText(record.view.getFloat64(CELL_HEADER_BYTES, true), style, row));
|
|
4701
|
+
}
|
|
4702
|
+
finishString();
|
|
4703
|
+
return {
|
|
4704
|
+
rows,
|
|
4705
|
+
width
|
|
4706
|
+
};
|
|
4707
|
+
}
|
|
4708
|
+
//#endregion
|
|
4709
|
+
//#region src/node/xlsx.ts
|
|
4710
|
+
const LETTERS = 26;
|
|
4711
|
+
const FIRST_LETTER = "A".charCodeAt(0);
|
|
4712
|
+
/**
|
|
4713
|
+
* The strings of a workbook's shared string table, in index order: each item's text runs joined, the phonetic runs of East Asian text left out.
|
|
4714
|
+
* @param xml - The `sharedStrings.xml` text.
|
|
4715
|
+
* @returns The strings.
|
|
4716
|
+
*/
|
|
4717
|
+
function sharedStrings(xml) {
|
|
4718
|
+
return [...xml.matchAll(/<si(?:\s[^>]*)?>([\s\S]*?)<\/si>/g)].map((item) => {
|
|
4719
|
+
return [...(item[1] ?? "").replaceAll(/<rPh[\s\S]*?<\/rPh>/g, "").matchAll(/<t(?:\s[^>]*)?>([\s\S]*?)<\/t>/g)].map((text) => unescape(text[1] ?? "")).join("");
|
|
4720
|
+
});
|
|
4721
|
+
}
|
|
4722
|
+
/**
|
|
4723
|
+
* Which cell styles show dates, by the style's index.
|
|
4724
|
+
* @param xml - The `styles.xml` text, or undefined when the workbook has none.
|
|
4725
|
+
* @returns The style indexes whose number format is a date or a time.
|
|
4726
|
+
*/
|
|
4727
|
+
function dateStyles(xml) {
|
|
4728
|
+
if (xml === void 0) return /* @__PURE__ */ new Set();
|
|
4729
|
+
const custom = /* @__PURE__ */ new Map();
|
|
4730
|
+
for (const format of elements(xml, "numFmt")) if (format["numFmtId"] !== void 0 && format["formatCode"] !== void 0) custom.set(Number(format["numFmtId"]), format["formatCode"]);
|
|
4731
|
+
const cellFormats = /<cellXfs(?:\s[^>]*)?>([\s\S]*?)<\/cellXfs>/.exec(xml)?.[1] ?? "";
|
|
4732
|
+
const styles = /* @__PURE__ */ new Set();
|
|
4733
|
+
for (const [index, format] of elements(cellFormats, "xf").entries()) {
|
|
4734
|
+
const id = Number(format["numFmtId"] ?? "0");
|
|
4735
|
+
const code = custom.get(id);
|
|
4736
|
+
if (code === void 0 ? BUILT_IN_DATE_FORMATS.some((range) => id >= range.first && id <= range.last) : isDateFormat(code)) styles.add(index);
|
|
4737
|
+
}
|
|
4738
|
+
return styles;
|
|
4739
|
+
}
|
|
4740
|
+
/**
|
|
3695
4741
|
* The 0-based column of a cell reference such as `AB12`.
|
|
3696
4742
|
* @param reference - The reference.
|
|
3697
4743
|
* @returns The column index.
|
|
@@ -3785,13 +4831,13 @@ function sheetNames(bytes) {
|
|
|
3785
4831
|
});
|
|
3786
4832
|
}
|
|
3787
4833
|
/**
|
|
3788
|
-
*
|
|
4834
|
+
* Reads one sheet of an xlsx workbook: a date cell as ISO 8601 text (recognised by its cell style's number format, in either date system), a boolean as TRUE or FALSE, a number as the text the workbook stores, and a formula as its stored result.
|
|
3789
4835
|
* @param bytes - The workbook.
|
|
3790
4836
|
* @param sheet - The sheet's name.
|
|
3791
|
-
* @returns The
|
|
4837
|
+
* @returns The sheet's cells.
|
|
3792
4838
|
* @throws ArchiveError when the bytes are not a workbook, the sheet is missing, or a cell cannot be read.
|
|
3793
4839
|
*/
|
|
3794
|
-
function
|
|
4840
|
+
function xlsxGrid(bytes, sheet) {
|
|
3795
4841
|
let parts;
|
|
3796
4842
|
try {
|
|
3797
4843
|
const wanted = /* @__PURE__ */ new Set([
|
|
@@ -3815,14 +4861,7 @@ function sheetToCsv(bytes, sheet) {
|
|
|
3815
4861
|
const sheetBytes = getOwn(unzipSync(bytes, { filter: (file) => file.name === path }), path);
|
|
3816
4862
|
if (sheetBytes === void 0) throw new ArchiveError(`the workbook names the sheet ${sheet} in ${path}, which the file does not have`);
|
|
3817
4863
|
const strings = text("xl/sharedStrings.xml");
|
|
3818
|
-
|
|
3819
|
-
const last = Math.max(0, ...rows.keys());
|
|
3820
|
-
const lines = [];
|
|
3821
|
-
for (let number = 1; number <= last; number += 1) {
|
|
3822
|
-
const cells = rows.get(number) ?? [];
|
|
3823
|
-
lines.push(Array.from({ length: width }, (_empty, column) => csvField(cells[column] ?? "")).join(","));
|
|
3824
|
-
}
|
|
3825
|
-
return new TextEncoder().encode(lines.length === 0 ? "" : `${lines.join("\n")}\n`);
|
|
4864
|
+
return readRows(new TextDecoder("utf-8").decode(sheetBytes), strings === void 0 ? [] : sharedStrings(strings), dateStyles(text("xl/styles.xml")), /<workbookPr(?=[\s/>])[^>]*date1904="(?:1|true)"/.test(workbook));
|
|
3826
4865
|
}
|
|
3827
4866
|
//#endregion
|
|
3828
4867
|
//#region src/node/archive.ts
|
|
@@ -3882,7 +4921,7 @@ function pick(paths, member) {
|
|
|
3882
4921
|
return only;
|
|
3883
4922
|
}
|
|
3884
4923
|
/**
|
|
3885
|
-
* Unpacks downloaded bytes into the one file the parser reads. A member with an `inner` archive is unpacked again by that layout, and the path reported is the outer member's, joined by `/` to the inner member's when the inner layout selects one, as though the inner archive were a directory.
|
|
4924
|
+
* Unpacks downloaded bytes into the one file the parser reads. A member with an `inner` archive is unpacked again by that layout, and the path reported is the outer member's, joined by `/` to the inner member's or the inner sheet's name when the inner layout selects one, as though the inner archive were a directory.
|
|
3886
4925
|
* @param bytes - The downloaded bytes.
|
|
3887
4926
|
* @param archive - The recipe's archive layout.
|
|
3888
4927
|
* @returns The file's bytes, and its member path for a multi-file archive.
|
|
@@ -3892,8 +4931,8 @@ function unpack(bytes, archive) {
|
|
|
3892
4931
|
if (archive.kind === "none") return { bytes };
|
|
3893
4932
|
if (archive.kind === "gzip") return { bytes: gunzip(bytes) };
|
|
3894
4933
|
if (archive.kind === "bzip2") return { bytes: bunzip2(bytes) };
|
|
3895
|
-
if (
|
|
3896
|
-
bytes: sheetToCsv(bytes, archive
|
|
4934
|
+
if ("sheet" in archive) return {
|
|
4935
|
+
bytes: sheetToCsv(bytes, archive),
|
|
3897
4936
|
member: archive.sheet
|
|
3898
4937
|
};
|
|
3899
4938
|
if (archive.kind === "shapefile") return unpackShapefile(bytes, archive.member);
|
|
@@ -3903,6 +4942,10 @@ function unpack(bytes, archive) {
|
|
|
3903
4942
|
bytes: gunzip(outer.bytes),
|
|
3904
4943
|
member: outer.member
|
|
3905
4944
|
};
|
|
4945
|
+
if ("sheet" in archive.inner) return {
|
|
4946
|
+
bytes: sheetToCsv(outer.bytes, archive.inner),
|
|
4947
|
+
member: `${outer.member}/${archive.inner.sheet}`
|
|
4948
|
+
};
|
|
3906
4949
|
const inner = selectMember(outer.bytes, archive.inner);
|
|
3907
4950
|
return {
|
|
3908
4951
|
bytes: inner.bytes,
|
|
@@ -4007,7 +5050,10 @@ function unpackMembers(bytes, archive) {
|
|
|
4007
5050
|
if (names.length === 0) throw new ArchiveError(`member ${member.path} has no sheet matching ${sheets ?? ""}`);
|
|
4008
5051
|
return names.map((sheet) => ({
|
|
4009
5052
|
path: `${member.path}/${sheet}`,
|
|
4010
|
-
bytes: sheetToCsv(member.bytes,
|
|
5053
|
+
bytes: sheetToCsv(member.bytes, {
|
|
5054
|
+
kind: "xlsx",
|
|
5055
|
+
sheet
|
|
5056
|
+
})
|
|
4011
5057
|
}));
|
|
4012
5058
|
});
|
|
4013
5059
|
}
|
|
@@ -4082,6 +5128,45 @@ function untarred(bytes, kind) {
|
|
|
4082
5128
|
return kind === "tar.gz" ? gunzip(bytes) : bunzip2(bytes);
|
|
4083
5129
|
}
|
|
4084
5130
|
/**
|
|
5131
|
+
* Reads one sheet of a spreadsheet as CSV text, by the spreadsheet's format.
|
|
5132
|
+
* @param bytes - The spreadsheet.
|
|
5133
|
+
* @param archive - Its format and the sheet to read.
|
|
5134
|
+
* @returns The CSV, in UTF-8.
|
|
5135
|
+
* @throws ArchiveError when the bytes are not a spreadsheet of that format, the sheet is missing, or a cell cannot be read.
|
|
5136
|
+
*/
|
|
5137
|
+
function sheetToCsv(bytes, archive) {
|
|
5138
|
+
const read = {
|
|
5139
|
+
xlsx: xlsxGrid,
|
|
5140
|
+
xls: xlsGrid,
|
|
5141
|
+
ods: odsGrid
|
|
5142
|
+
}[archive.kind];
|
|
5143
|
+
return gridToCsv(read(bytes, archive.sheet));
|
|
5144
|
+
}
|
|
5145
|
+
/**
|
|
5146
|
+
* Unpacks downloaded bytes into every file a multi-file format's selector matches, gunzipping each when the layout's `inner` says so. The paths stay the archive's own, so a gunzipped `edge.csv.gz` keeps its name.
|
|
5147
|
+
* @param bytes - The downloaded bytes.
|
|
5148
|
+
* @param archive - The recipe's archive layout, which selects files with `files`.
|
|
5149
|
+
* @returns Each matching file, in path order.
|
|
5150
|
+
* @throws ArchiveError when the bytes, or a file `inner` gunzips, are not well-formed, or the selector matches no member.
|
|
5151
|
+
*/
|
|
5152
|
+
function unpackFiles(bytes, archive) {
|
|
5153
|
+
const files = unpackMembers(bytes, {
|
|
5154
|
+
kind: archive.kind,
|
|
5155
|
+
members: archive.files
|
|
5156
|
+
});
|
|
5157
|
+
if (archive.inner === void 0) return files;
|
|
5158
|
+
return files.map((file) => {
|
|
5159
|
+
try {
|
|
5160
|
+
return {
|
|
5161
|
+
path: file.path,
|
|
5162
|
+
bytes: gunzip(file.bytes)
|
|
5163
|
+
};
|
|
5164
|
+
} catch (error) {
|
|
5165
|
+
throw new ArchiveError(`member ${file.path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
5166
|
+
}
|
|
5167
|
+
});
|
|
5168
|
+
}
|
|
5169
|
+
/**
|
|
4085
5170
|
* Gunzips bytes, rethrowing zlib's failure in the package's taxonomy.
|
|
4086
5171
|
* @param bytes - Gzip data.
|
|
4087
5172
|
* @returns The decompressed bytes.
|
|
@@ -4620,6 +5705,20 @@ async function runToNative(dataset, options) {
|
|
|
4620
5705
|
provenance: provenanceOf({ members: parts.map((part) => part.path) })
|
|
4621
5706
|
};
|
|
4622
5707
|
}
|
|
5708
|
+
if ("files" in archive) {
|
|
5709
|
+
const files = await step(entry.id, "extract", () => unpackFiles(bytes, archive));
|
|
5710
|
+
return {
|
|
5711
|
+
collection: false,
|
|
5712
|
+
documents: [{
|
|
5713
|
+
id: void 0,
|
|
5714
|
+
native: {
|
|
5715
|
+
...await step(entry.id, "parse", () => parseNativeFiles(files, recipe.parser)),
|
|
5716
|
+
meta: metaOf(entry)
|
|
5717
|
+
}
|
|
5718
|
+
}],
|
|
5719
|
+
provenance: provenanceOf({ members: files.map((file) => file.path) })
|
|
5720
|
+
};
|
|
5721
|
+
}
|
|
4623
5722
|
const unpacked = await step(entry.id, "extract", () => unpack(bytes, archive));
|
|
4624
5723
|
const parsed = await step(entry.id, "parse", () => parseNative(unpacked.bytes, recipe.parser));
|
|
4625
5724
|
return {
|