keepset 0.16.1 → 0.18.0

package/dist/{categories-BYezdHCk.d.ts → categories-CiJbssql.d.ts}

@@ -153,6 +153,7 @@ declare const ParserSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     column: z.ZodString;
     factor: z.ZodNumber;
   }, z.core.$strict>>;
+  columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strict>], "format">;
 type Parser = z.infer<typeof ParserSchema>;
 /** How to obtain the dataset: a fetchable recipe naming the source (by index into `sources`), the archive layout and the parser, or a statement that no declarative recipe exists yet and why. */
@@ -237,6 +238,7 @@ declare const RecipeSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
       column: z.ZodString;
       factor: z.ZodNumber;
     }, z.core.$strict>>;
+    columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
   }, z.core.$strict>], "format">;
 }, z.core.$strict>, z.ZodObject<{
   status: z.ZodLiteral<"not-fetchable">;
@@ -375,6 +377,7 @@ declare const EntrySchema: z.ZodObject<{
         column: z.ZodString;
         factor: z.ZodNumber;
       }, z.core.$strict>>;
+      columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
     }, z.core.$strict>], "format">;
   }, z.core.$strict>, z.ZodObject<{
     status: z.ZodLiteral<"not-fetchable">;
@@ -464,6 +467,7 @@ declare const EntrySchema: z.ZodObject<{
           column: string;
           factor: number;
         }[];
+        columns?: string[] | undefined;
       } | {
         format: "matrix-market";
         encoding: "utf-8" | "latin1";
@@ -702,6 +706,7 @@ declare const CatalogueIndexSchema: z.ZodObject<{
           column: z.ZodString;
           factor: z.ZodNumber;
         }, z.core.$strict>>;
+        columns: z.ZodOptional<z.ZodArray<z.ZodString>>;
       }, z.core.$strict>], "format">;
     }, z.core.$strict>, z.ZodObject<{
       status: z.ZodLiteral<"not-fetchable">;
@@ -791,6 +796,7 @@ declare const CatalogueIndexSchema: z.ZodObject<{
             column: string;
             factor: number;
           }[];
+          columns?: string[] | undefined;
         } | {
           format: "matrix-market";
           encoding: "utf-8" | "latin1";
@@ -908,6 +914,7 @@ declare const CatalogueIndexSchema: z.ZodObject<{
             column: string;
             factor: number;
           }[];
+          columns?: string[] | undefined;
         } | {
           format: "matrix-market";
           encoding: "utf-8" | "latin1";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as Parser, A as LineageSchema, B as Citation, C as CatalogueIndex, D as NamespaceNode, E as DatasetQuery, F as AliasSchema, G as Entry, H as ColumnNameSchema, I as Archive, J as IdSchema, K as EntrySchema, L as ArchiveMember, M as ManifestSchema, N as SemVerSchema, O as createCatalogue, P as parseManifest, Q as NamespaceSchema, R as ArchiveMemberSchema, S as Catalogue, T as Dataset, U as CrosswalkReference, V as CitationSchema, W as CrosswalkReferenceSchema, X as LicenceSchema, Y as Licence, Z as Namespace, _ as GraphEdgeSchema, a as CategoryTargets, at as Snapshot, b as GraphV1, c as getCategory, ct as SourceSchema, d as listCategories, dt as waybackUrl, et as ParserSchema, f as OrNode, g as GraphEdge, h as OrV1Schema, i as CategoryTarget, it as RelativePathSchema, j as Manifest, k as Lineage, l as getTarget, lt as WaybackTimestampSchema, m as OrV1, n as CategoryId, nt as Recipe, o as CountedKey, ot as SnapshotSchema, p as OrNodeSchema, q as HttpUrlSchema, r as CategoryIdSchema, rt as RecipeSchema, s as countTarget, st as Source, t as Category, tt as RESERVED_META_KEYS, u as kindOf, ut as parseEntry, v as GraphNode, w as CatalogueIndexSchema, x as GraphV1Schema, y as GraphNodeSchema, z as ArchiveSchema } from "./categories-BYezdHCk.js";
+import { $ as Parser, A as LineageSchema, B as Citation, C as CatalogueIndex, D as NamespaceNode, E as DatasetQuery, F as AliasSchema, G as Entry, H as ColumnNameSchema, I as Archive, J as IdSchema, K as EntrySchema, L as ArchiveMember, M as ManifestSchema, N as SemVerSchema, O as createCatalogue, P as parseManifest, Q as NamespaceSchema, R as ArchiveMemberSchema, S as Catalogue, T as Dataset, U as CrosswalkReference, V as CitationSchema, W as CrosswalkReferenceSchema, X as LicenceSchema, Y as Licence, Z as Namespace, _ as GraphEdgeSchema, a as CategoryTargets, at as Snapshot, b as GraphV1, c as getCategory, ct as SourceSchema, d as listCategories, dt as waybackUrl, et as ParserSchema, f as OrNode, g as GraphEdge, h as OrV1Schema, i as CategoryTarget, it as RelativePathSchema, j as Manifest, k as Lineage, l as getTarget, lt as WaybackTimestampSchema, m as OrV1, n as CategoryId, nt as Recipe, o as CountedKey, ot as SnapshotSchema, p as OrNodeSchema, q as HttpUrlSchema, r as CategoryIdSchema, rt as RecipeSchema, s as countTarget, st as Source, t as Category, tt as RESERVED_META_KEYS, u as kindOf, ut as parseEntry, v as GraphNode, w as CatalogueIndexSchema, x as GraphV1Schema, y as GraphNodeSchema, z as ArchiveSchema } from "./categories-CiJbssql.js";
 import { Crosswalk } from "twindex";
 import { z } from "zod";
 //#region src/errors.d.ts

package/dist/index.js

@@ -1,4 +1,4 @@
-import { $ as NativeParseError, A as toIssues, B as kindOf, C as RESERVED_META_KEYS, D as SourceSchema, E as SnapshotSchema, G as GraphNodeSchema, H as OrNodeSchema, I as CategoryIdSchema, J as ArchiveError, K as GraphV1Schema, L as countTarget, M as NAME_SEPARATOR, N as groupsOf, O as WaybackTimestampSchema, P as namesOf, Q as KeepsetError, R as getCategory, S as ParserSchema, T as RelativePathSchema, U as OrV1Schema, V as listCategories, W as GraphEdgeSchema, X as DatasetAmbiguousError, Y as CatalogueSyntaxError, Z as DatasetNotFoundError, _ as EntrySchema, a as CatalogueIndexSchema, b as LicenceSchema, c as ManifestSchema, d as AliasSchema, et as RecipeError, f as ArchiveMemberSchema, g as CrosswalkReferenceSchema, h as ColumnNameSchema, i as listNamespaces, j as waybackUrl, k as parseEntry, l as SemVerSchema, m as CitationSchema, n as getManifest, o as createCatalogue, p as ArchiveSchema, q as JsonScalarSchema, r as listDatasets, s as LineageSchema, t as getDataset, tt as UnknownCategoryError, u as parseManifest, v as HttpUrlSchema, w as RecipeSchema, x as NamespaceSchema, y as IdSchema, z as getTarget } from "./bundled-DPEAZdj9.js";
+import { $ as NativeParseError, A as toIssues, B as kindOf, C as RESERVED_META_KEYS, D as SourceSchema, E as SnapshotSchema, G as GraphNodeSchema, H as OrNodeSchema, I as CategoryIdSchema, J as ArchiveError, K as GraphV1Schema, L as countTarget, M as NAME_SEPARATOR, N as groupsOf, O as WaybackTimestampSchema, P as namesOf, Q as KeepsetError, R as getCategory, S as ParserSchema, T as RelativePathSchema, U as OrV1Schema, V as listCategories, W as GraphEdgeSchema, X as DatasetAmbiguousError, Y as CatalogueSyntaxError, Z as DatasetNotFoundError, _ as EntrySchema, a as CatalogueIndexSchema, b as LicenceSchema, c as ManifestSchema, d as AliasSchema, et as RecipeError, f as ArchiveMemberSchema, g as CrosswalkReferenceSchema, h as ColumnNameSchema, i as listNamespaces, j as waybackUrl, k as parseEntry, l as SemVerSchema, m as CitationSchema, n as getManifest, o as createCatalogue, p as ArchiveSchema, q as JsonScalarSchema, r as listDatasets, s as LineageSchema, t as getDataset, tt as UnknownCategoryError, u as parseManifest, v as HttpUrlSchema, w as RecipeSchema, x as NamespaceSchema, y as IdSchema, z as getTarget } from "./bundled-BiuG_xLd.js";
 import { validateCrosswalk } from "twindex";
 //#region src/serialise.ts
 /**

package/dist/node/fetch-dataset.d.ts

@@ -1,4 +1,4 @@
-import { G as Entry, T as Dataset, at as Snapshot, i as CategoryTarget, st as Source } from "../categories-BYezdHCk.js";
+import { G as Entry, T as Dataset, at as Snapshot, i as CategoryTarget, st as Source } from "../categories-CiJbssql.js";
 //#region src/node/fetch-dataset.d.ts
 /** The network port a recipe downloads through: `downloadRaw` by default, which returns the bytes exactly as served, or a stand-in under test. */
 export type FetchPort = (url: string, init: {

package/dist/node/fetch-dataset.js

@@ -1,4 +1,4 @@
-import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-DPEAZdj9.js";
+import { $ as NativeParseError, F as getOwn, J as ArchiveError, L as countTarget, Q as KeepsetError, et as RecipeError, t as getDataset, z as getTarget } from "../bundled-BiuG_xLd.js";
 import { RoundTripMismatchError, assertRoundTrip, encode } from "twindex";
 import { createHash } from "node:crypto";
 import { gunzipSync } from "node:zlib";
@@ -388,76 +388,164 @@ function columnNames(cells, idColumn, line) {
 	}
 	return names;
 }
+/** The code unit of each character the scanner stops at. */
+const QUOTE = "\"".charCodeAt(0);
+const LINE_FEED = "\n".charCodeAt(0);
+const CARRIAGE_RETURN = "\r".charCodeAt(0);
+/** Scans a table file's text as it arrives in pieces, holding what a record in progress needs between them: the fields and field so far, whether a quoted field is open, and the line the record began on and the line the scan has reached. */
+var RecordScanner = class {
+	delimiterCode;
+	fields = [];
+	field = "";
+	quoted = false;
+	started;
+	line;
+	constructor(delimiterCode, firstLine) {
+		this.delimiterCode = delimiterCode;
+		this.started = firstLine;
+		this.line = firstLine;
+	}
+	/**
+	* Scans text, returning the records it completes.
+	* @param text - The unscanned text.
+	* @param final - Whether no more text follows, so a character that needs the next one to be understood can be taken as it stands.
+	* @returns The records the text completes, and the unscanned end of it, which starts at a character that needs the next one when more text may follow.
+	* @throws NativeParseError when a quote opens inside an unquoted field.
+	*/
+	scan(text, final) {
+		const completed = [];
+		let index = 0;
+		while (index < text.length) {
+			if (this.quoted) {
+				const close = text.indexOf("\"", index);
+				const end = close === -1 ? text.length : close;
+				const run = text.slice(index, end);
+				for (let at = run.indexOf("\n"); at !== -1; at = run.indexOf("\n", at + 1)) this.line += 1;
+				this.field += run;
+				index = end;
+				if (close === -1) break;
+				if (index + 1 >= text.length && !final) return {
+					completed,
+					carry: text.slice(index)
+				};
+				if (text.charCodeAt(index + 1) === QUOTE) {
+					this.field += "\"";
+					index += 2;
+				} else {
+					this.quoted = false;
+					index += 1;
+				}
+				continue;
+			}
+			const code = text.charCodeAt(index);
+			if (code === QUOTE) {
+				if (this.field !== "") throw new NativeParseError("csv-table", this.line, "a quote opens inside an unquoted field");
+				this.quoted = true;
+				index += 1;
+			} else if (code === this.delimiterCode) {
+				this.fields.push(this.field);
+				this.field = "";
+				index += 1;
+			} else if (code === LINE_FEED || code === CARRIAGE_RETURN) {
+				if (code === CARRIAGE_RETURN && index + 1 >= text.length && !final) return {
+					completed,
+					carry: text.slice(index)
+				};
+				index += code === CARRIAGE_RETURN && text.charCodeAt(index + 1) === LINE_FEED ? 2 : 1;
+				this.line += 1;
+				completed.push(this.endRecord());
+			} else {
+				let next = index + 1;
+				while (next < text.length) {
+					const following = text.charCodeAt(next);
+					if (following === QUOTE || following === this.delimiterCode || following === LINE_FEED || following === CARRIAGE_RETURN) break;
+					next += 1;
+				}
+				this.field += text.slice(index, next);
+				index = next;
+			}
+		}
+		return {
+			completed,
+			carry: ""
+		};
+	}
+	/**
+	* The record left unfinished when the text ends without a line break, if any.
+	* @returns The record, or undefined when the text ended on a line break.
+	* @throws NativeParseError when a quoted field is never closed.
+	*/
+	finish() {
+		if (this.quoted) throw new NativeParseError("csv-table", this.started, "a quoted field is never closed");
+		return this.field !== "" || this.fields.length > 0 ? this.endRecord() : void 0;
+	}
+	endRecord() {
+		this.fields.push(this.field);
+		const record = {
+			line: this.started,
+			fields: this.fields
+		};
+		this.fields = [];
+		this.field = "";
+		this.started = this.line;
+		return record;
+	}
+};
 /**
-* Splits text into records and fields by RFC 4180: a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR.
-* @param text - The file text, after the lines the recipe skips.
+* Splits text into records and fields by RFC 4180 as the text arrives in pieces, one piece's records at a time, so neither the whole file nor every cell is held at once: a field may be quoted, a quoted field may hold the delimiter, line breaks and doubled quotes, and a record ends at an unquoted LF, CRLF or CR. A piece may end anywhere, even between the two quotes of a doubled quote or between a CR and its LF.
+* @param pieces - The file text in order, after the lines the recipe skips.
 * @param delimiter - The field delimiter.
 * @param firstLine - The 1-based line of the file the text starts on.
-* @returns The records, blank lines included as a record of one empty field.
+* @returns The records in order, blank lines included as a record of one empty field.
 * @throws NativeParseError when a quoted field is never closed or a quote opens inside an unquoted field.
 */
-function records(text, delimiter, firstLine) {
-	const out = [];
-	let fields = [];
-	let field = "";
-	let quoted = false;
-	let started = firstLine;
-	let line = firstLine;
-	let index = 0;
-	const endRecord = () => {
-		fields.push(field);
-		out.push({
-			line: started,
-			fields
-		});
-		fields = [];
-		field = "";
-		started = line;
-	};
-	while (index < text.length) {
-		const char = text.charAt(index);
-		if (quoted) {
-			if (char === "\"") {
-				if (text.charAt(index + 1) === "\"") {
-					field += "\"";
-					index += 1;
-				} else quoted = false;
-			} else {
-				if (char === "\n") line += 1;
-				field += char;
-			}
-		} else if (char === "\"") {
-			if (field !== "") throw new NativeParseError("csv-table", line, "a quote opens inside an unquoted field");
-			quoted = true;
-		} else if (char === delimiter) {
-			fields.push(field);
-			field = "";
-		} else if (char === "\n" || char === "\r") {
-			if (char === "\r" && text.charAt(index + 1) === "\n") index += 1;
-			line += 1;
-			endRecord();
-		} else field += char;
-		index += 1;
+function* records(pieces, delimiter, firstLine) {
+	const scanner = new RecordScanner(delimiter.charCodeAt(0), firstLine);
+	let carry = "";
+	for (const piece of pieces) {
+		const scanned = scanner.scan(carry + piece, false);
+		carry = scanned.carry;
+		yield* scanned.completed;
 	}
-	if (quoted) throw new NativeParseError("csv-table", started, "a quoted field is never closed");
-	if (field !== "" || fields.length > 0) endRecord();
-	return out;
+	yield* scanner.scan(carry, true).completed;
+	const last = scanner.finish();
+	if (last !== void 0) yield last;
 }
 /**
-* Drops the first lines of a file.
-* @param text - The file text.
+* Drops the first physical lines of a file given in pieces.
+* @param pieces - The file text in order.
 * @param count - How many physical lines to drop.
-* @returns The rest of the text.
+* @returns The rest of the text, in pieces.
 * @throws NativeParseError when the file has fewer lines than that.
 */
-function afterLines(text, count) {
-	let index = 0;
-	for (let dropped = 0; dropped < count; dropped += 1) {
-		const next = text.slice(index).search(/\n/);
-		if (next === -1) throw new NativeParseError("csv-table", dropped + 1, `the file has fewer than ${String(count)} lines to skip before the header`);
-		index += next + 1;
+function* afterLines(pieces, count) {
+	let remaining = count;
+	for (const piece of pieces) {
+		let rest = piece;
+		while (remaining > 0) {
+			const next = rest.indexOf("\n");
+			if (next === -1) {
+				rest = "";
+				break;
+			}
+			rest = rest.slice(next + 1);
+			remaining -= 1;
+		}
+		if (remaining === 0 && rest !== "") yield rest;
+	}
+	if (remaining > 0) throw new NativeParseError("csv-table", count - remaining + 1, `the file has fewer than ${String(count)} lines to skip before the header`);
+}
+/**
+* The text of a file as its pieces, with a leading byte order mark removed.
+* @param text - The file text, or its pieces in order.
+* @returns The pieces.
+*/
+function* withoutByteOrderMark(text) {
+	let first = true;
+	for (const piece of typeof text === "string" ? [text] : text) {
+		yield first && piece.startsWith(BYTE_ORDER_MARK) ? piece.slice(1) : piece;
+		first = false;
 	}
-	return text.slice(index);
 }
 /**
 * Reads a numeric cell.
@@ -474,44 +562,59 @@ function numeric$1(cell, factor, column, line) {
 	return Number(cell) * factor;
 }
 /**
-* The records of a table file after the lines the recipe skips and without its blank lines: the header first, then the rows.
-* @param text - The file text, which may begin with a byte order mark.
+* The records of a table file after the lines the recipe skips and without its blank lines: the header first, then the rows, one at a time.
+* @param text - The file text, or its pieces in order, which may begin with a byte order mark.
 * @param options - The recipe's table parser, of which the delimiter and the lines to skip are read.
-* @returns The records.
+* @returns Each record.
 * @throws NativeParseError when a quote is never closed or opens inside an unquoted field, or the file has fewer lines than the recipe skips.
 */
-function tableRecords(text, options) {
-	return records(afterLines(text.startsWith(BYTE_ORDER_MARK) ? text.slice(1) : text, options.skipLines), DELIMITERS[options.delimiter], options.skipLines + 1).filter((record) => record.fields.length > 1 || record.fields[0]?.trim() !== "");
+function* tableRecords(text, options) {
+	const body = afterLines(withoutByteOrderMark(text), options.skipLines);
+	for (const record of records(body, DELIMITERS[options.delimiter], options.skipLines + 1)) if (record.fields.length > 1 || record.fields[0]?.trim() !== "") yield record;
 }
 /**
-* Parses a delimited table into rows: the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column, an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
-* @param text - The file text.
+* Parses a delimited table into rows: the first record after `skipLines` lines is the header, each later record is a row, and every row carries every column but the id column (or, when the recipe lists `columns`, exactly those, so a wide table is not held cell for cell), an empty cell being null. Cells are trimmed. A column the recipe declares numeric is read as a decimal number times its factor and any other column stays a string. `id` is the row's id, so a column of that name which is not the id column is named by its position. A row's id is the cell of the recipe's id column, which is then not also a column, or its 1-based row number when the recipe names none. Blank lines are skipped; a row with a different number of fields than the header, two columns with the same name, and an id cell that is empty are errors.
+* @param text - The file text, or its pieces in order.
 * @param options - The recipe's table parser.
 * @returns The rows, in file order.
-* @throws NativeParseError naming the line of the first problem.
+* @throws NativeParseError naming the line of the first problem, including a listed or numeric column the header lacks or a numeric column that is not listed.
 */
 function parseCsvTable(text, options) {
-	const [header, ...rows] = tableRecords(text, options);
+	const table = tableRecords(text, options);
+	const { value: header } = table.next();
 	if (header === void 0) throw new NativeParseError("csv-table", options.skipLines + 1, "the file has no header row");
 	const names = columnNames(header.fields, options.idColumn, header.line);
-	const { idColumn, numericColumns } = options;
-	for (const wanted of [...idColumn === null ? [] : [idColumn], ...numericColumns.map((entry) => entry.column)]) if (!names.includes(wanted)) throw new NativeParseError("csv-table", header.line, `the recipe names the column ${wanted}, which the header does not have`);
+	const { idColumn, numericColumns, columns } = options;
+	const wanted = new Set(columns ?? names.filter((name) => name !== idColumn));
+	for (const named of [
+		...idColumn === null ? [] : [idColumn],
+		...numericColumns.map((entry) => entry.column),
+		...columns ?? []
+	]) if (!names.includes(named)) throw new NativeParseError("csv-table", header.line, `the recipe names the column ${named}, which the header does not have`);
+	if (idColumn !== null && wanted.has(idColumn)) throw new NativeParseError("csv-table", header.line, `the id column ${idColumn} is the row's id, so it is not listed among the columns`);
+	for (const { column } of numericColumns) if (!wanted.has(column)) throw new NativeParseError("csv-table", header.line, `the column ${column} is declared numeric but is not among the columns kept`);
 	const factors = new Map(numericColumns.map((entry) => [entry.column, entry.factor]));
-	return { nodes: rows.map((record, rowIndex) => {
+	const idPosition = idColumn === null ? -1 : names.indexOf(idColumn);
+	const kept = names.flatMap((name, position) => name !== idColumn && wanted.has(name) ? [{
+		name,
+		position,
+		factor: factors.get(name)
+	}] : []);
+	const nodes = [];
+	for (const record of table) {
 		if (record.fields.length !== names.length) throw new NativeParseError("csv-table", record.line, `the row has ${String(record.fields.length)} fields, the header has ${String(names.length)}`);
-		const cells = new Map(names.map((name, position) => [name, (record.fields[position] ?? "").trim()]));
-		const id = idColumn === null ? String(rowIndex + 1) : cells.get(idColumn) ?? "";
+		const id = idColumn === null ? String(nodes.length + 1) : (record.fields[idPosition] ?? "").trim();
 		if (id === "") throw new NativeParseError("csv-table", record.line, `the id column ${idColumn ?? ""} is empty`);
-		const columns = names.filter((name) => name !== idColumn).map((name) => {
-			const cell = cells.get(name) ?? "";
-			const factor = factors.get(name);
+		const cells = kept.map(({ name, position, factor }) => {
+			const cell = (record.fields[position] ?? "").trim();
 			return [name, factor === void 0 ? cell === "" ? null : cell : numeric$1(cell, factor, name, record.line)];
 		});
-		return {
-			...Object.fromEntries(columns),
+		nodes.push({
+			...Object.fromEntries(cells),
 			id
-		};
-	}) };
+		});
+	}
+	return { nodes };
 }
 //#endregion
 //#region src/parsers/pajek.ts
@@ -647,12 +750,39 @@ function decode(bytes, parser) {
 		throw new NativeParseError(parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
 	}
 }
+const KIBIBYTE = 1024;
+/** Bytes decoded at a time for a format that is read in pieces. The decoder holds only a piece's text at once, so this bounds what a large table costs beyond its bytes; it is large enough that the per-piece work is small beside the piece. */
+const DECODE_PIECE_BYTES = 8 * KIBIBYTE * KIBIBYTE;
+/**
+* Decodes a file's bytes in the recipe's encoding a piece at a time, as strictly as `decode`, so a file whose text would not fit in one string can still be read.
+* @param bytes - The unpacked file.
+* @param parser - The recipe's parser, which names the encoding.
+* @param pieceBytes - How many bytes to decode at a time.
+* @returns The text of each piece, in order; a character that spans two pieces is yielded whole with the later one.
+* @throws NativeParseError when the bytes are not valid in the stated encoding.
+*/
+function* decodePieces(bytes, parser, pieceBytes = DECODE_PIECE_BYTES) {
+	const decoder = parser.encoding === "latin1" ? void 0 : new TextDecoder("utf-8", { fatal: true });
+	try {
+		for (let start = 0; start < bytes.length; start += pieceBytes) {
+			const piece = bytes.subarray(start, start + pieceBytes);
+			yield decoder === void 0 ? decode(piece, parser) : decoder.decode(piece, { stream: true });
+		}
+		if (decoder !== void 0) {
+			const rest = decoder.decode();
+			if (rest !== "") yield rest;
+		}
+	} catch (error) {
+		if (error instanceof NativeParseError) throw error;
+		throw new NativeParseError(parser.format, 1, `the file is not valid utf-8 (${error instanceof Error ? error.message : String(error)}); a recipe for a latin1 file states encoding latin1`);
+	}
+}
 function parseNative(bytes, parser) {
+	if (parser.format === "csv-table") return parseCsvTable(decodePieces(bytes, parser), parser);
 	const text = decode(bytes, parser);
 	if (parser.format === "edge-list") return parseEdgeList(text, parser);
 	if (parser.format === "matrix-market") return parseMatrixMarket(text);
 	if (parser.format === "gml") return parseGml(text);
-	if (parser.format === "csv-table") return parseCsvTable(text, parser);
 	return parsePajek(text);
 }
 /** The ustar header layout (POSIX.1-1988, pax(1) "ustar Interchange Format"): each field's byte offset and length. */

package/dist-cli/keepset.mjs

@@ -4,7 +4,7 @@ import { KeepsetError, NAME_SEPARATOR, countTarget, getDataset, listDatasets, li
 import { fetchDataset, roundTripDataset } from "keepset/node";
 import { parseArgs } from "node:util";
 //#region package.json
-var version = "0.16.1";
+var version = "0.18.0";
 const usage = `Usage: keepset <command> [options]
 
 Commands:

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "keepset",
   "description": "Library, CLI and statically generated catalogue for dataset fetch-and-normalise recipes",
-  "version": "0.16.1",
+  "version": "0.18.0",
   "bin": {
     "keepset": "./dist-cli/keepset.mjs"
   },

package/schemas/entry.schema.json

@@ -1 +1 @@
-{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.16.1/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Crosswalk":{"$ref":"https://cdn.jsdelivr.net/npm/twindex@0.1.4/schemas/twindex.schema.json"},"Entry":{"additionalProperties":false,"properties":{"aliases":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\/[A-Za-z0-9][A-Za-z0-9._-]*)*$","type":"string"},"minItems":1,"type":"array"},"category":{"enum":["graph/v1","or/v1"],"type":"string"},"citations":{"items":{"additionalProperties":false,"properties":{"doi":{"pattern":"^10\\.\\d{4,9}\\/\\S+$","type":"string"},"text":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["text"],"type":"object"},"type":"array"},"crosswalk":{"anyOf":[{"$ref":"#/$defs/Crosswalk"},{"additionalProperties":false,"properties":{"file":{"type":"string"}},"required":["file"],"type":"object"}]},"description":{"minLength":1,"type":"string"},"domain":{"minLength":1,"type":"string"},"expectedCounts":{"additionalProperties":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"licence":{"oneOf":[{"additionalProperties":false,"properties":{"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9.+-]*$","type":"string"},"kind":{"const":"spdx","type":"string"}},"required":["kind","id"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"custom","type":"string"},"name":{"minLength":1,"type":"string"},"url":{"format":"uri","type":"string"}},"required":["kind","name","url"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"unrecorded","type":"string"}},"required":["kind"],"type":"object"}]},"name":{"minLength":1,"type":"string"},"namespace":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"properties":{"additionalProperties":{"type":["string","number","boolean","null"]},"propertyNames":{"pattern":"^[A-Za-z][A-Za-z0-9]*$","type":"string"},"type":"object"},"recipe":{"oneOf":[{"additionalProperties":false,"properties":{"archive":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"gzip","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"zip","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"tar.gz","type":"string"},"member":{"anyOf":[{"additionalProperties":false,"properties":{"path":{"type":"string"}},"required":["path"],"type":"object"},{"additionalProperties":false,"properties":{"extensions":{"items":{"pattern":"^\\.[A-Za-z0-9][A-Za-z0-9.]*$","type":"string"},"minItems":1,"type":"array"}},"required":["extensions"],"type":"object"}]}},"required":["kind","member"],"type":"object"}]},"parser":{"oneOf":[{"additionalProperties":false,"properties":{"commentPrefixes":{"items":{"minLength":1,"type":"string"},"type":"array"},"delimiter":{"enum":["detect","whitespace","tab","comma","space"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"edge-list","type":"string"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"sourceColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"targetColumn":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","commentPrefixes","sourceColumn","targetColumn"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"matrix-market","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"gml","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"pajek","type":"string"}},"required":["format","encoding"],"type":"object"},{"additionalProperties":false,"properties":{"columns":{"items":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"type":"array"},"delimiter":{"enum":["comma","tab","semicolon"],"type":"string"},"encoding":{"enum":["utf-8","latin1"],"type":"string"},"format":{"const":"csv-table","type":"string"},"idColumn":{"anyOf":[{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},{"type":"null"}]},"numericColumns":{"items":{"additionalProperties":false,"properties":{"column":{"pattern":"^[A-Za-z](?:[A-Za-z0-9_]*[A-Za-z0-9])?$","type":"string"},"factor":{"exclusiveMinimum":0,"type":"number"}},"required":["column","factor"],"type":"object"},"type":"array"},"skipLines":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["format","encoding","delimiter","skipLines","idColumn","numericColumns"],"type":"object"}]},"source":{"maximum":9007199254740991,"minimum":0,"type":"integer"},"status":{"const":"fetchable","type":"string"}},"required":["status","source","archive","parser"],"type":"object"},{"additionalProperties":false,"properties":{"reason":{"minLength":1,"type":"string"},"status":{"const":"not-fetchable","type":"string"}},"required":["status","reason"],"type":"object"}]},"sources":{"items":{"additionalProperties":false,"properties":{"retrieved":{"format":"date","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$","type":"string"},"snapshot":{"additionalProperties":false,"properties":{"timestamp":{"pattern":"^\\d{8}(?:\\d{6})?$","type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","timestamp"],"type":"object"},"status":{"enum":["live","dead"],"type":"string"},"url":{"format":"uri","type":"string"}},"required":["url","retrieved","status"],"type":"object"},"minItems":1,"type":"array"},"tags":{"items":{"minLength":1,"type":"string"},"type":"array"}},"required":["id","name","tags","category","sources","licence","citations","recipe","crosswalk","expectedCounts","properties"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.18.0/schemas/entry.schema.json","$ref":"#/$defs/Entry","$schema":"https://json-schema.org/draft/2020-12/schema"}

package/schemas/manifest.schema.json

@@ -1 +1 @@
-{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.16.1/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}
+{"$defs":{"Manifest":{"additionalProperties":false,"properties":{"description":{"minLength":1,"type":"string"},"id":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"lineage":{"additionalProperties":false,"properties":{"catalogue":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["catalogue","version","repository"],"type":"object"},"name":{"minLength":1,"type":"string"},"namespaceAliases":{"additionalProperties":{"items":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"minItems":1,"type":"array"},"propertyNames":{"pattern":"^[A-Za-z0-9][A-Za-z0-9._-]*$","type":"string"},"type":"object"},"repository":{"format":"uri","type":"string"},"version":{"pattern":"^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$","type":"string"}},"required":["id","name","description","version","repository"],"type":"object"}},"$id":"https://cdn.jsdelivr.net/npm/keepset@0.18.0/schemas/manifest.schema.json","$ref":"#/$defs/Manifest","$schema":"https://json-schema.org/draft/2020-12/schema"}